naf 1.1.0

data/app/models/process/naf/application.rb

@@ -0,0 +1,164 @@
+require 'log4r/formatter/patternformatter'
+
+module Process::Naf
+  class Application < ::Af::Application
+    class TerminationRequest < StandardError
+      attr_reader :job, :reason
+      def initialize(job, reason)
+        @job = job
+        @reason = reason
+        super("Requested to terminate: #{reason}")
+      end
+    end
+
+    #----------------
+    # *** Options ***
+    #+++++++++++++++++
+
+    opt :naf_job_id,
+        "#{::Naf.schema_name}.historical_jobs.id for communication with scheduling system",
+        env: "NAF_JOB_ID",
+        type: :int
+    opt :do_not_terminate,
+        "refuse to terminate by job and machine IPC mechanics"
+
+    def initialize
+      super
+      opt :log_configuration_files, default: ["af.yml",
+                                              "af-#{Rails.env}.yml",
+                                              "naf.yml",
+                                              "naf-#{Rails.env}.yml",
+                                              "nafjob.yml",
+                                              "nafjob-#{Rails.env}.yml",
+                                              "#{af_name}.yml",
+                                              "#{af_name}-#{Rails.env}.yml"]
+    end
+
+    def database_application_name
+      return "//pid=#{Process.pid}/jid=#{@naf_job_id}/#{af_name}"
+    end
+
+    def fetch_naf_job
+      if @naf_job_id.is_a?(Integer) && @naf_job_id > 0
+        return ::Naf::HistoricalJob.from_partition(@naf_job_id).find(@naf_job_id)
+      end
+      return nil
+    end
+
+    def post_command_line_parsing
+      super
+      Af::Logging::Configurator.log_ignore_configuration = (naf_job_id.blank? && Af::Logging::Configurator.log_console != false)
+    end
+
+    def pre_work
+      set_connection_application_name(database_application_name)
+
+      Log4r::GDC.set(naf_job_id.to_s)
+
+      super
+
+      update_job_status
+    end
+
+    def update_job_status
+      periodic_application_checkpoint
+
+      job = fetch_naf_job
+      if job
+        unless @do_not_terminate
+          if job.request_to_terminate
+            logger.warn "terminating by request"
+            raise TerminationRequest.new(job, "job requested to terminate")
+          end
+          unless job.started_on_machine
+            logger.alarm "terminating: #{job.started_on_machine} is misconfigured"
+            raise TerminationRequest.new(job, "machine not configured correctly")
+          end
+          unless job.started_on_machine.enabled
+            logger.alarm "terminating: #{job.started_on_machine} is disabled"
+            raise TerminationRequest.new(job, "machine disabled")
+          end
+          if job.started_on_machine.marked_down
+            logger.alarm "terminating: #{job.started_on_machine} is marked down"
+            raise TerminationRequest.new(job, "machine marked down")
+          end
+        end
+        if job.log_level != @last_log_level
+          @last_log_level = job.log_level
+          unless @last_log_level.blank?
+            logging_configurator.parse_and_set_logger_levels(@last_log_level)
+          end
+        end
+      end
+    end
+
+    def job_tag_block(*tags, &block)
+      job = fetch_naf_job
+      begin
+        if job
+          add_job_tags(*tags)
+        end
+        yield if block.present?
+      ensure
+        if job
+          remove_job_tags(*tags)
+        end
+      end
+    end
+
+    def update_job_tags(old_tags, new_tags)
+      job = fetch_naf_job
+      if job
+        job.remove_tags(old_tags.map(&:to_s))
+        job.add_tags(new_tags.map(&:to_s))
+      end
+    end
+
+    def add_job_tags(*new_tags)
+      update_job_tags([], new_tags)
+    end
+
+    def remove_job_tags(*old_tags)
+      update_job_tags(old_tags, [])
+    end
+
+    def work
+    end
+
+    module Component
+      def self.included(base)
+        base.send(:include, ::Af::Application::Component)
+        base.extend(ClassMethods)
+      end
+
+      module ClassMethods
+        # XXX not used yet
+      end
+
+      def fetch_naf_job
+        return af_application.try(:fetch_naf_job)
+      end
+
+      def update_job_status
+        return af_application.try(:update_job_status)
+      end
+
+      def job_tag_block(*tags, &block)
+        return af_application.try(:job_tag_block, *tags, &block)
+      end
+
+      def update_job_tags(old_tags, new_tags)
+        return af_application.try(:update_job_tags, old_tags, new_tags)
+      end
+
+      def add_job_tags(*new_tags)
+        return af_application.try(:add_job_tags, *new_tags)
+      end
+
+      def remove_job_tags(*old_tags)
+        return af_application.try(:remove_job_tags, *old_tags)
+      end
+    end
+
+  end
+end

data/app/models/process/naf/janitor.rb

@@ -0,0 +1,117 @@
+module Process::Naf
+  class Janitor < ::Process::Naf::Application
+    ASSIGNMENTS = {
+      creates: ::Naf::JanitorialCreateAssignment,
+      archives: ::Naf::JanitorialArchiveAssignment,
+      drops: ::Naf::JanitorialDropAssignment
+    }
+    ASSIGNMENT_TYPES = ASSIGNMENTS.keys
+
+    #----------------
+    # *** Options ***
+    #+++++++++++++++++
+
+    opt :no_writes,
+        "don't modify database"
+    opt :list_assignments,
+        "list models that would be processed (and exit without processing)",
+        short: :L
+
+    # the following three opt are in specific order because the last sets the default
+    opt :all,
+        "process all models",
+        short: :a,
+        var: :assignments,
+        set: :all
+    opt :assignment,
+        "process specific assignment(s)",
+        short: :j,
+        var: :assignments,
+        type: :ints
+    opt :all_enabled,
+        "process all enabled models",
+        short: :E,
+        var: :assignments,
+        set: :all_enabled,
+        default: :all_enabled
+
+    # the following two opt are in specific order because the last sets the default
+    opt :all_types,
+        "process all assignment types",
+        short: :A,
+        var: :assignment_types,
+        set: ASSIGNMENT_TYPES
+    opt :assignment_type,
+        "process specific assignment types(s)",
+        short: :t,
+        var: :assignment_types,
+        type: :choices,
+        choices: ASSIGNMENT_TYPES,
+        default: ASSIGNMENT_TYPES
+    opt :create_infrastructure,
+        "create infrastructure for a given model",
+        type: :strings
+
+    def assignments_to_process(assignment_type = ::Naf::JanitorialAssignment)
+      if @assignments == :all
+        return assignment_type.all.order('type,assignment_order')
+      elsif @assignments == :all_enabled
+        return assignment_type.where('enabled').order('type,assignment_order')
+      else
+        return [*assignment_type.find(@assignments).order('type,assignment_order')]
+      end
+    end
+
+    def post_command_line_parsing
+      if @list_assignments
+        assignment_list = [["ID", "TYPE", "ORDER", "MODEL"]]
+        [:creates, :archives, :drops].each do |assignment_type_name|
+          unless @assignment_types.include? assignment_type_name
+            next
+          end
+          assignment_type_processor = ASSIGNMENTS[assignment_type_name]
+          assignments_to_process(assignment_type_processor).each do |a|
+            assignment_list << [a.id.to_s, a.type.to_s, a.assignment_order.to_s, a.model_name]
+          end
+        end
+        puts self.class.columnized(assignment_list).join("\n")
+        exit 0
+      end
+      unless @create_infrastructure.blank?
+        logger.info "creating infrastructure: #{@create_infrastructure.inspect}"
+        ::Logical::Naf::CreateInfrastructure.new(@create_infrastructure).work
+        exit 0
+      end
+      super
+    end
+
+    def work
+      logger.info "Janitor STARTED -- let's see what there is to do."
+
+      # assignment types are handled in this order
+      [:creates, :archives, :drops].each do |assignment_type_name|
+        unless @assignment_types.include? assignment_type_name
+          next
+        end
+        assignment_type_processor = ASSIGNMENTS[assignment_type_name]
+        assignments_to_process(assignment_type_processor).each do |assignment|
+          logger.info "#{assignment_type_processor}: #{assignment.model_name} START"
+          target_model = assignment.target_model
+          if target_model.nil?
+            logger.alarm "#{assignment_type_processor}: failed to instantiate target model: #{assignment.model_name}"
+            next
+          end
+          begin
+            assignment_type_processor.new.do_janitorial_work(target_model) unless @no_writes
+          rescue StandardError => e
+            logger.alarm "failed to use #{assignment_type_processor} for partitions model: #{assignment.model_name}"
+            logger.alarm e
+          end
+          logger.info "#{assignment_type_processor}: #{assignment.model_name} DONE"
+        end
+      end
+
+      logger.info "janitor DONE"
+    end
+  end
+end

data/app/models/process/naf/machine_manager.rb

@@ -0,0 +1,150 @@
+module Process::Naf
+  class MachineManager < ::Process::Naf::Application
+    opt :server_name, "set the machines server name (use with --update-machine)", type: :string
+    opt :server_note, "set the machines server note (use with --update-machine)", type: :string
+    opt :server_address, "set the machines server address (use with --update-machine)", default: ::Naf::Machine.machine_ip_address
+    opt :update_machine, "create or update an machine entry"
+    opt :enabled, "enable machine"
+    opt :disabled, "disable machine", var: :enabled, set: :false
+    opt :thread_pool_size, "how many scripts can run at once", type: :int
+    opt :list_affinities, "show all affinities"
+    opt :add_affinities, "add an affinity slot", type: :strings
+    opt :add_weight_affinities, "add afffinity weights for cpu and memory", default: true
+
+    def work
+      if @list_affinities
+        puts "Affinities:"
+        ::Naf::Affinity.all.each do |affinity|
+          parts = [
+                   affinity.affinity_classification_name,
+                   affinity.affinity_name,
+                  ]
+          puts "  #{parts.join('_')}"
+        end
+        exit 0
+      end
+
+      if @update_machine
+        machine = ::Naf::Machine.find_by_server_address(@server_address)
+        if machine.blank?
+          machine = ::Naf::Machine.create(server_address: @server_address)
+          add_default_affinities(machine)
+        end
+
+        machine.server_note = @server_note unless @server_note.nil?
+        machine.server_name = @server_name unless @server_name.nil?
+        machine.enabled = @enabled unless @enabled.nil?
+        machine.thread_pool_size = @thread_pool_size unless @thread_pool_size.nil?
+        machine.save!
+      else
+        machine = ::Naf::Machine.find_by_server_address(@server_address)
+
+        unless machine.present?
+          puts "Machine address #{@server_address} is not present -- use --update-machine"
+          exit 1
+        end
+      end
+
+      if @add_affinities
+        @add_affinities.each do |affinity_string|
+          #
+          # Parse the argument string. It should consists of 2 or 3 words separated
+          # by underscores.
+          #
+          # Example:
+          #   - location_1_required
+          #   - purpose_large
+          #
+          parts = affinity_string.split('_')
+          if parts.length == 2
+            classification_name = parts[0]
+            affinity_name = parts[1]
+            required = false
+          elsif parts.length == 3
+            classification_name = parts[0]
+            affinity_name = parts[1]
+            required = true
+          else
+            puts "no idea how to interpret affinity classification in: '#{affinity_string}'"
+            exit 1
+          end
+
+          # Find the Affinity Classification in the Database
+          affinity_classification = ::Naf::AffinityClassification.
+            find_by_affinity_classification_name(classification_name)
+          unless affinity_classification
+            puts "could not find affinity classification: '#{classification_name}'"
+            exit 1
+          end
+
+          # Find the Affinity in the Database
+          affinity = ::Naf::Affinity.
+            find_by_affinity_classification_id_and_affinity_name(affinity_classification.id,
+                                                                 affinity_name)
+          unless affinity
+            puts "could not find affinity: '#{affinity_name}' with classification: '#{classification_name}'"
+            exit 1
+          end
+
+          # Create an affinity slot for the machine
+          machine.machine_affinity_slots.create(affinity_id: affinity.id,
+                                                required: required)
+        end
+      end
+
+      puts "Address: #{machine.server_address}"
+      puts "Name: #{machine.server_name}" unless machine.server_name.nil?
+      puts "Note: #{machine.server_note}" unless machine.server_note.nil?
+      puts "Enabled: #{machine.enabled}"
+      puts "Thread Pool Size: #{machine.thread_pool_size}"
+
+      if machine.affinities.empty?
+        puts "No machine affinity slots"
+      else
+        puts "Machine Affinity Slots:"
+        machine.machine_affinity_slots.each do |affinity_slot|
+          affinity = affinity_slot.affinity
+          parts = [
+                   affinity.affinity_classification_name,
+                   affinity.affinity_name,
+                  ]
+          parts << "required" if affinity_slot.required
+          puts "  #{parts.join('_')}"
+        end
+      end
+    end
+
+    private
+
+    def add_default_affinities(machine)
+      # Add Machine Affinity
+      classification = ::Naf::AffinityClassification.machine.id
+      affinity = ::Naf::Affinity.
+        find_or_create_by_affinity_classification_id_and_affinity_name(classification, machine.id.to_s)
+      machine.machine_affinity_slots.create(affinity_id: affinity.id)
+
+      if machine == ::Naf::Machine.current
+        # Add Purpose Affinity
+        instance_type = `source /var/spool/ec2/meta-data.sh && echo $EC2_INSTANCE_TYPE`
+        if instance_type.present?
+          classification = ::Naf::AffinityClassification.purpose.id
+          affinity = ::Naf::Affinity.
+            find_or_create_by_affinity_classification_id_and_affinity_name(classification, instance_type)
+          machine.machine_affinity_slots.create(affinity_id: affinity.id)
+        end
+
+        # Add Weight Affinity
+        classification = ::Naf::AffinityClassification.weight.id
+        machine_cpus = (`cat /proc/cpuinfo | grep processor | wc -l`).strip.to_i
+        machine_memory = (`cat /proc/meminfo | grep MemTotal`).slice(/\d+/).to_i / (1024 * 1024)
+        cpu_affinity = ::Naf::Affinity.
+          find_or_create_by_affinity_classification_id_and_affinity_name(classification, 'cpus')
+        memory_affinity = ::Naf::Affinity.
+          find_or_create_by_affinity_classification_id_and_affinity_name(classification, 'memory')
+        machine.machine_affinity_slots.create(affinity_id: cpu_affinity.id, affinity_parameter: machine_cpus)
+        machine.machine_affinity_slots.create(affinity_id: memory_affinity.id, affinity_parameter: machine_memory)
+      end
+    end
+
+  end
+end

data/app/models/process/naf/machine_upgrader.rb

@@ -0,0 +1,112 @@
+require 'naf/version'
+
+module Process::Naf
+  class MachineUpgrader < ::Process::Naf::Application
+    opt :upgrade_option,
+        "what should we do",
+        default: :dump,
+        choices: [:dump, :restore]
+    opt :pretty,
+        "make things pretty",
+        default: false
+    opt :no_updates,
+        "don't update the db (via transaction rollback)",
+        default: false
+    opt :force,
+        "run even if system looks unclean",
+        default: false
+
+    PRESERVABLES = [
+                    ::Naf::ApplicationType,
+                    ::Naf::Application,
+                    ::Naf::ApplicationRunGroupRestriction,
+                    ::Naf::ApplicationSchedule,
+                    ::Naf::ApplicationSchedulePrerequisite,
+                    ::Naf::AffinityClassification,
+                    ::Naf::Affinity,
+                    ::Naf::ApplicationScheduleAffinityTab,
+                    ::Naf::LoggerLevel,
+                    ::Naf::LoggerName,
+                    ::Naf::LoggerStyle,
+                    ::Naf::LoggerStyleName,
+                    ::Naf::Machine,
+                    ::Naf::MachineAffinitySlot,
+                    ::Naf::JanitorialAssignment
+                   ]
+
+    CLEAN_SYSTEM_MODELS = [
+                           ::Naf::Machine,
+                           ::Naf::VERSION == '1.0.3' ? ::Naf::HistoricalJob : ::Naf::Job
+                          ]
+
+
+    def work
+      self.send("work_#{@upgrade_option}")
+    end
+
+    def work_dump
+      pickler = ::Logical::Naf::Pickler.new(::Naf::VERSION, PRESERVABLES)
+      pickler.preserve
+      if @pretty
+        puts JSON.pretty_generate(pickler.pickle_jar)
+      else
+        puts JSON.generate(pickler.pickle_jar)
+      end
+    end
+
+    def work_restore
+      begin
+        pickle_jar = JSON.parse(STDIN.read)
+      rescue StandardError => e
+        logger.fatal "this doesn't look like a naf upgrade stream to me, it is not json parserable!"
+        exit 1
+      end
+
+      unless pickle_jar.is_a?(Hash)
+        logger.fatal "this doesn't look like a naf upgrade stream to me, it is of type: #{pickle_jar.class.name}"
+        exit 1
+      end
+
+      pickle_jar_version = pickle_jar['version']
+      if pickle_jar_version.blank?
+        logger.fatal "this doesn't look like a naf upgrade stream to me, it has no version!"
+        exit 1
+      end
+
+      preserves = pickle_jar['preserves']
+      if preserves.nil?
+        logger.fatal "this doesn't look like a naf upgrade stream to me, there are no preserves!"
+        exit 1
+      end
+
+      unless preserves.is_a?(Hash)
+        logger.fatal "this doesn't look like a naf upgrade stream to me, the preserves are type: #{preserves.class.name}"
+        exit 1
+      end
+
+      check_for_clean_system unless @force
+
+      preserved_at_text = pickle_jar['preserved_at']
+      preserved_at = Time.parse(preserved_at_text) rescue "unknown"
+
+      logger.info "restoring:"
+      logger.info " preserved_at: #{preserved_at}"
+      logger.info " version: #{pickle_jar_version}"
+      logger.info " models: #{preserves.length}"
+
+      unpickler = ::Logical::Naf::Unpickler.new(preserves, PRESERVABLES, pickle_jar_version)
+      ::Naf::NafBase.transaction do
+        unpickler.reconstitute
+        raise ActiveRecord::Rollback if @no_updates
+      end
+      logger.info "restoration complete! thank you!"
+    end
+
+    def check_for_clean_system
+      CLEAN_SYSTEM_MODELS.each do |model|
+        raise "the system is unclean" if model.count > 0
+      end
+    end
+
+  end
+end