naf 1.1.0

data/app/models/naf/application_schedule_prerequisite.rb

@@ -0,0 +1,50 @@
+module Naf
+  class ApplicationSchedulePrerequisite < ::Naf::NafBase
+    # Protect from mass-assignment issue
+    attr_accessible :application_schedule_id,
+                    :prerequisite_application_schedule_id
+
+    #---------------------
+    # *** Associations ***
+    #+++++++++++++++++++++
+
+    belongs_to :application_schedule,
+      class_name: "::Naf::ApplicationSchedule"
+    belongs_to :prerequisite_application_schedule,
+      class_name: "::Naf::ApplicationSchedule"
+
+    #--------------------
+    # *** Validations ***
+    #++++++++++++++++++++
+
+    validates :prerequisite_application_schedule_id, presence: true
+    validates :application_schedule_id, uniqueness: {
+                                          scope: :prerequisite_application_schedule_id
+                                        }
+
+    def self.pickleables(pickler)
+			return self.joins([application_schedule: :application]).
+				where('applications.deleted = false').
+				where(
+					"NOT EXISTS(
+					   SELECT
+						   1
+						 FROM
+						   #{::Naf.schema_name}.application_schedules AS a_s
+						 WHERE
+						   application_schedule_prerequisites.prerequisite_application_schedule_id = a_s.id AND
+						 EXISTS(
+						   SELECT
+							   1
+							 FROM
+							   #{::Naf.schema_name}.applications AS a
+							 WHERE
+							   a_s.application_id = a.id AND
+								   deleted IS TRUE
+						 )
+					)"
+				)
+    end
+
+  end
+end

data/app/models/naf/application_type.rb

@@ -0,0 +1,72 @@
+module Naf
+  class ApplicationType < NafBase
+    # Protect from mass-assignment issue
+    attr_accessible :enabled,
+                    :script_type_name,
+                    :description,
+                    :invocation_method
+
+    SCRIPT_RUNNER = "#{Gem.ruby} #{Rails.root}/script/rails runner"
+
+    #---------------------
+    # *** Associations ***
+    #+++++++++++++++++++++
+
+    has_many :applications,
+      class_name: "::Naf::Application"
+    has_many :historical_jobs,
+      class_name: "::Naf::HistoricalJob"
+
+    #--------------------
+    # *** Validations ***
+    #++++++++++++++++++++
+
+    validates :script_type_name,
+              :invocation_method, presence: true
+
+    #-------------------------
+    # *** Instance Methods ***
+    #+++++++++++++++++++++++++
+
+    def spawn(job)
+      self.send(invocation_method.to_sym, job)
+    end
+
+    def invoke(job, command)
+      Process.spawn({ "NAF_JOB_ID" => job.id.to_s }, command)
+    end
+
+    def rails_invocator(job)
+      invoke(job, SCRIPT_RUNNER + " " + job.command)
+    end
+
+    def bash_command_invocator(job)
+      invoke(job, job.command)
+    end
+
+    def bash_script_invocator(job)
+      invoke(job, job.command)
+    end
+
+    def ruby_script_invocator(job)
+      invoke(job, job.command)
+    end
+
+    def self.rails
+      @rails ||= find_by_script_type_name('rails')
+    end
+
+    def self.ruby
+      @ruby ||= find_by_script_type_name('ruby')
+    end
+
+    def self.bash_command
+      @bash_command ||= find_by_script_type_name('bash command')
+    end
+
+    def self.bash_script
+      @bash_script ||= find_by_script_type_name('bash script')
+    end
+
+  end
+end

data/app/models/naf/by_historical_job_id.rb

@@ -0,0 +1,86 @@
+module Naf
+  class ByHistoricalJobId < ::Partitioned::ByIntegerField
+    self.abstract_class = true
+
+    # Protect from mass-assignment issue
+    attr_accessible :historical_job_id
+
+    #---------------------
+    # *** Associations ***
+    #+++++++++++++++++++++
+
+    belongs_to :historical_job,
+      class_name: "::Naf::HistoricalJob"
+
+    #--------------------
+    # *** Validations ***
+    #++++++++++++++++++++
+
+    validates :historical_job_id, presence: true
+
+    #------------------
+    # *** Partition ***
+    #++++++++++++++++++
+
+    partitioned do |partition|
+      partition.index :id, unique: true
+
+      partition.foreign_key lambda { |model, *partition_key_values|
+        return ::Partitioned::PartitionedBase::Configurator::Data::ForeignKey.
+          new(model.partition_integer_field,
+              ::Naf::HistoricalJob.partition_table_name(*partition_key_values),
+              :id)
+      }
+
+      partition.janitorial_creates_needed lambda { |model, *partition_key_values|
+        sequence_name = model.connection.default_sequence_name(model.table_name)
+        current_id = model.find_by_sql("select last_value as id from #{sequence_name}").first.id
+        start_range = [0, current_id - (model.partition_table_size * model.partition_num_lead_buffers)].max
+        end_range = current_id + (model.partition_table_size * model.partition_num_lead_buffers)
+
+        return model.partition_generate_range(start_range, end_range).reject{ |p| model.sql_adapter.partition_exists?(p) }
+      }
+
+      partition.janitorial_archives_needed []
+      partition.janitorial_drops_needed lambda { |model, *partition_key_values|
+        sequence_name = model.connection.default_sequence_name(model.table_name)
+        current_id = model.find_by_sql("select last_value as id from #{sequence_name}").first.id
+        partition_key_value = current_id - (model.partition_table_size * model.partition_num_lead_buffers)
+        partition_key_values_to_drop = []
+        while model.sql_adapter.partition_exists?(partition_key_value)
+          partition_key_values_to_drop << partition_key_value
+          partition_key_value -= model.partition_table_size
+        end
+
+        return partition_key_values_to_drop
+      }
+    end
+
+    #----------------------
+    # *** Class Methods ***
+    #++++++++++++++++++++++
+
+    # the field to partition on
+    # @return [Integer] re-routed to {#self.partition_foreign_key}
+    def self.partition_integer_field
+      return :historical_job_id
+    end
+
+    def self.connection
+      return ::Naf::NafBase.connection
+    end
+
+    def self.full_table_name_prefix
+      return ::Naf::NafBase.full_table_name_prefix
+    end
+
+    def self.partition_table_size
+      return ::Naf::HistoricalJob.partition_table_size
+    end
+
+    def self.partition_num_lead_buffers
+      return ::Naf::HistoricalJob.partition_num_lead_buffers
+    end
+
+  end
+end

data/app/models/naf/historical_job.rb

@@ -0,0 +1,334 @@
+module Naf
+  #
+  # Things you should know about jobs
+  # new jobs older than 1.week will not be searched in the queue.  You should not run programs
+  # for more than a 1.week (instead, have them exit and restarted periodically)
+  #
+  class HistoricalJob < ::Partitioned::ById
+    class JobPrerequisiteLoop < StandardError
+      def initialize(job)
+        super("loop found in prerequisites for #{job}")
+      end
+    end
+
+    include PgAdvisoryLocker
+
+    # Protect from mass-assignment issue
+    attr_accessible :application_id,
+                    :application_type_id,
+                    :command,
+                    :application_run_group_restriction_id,
+                    :application_run_group_name,
+                    :application_run_group_limit,
+                    :priority,
+                    :started_on_machine_id,
+                    :failed_to_start,
+                    :pid,
+                    :exit_status,
+                    :termination_signal,
+                    :state,
+                    :request_to_terminate,
+                    :marked_dead_by_machine_id,
+                    :log_level,
+                    :tags,
+                    :machine_runner_invocation_id
+
+    JOB_STALE_TIME = 1.week
+    SYSTEM_TAGS = {
+      startup: '$startup',
+      pre_work: '$pre-work',
+      work: '$work',
+      cleanup: '$cleanup'
+    }
+
+    #---------------------
+    # *** Associations ***
+    #+++++++++++++++++++++
+
+    belongs_to :application_type,
+      class_name: '::Naf::ApplicationType'
+    belongs_to :started_on_machine,
+      class_name: '::Naf::Machine',
+      foreign_key: 'started_on_machine_id'
+    belongs_to :marked_dead_by_machine,
+      class_name: '::Naf::Machine',
+      foreign_key: 'marked_dead_by_machine_id'
+    belongs_to :application,
+      class_name: "::Naf::Application"
+    belongs_to :application_run_group_restriction,
+      class_name: "::Naf::ApplicationRunGroupRestriction"
+    belongs_to :machine_runner_invocation,
+      class_name: "::Naf::MachineRunnerInvocation"
+    # Must access instance methods job_prerequisites through helper methods so we can use partitioning sql
+    has_many :historical_job_prerequisites,
+      class_name: "::Naf::HistoricalJobPrerequisite",
+      dependent: :destroy
+    has_many :prerequisites,
+      class_name: "::Naf::HistoricalJob",
+      through: :historical_job_prerequisites,
+      source: :prerequisite_historical_job
+    # Access supported through instance methods
+    has_many :historical_job_affinity_tabs,
+      class_name: "::Naf::HistoricalJobAffinityTab",
+      dependent: :destroy
+    has_many :affinities,
+      class_name: "::Naf::Affinity",
+      through: :historical_job_affinity_tabs
+
+    #--------------------
+    # *** Validations ***
+    #++++++++++++++++++++
+
+    validates :application_type_id,
+              :command,
+              :application_run_group_restriction_id, presence: true
+
+    validates :command, length: {
+                          minimum: 3
+                        }
+    validates :application_run_group_limit, numericality: {
+                                              only_integer: true,
+                                              greater_than_or_equal_to: 1,
+                                              less_than: 2147483647,
+                                              allow_blank: true
+                                            }
+
+    #--------------------
+    # *** Delegations ***
+    #++++++++++++++++++++
+
+    delegate :application_run_group_restriction_name, to: :application_run_group_restriction
+    delegate :script_type_name, to: :application_type
+
+    #------------------
+    # *** Partition ***
+    #++++++++++++++++++
+
+    partitioned do |partition|
+      partition.foreign_key :application_id, ::Naf::Application.table_name
+      partition.foreign_key :application_type_id, ::Naf::ApplicationType.table_name
+      partition.foreign_key :application_run_group_restriction_id, ::Naf::ApplicationRunGroupRestriction.table_name
+      partition.foreign_key :started_on_machine_id, ::Naf::Machine.table_name
+      partition.index :created_at
+      partition.index :application_id
+      partition.index :started_on_machine_id
+      partition.index :command
+      partition.index :application_run_group_name
+      partition.index :finished_at
+      partition.index :exit_status
+
+      partition.janitorial_creates_needed lambda { |model, *partition_key_values|
+        sequence_name = model.connection.default_sequence_name(model.table_name)
+        current_id = model.find_by_sql("select last_value as id from #{sequence_name}").first.id
+        start_range = [0, current_id - (model.partition_table_size * model.partition_num_lead_buffers)].max
+        end_range = current_id + (model.partition_table_size * model.partition_num_lead_buffers)
+        return model.partition_generate_range(start_range, end_range).reject{|p| model.sql_adapter.partition_exists?(p)}
+      }
+      partition.janitorial_archives_needed []
+      partition.janitorial_drops_needed lambda { |model, *partition_key_values|
+        sequence_name = model.connection.default_sequence_name(model.table_name)
+        current_id = model.find_by_sql("select last_value as id from #{sequence_name}").first.id
+        partition_key_value = current_id - (model.partition_table_size * model.partition_num_lead_buffers)
+        partition_key_values_to_drop = []
+        while model.sql_adapter.partition_exists?(partition_key_value)
+          partition_key_values_to_drop << partition_key_value
+          partition_key_value -= model.partition_table_size
+        end
+        return partition_key_values_to_drop
+      }
+    end
+
+    #----------------------
+    # *** Class Methods ***
+    #++++++++++++++++++++++
+
+    def self.connection
+      ::Naf::NafBase.connection
+    end
+
+    def self.full_table_name_prefix
+      ::Naf::NafBase.full_table_name_prefix
+    end
+
+    def self.partition_table_size
+      100000
+    end
+
+    def self.partition_num_lead_buffers
+      10
+    end
+
+    def self.queued_between(start_time, end_time)
+      where(["created_at >= ? AND created_at <= ?", start_time, end_time])
+    end
+
+    def self.canceled
+      where(request_to_terminate: true)
+    end
+
+    def self.application_last_runs
+      where("application_id IS NOT NULL").
+        group("application_id").
+        select("application_id, MAX(finished_at) AS finished_at").
+        reject{ |job| job.finished_at.nil? }
+    end
+
+    def self.application_last_queued
+      where("application_id IS NOT NULL").
+        group("application_id").
+        select("application_id, MAX(id) AS id, MAX(created_at) AS created_at")
+    end
+
+    def self.finished
+      where("finished_at IS NOT NULL OR request_to_terminate = true")
+    end
+
+    def self.queued_status
+      where("(started_at IS NULL AND request_to_terminate = false) OR
+             (finished_at > '#{Time.zone.now - 1.minute}') OR
+             (started_at IS NOT NULL AND finished_at IS NULL AND request_to_terminate = false)")
+    end
+
+    def self.running_status
+      where("(started_at IS NOT NULL AND finished_at IS NULL AND request_to_terminate = false) OR
+             (finished_at > '#{Time.zone.now - 1.minute}')")
+    end
+
+    def self.queued_with_waiting
+      where("(started_at IS NULL AND request_to_terminate = false)")
+    end
+
+    def self.errored
+      where("finished_at IS NOT NULL AND exit_status > 0 OR request_to_terminate = true")
+    end
+
+    def self.lock_for_job_queue(&block)
+      lock_record(0, &block)
+    end
+
+    #-------------------------
+    # *** Instance Methods ***
+    #+++++++++++++++++++++++++
+
+    def to_s
+      components = []
+
+      if started_at.nil?
+        components << "QUEUED"
+      else
+        if finished_at.nil?
+          if pid
+            extras = []
+            extras << pid.to_s
+            extras << 'RequestedToTerminate' if request_to_terminate
+            components << "RUNNING:#{extras.join(':')}"
+          else
+            components << "SPAWNING"
+          end
+        else
+          extras = []
+          extras << 'RequestedToTerminate' if request_to_terminate
+          extras << "FailedToStart" if failed_to_start
+          extras << "SIG#{termination_signal}" if termination_signal
+          if exit_status && exit_status != 0
+            extras << "STATUS=#{exit_status}"
+          end
+          if extras.length
+            extras_str = " (#{extras.join(',')})"
+          else
+            extras_str = ""
+          end
+          components << "FINISHED#{extras_str}"
+        end
+      end
+      components << "id: #{id}"
+      components << "\"#{command}\""
+
+      return "::Naf::HistoricalJob<#{components.join(', ')}>"
+    end
+
+    def title
+      application.try(:title)
+    end
+
+    def machine_started_on_server_name
+      started_on_machine.try(:server_name)
+    end
+
+    def machine_started_on_server_address
+      started_on_machine.try(:server_address)
+    end
+
+    def historical_job_affinity_tabs
+      ::Naf::HistoricalJobAffinityTab.
+        from_partition(id).
+        where(historical_job_id: id)
+    end
+
+    def job_affinities
+      historical_job_affinity_tabs.map{ |jat| jat.affinity }
+    end
+
+    def affinity_ids
+      historical_job_affinity_tabs.map{ |jat| jat.affinity_id }
+    end
+
+    def historical_job_prerequisites
+      ::Naf::HistoricalJobPrerequisite.
+        from_partition(id).
+        where(historical_job_id: id)
+    end
+
+    def prerequisites
+      historical_job_prerequisites.
+        map{ |hjp| ::Naf::HistoricalJob.from_partition(hjp.prerequisite_historical_job_id).
+        find_by_id(hjp.prerequisite_historical_job_id) }.
+        reject{ |j| j.nil? }
+    end
+
+    # XXX This should go away (it was moved to ConstructionZone::Foreman)
+
+    def verify_prerequisites(these_jobs)
+      these_jobs.each do |this_job|
+        if this_job.id == id
+          raise JobPrerequisiteLoop.new(self)
+        else
+          verify_prerequisites(this_job.prerequisites)
+        end
+      end
+    end
+
+    def spawn
+      application_type.spawn(self)
+    end
+
+    def add_tags(tags_to_add)
+      tags_array = nil
+      if self.tags.present?
+        tags_array = self.tags.gsub(/[{}]/,'').split(',')
+        new_tags = '{' + (tags_array | tags_to_add).join(',') + '}'
+      else
+        new_tags = '{' + tags_to_add.join(',') + '}'
+      end
+
+      self.tags = new_tags
+      self.save!
+    end
+
+    def remove_tags(tags_to_remove)
+      if self.tags.present?
+        tags_array = self.tags.gsub(/[{}]/,'').split(',')
+        new_tags = '{' + (tags_array - tags_to_remove).join(',') + '}'
+
+        self.tags = new_tags
+        self.save!
+      end
+    end
+
+    def remove_all_tags
+      self.tags = '{}'
+      self.save!
+    end
+
+  end
+end