naf 1.1.0

data/app/models/naf/historical_job_affinity_tab.rb

@@ -0,0 +1,61 @@
+module Naf
+  class HistoricalJobAffinityTab < ::Naf::ByHistoricalJobId
+    # Protect from mass-assignment issue
+    attr_accessible :affinity_id,
+                    :historical_job_id,
+                    :historical_job,
+                    :affinity_parameter
+
+    #---------------------
+    # *** Associations ***
+    #+++++++++++++++++++++
+
+    belongs_to :affinity,
+      class_name: "::Naf::Affinity"
+
+    #--------------------
+    # *** Validations ***
+    #++++++++++++++++++++
+
+    validates :affinity_id, presence: true,
+                            uniqueness: {
+                              scope: :historical_job_id,
+                              message: "has already been taken for this job"
+                            }
+
+    #--------------------
+    # *** Delegations ***
+    #++++++++++++++++++++
+
+    delegate :affinity_name,
+             :affinity_classification_name,
+             :affinity_short_name, to: :affinity
+
+    #------------------
+    # *** Partition ***
+    #++++++++++++++++++
+
+    partitioned do |partition|
+      partition.foreign_key :affinity_id, ::Naf::Affinity.table_name
+    end
+
+    #-------------------------
+    # *** Instance Methods ***
+    #+++++++++++++++++++++++++
+
+    def job
+      ::Naf::HistoricalJob.
+        from_partition(id).
+        where(id: historical_job_id).first
+    end
+
+    def script_type_name
+      job.script_type_name
+    end
+
+    def command
+      job.command
+    end
+
+  end
+end

data/app/models/naf/historical_job_prerequisite.rb

@@ -0,0 +1,19 @@
+module Naf
+  class HistoricalJobPrerequisite < ::Naf::ByHistoricalJobId
+    # Protect from mass-assignment issue
+    attr_accessible :historical_job_id,
+                    :prerequisite_historical_job_id
+
+    #---------------------
+    # *** Associations ***
+    #+++++++++++++++++++++
+
+    belongs_to :historical_job,
+      class_name: "::Naf::HistoricalJob",
+      foreign_key: 'historical_job_id'
+    belongs_to :prerequisite_historical_job,
+      class_name: "::Naf::HistoricalJob",
+      foreign_key: 'prerequisite_historical_job_id'
+
+  end
+end

data/app/models/naf/janitorial_archive_assignment.rb

@@ -0,0 +1,36 @@
+module Naf
+  class JanitorialArchiveAssignment < JanitorialAssignment
+    # Protect from mass-assignment issue
+    attr_accessible :model_name,
+                    :assignment_order,
+                    :enabled,
+                    :deleted
+
+    #--------------------
+    # *** Validations ***
+    #++++++++++++++++++++
+
+    validate :deleted_enabled_check
+    validates :assignment_order, numericality: {
+                                   only_integer: true,
+                                   greater_than_or_equal_to: 0,
+                                   less_than: 2147483647
+                                 }
+    validates :model_name, presence: true
+
+    #-------------------------
+    # *** Instance Methods ***
+    #+++++++++++++++++++++++++
+
+    def do_janitorial_work(target_model)
+      target_model.archive_old_partitions
+    end
+
+    def deleted_enabled_check
+      if deleted && enabled
+        errors.add(:deleted, "or Enabled must be false")
+      end
+    end
+
+  end
+end

data/app/models/naf/janitorial_assignment.rb

@@ -0,0 +1,37 @@
+module Naf
+  class JanitorialAssignment < ::Naf::NafBase
+
+    #----------------------
+    # *** Class Methods ***
+    #++++++++++++++++++++++
+
+    def self.enabled
+      return where("enabled")
+    end
+
+    def self.pickleables(pickler)
+      old_model_names = ['::Naf::Job',
+                         '::Naf::JobCreatedAt',
+                         '::Naf::JobPrerequisite',
+                         '::Naf::JobAffinityTab']
+      return where('model_name NOT IN (?)', old_model_names)
+    end
+
+    #-------------------------
+    # *** Instance Methods ***
+    #+++++++++++++++++++++++++
+
+    def target_model
+      return model_name.constantize rescue nil
+    end
+
+    def self.pickleables(pickler)
+      old_model_names = ['::Naf::Job',
+                         '::Naf::JobCreatedAt',
+                         '::Naf::JobPrerequisite',
+                         '::Naf::JobAffinityTab']
+      return where('model_name NOT IN (?)', old_model_names)
+    end
+
+  end
+end

data/app/models/naf/janitorial_create_assignment.rb

@@ -0,0 +1,36 @@
+module Naf
+  class JanitorialCreateAssignment < JanitorialAssignment
+    # Protect from mass-assignment issue
+    attr_accessible :model_name,
+                    :assignment_order,
+                    :enabled,
+                    :deleted
+
+    #--------------------
+    # *** Validations ***
+    #++++++++++++++++++++
+
+    validate :deleted_enabled_check
+    validates :assignment_order, numericality: {
+                                   only_integer: true,
+                                   greater_than_or_equal_to: 0,
+                                   less_than: 2147483647
+                                 }
+    validates :model_name, presence: true
+
+    #-------------------------
+    # *** Instance Methods ***
+    #+++++++++++++++++++++++++
+
+    def do_janitorial_work(target_model)
+      target_model.create_new_partitions
+    end
+
+    def deleted_enabled_check
+      if deleted && enabled
+        errors.add(:deleted, "or Enabled must be false")
+      end
+    end
+
+  end
+end

data/app/models/naf/janitorial_drop_assignment.rb

@@ -0,0 +1,36 @@
+module Naf
+  class JanitorialDropAssignment < JanitorialAssignment
+    # Protect from mass-assignment issue
+    attr_accessible :model_name,
+                    :assignment_order,
+                    :enabled,
+                    :deleted
+
+    #--------------------
+    # *** Validations ***
+    #++++++++++++++++++++
+
+    validate :deleted_enabled_check
+    validates :assignment_order, numericality: {
+                                   only_integer: true,
+                                   greater_than_or_equal_to: 0,
+                                   less_than: 2147483647
+                                 }
+    validates :model_name, presence: true
+
+    #-------------------------
+    # *** Instance Methods ***
+    #+++++++++++++++++++++++++
+
+    def do_janitorial_work(target_model)
+      target_model.drop_old_partitions
+    end
+
+    def deleted_enabled_check
+      if deleted && enabled
+        errors.add(:deleted, "or Enabled must be false")
+      end
+    end
+
+  end
+end

data/app/models/naf/logger_level.rb

@@ -0,0 +1,21 @@
+module Naf
+  class LoggerLevel < NafBase
+    # Protect from mass-assignment issue
+    attr_accessible :level
+
+    #---------------------
+    # *** Associations ***
+    #+++++++++++++++++++++
+
+    has_many :logger_style_names,
+      class_name: '::Naf::LoggerStyleName'
+
+    #--------------------
+    # *** Validations ***
+    #++++++++++++++++++++
+
+    validates :level, uniqueness: true,
+                      presence: true
+
+  end
+end

data/app/models/naf/logger_name.rb

@@ -0,0 +1,23 @@
+module Naf
+  class LoggerName < NafBase
+    # Protect from mass-assignment issue
+    attr_accessible :name
+
+    #---------------------
+    # *** Associations ***
+    #+++++++++++++++++++++
+
+    has_many :logger_style_names,
+      class_name: '::Naf::LoggerStyleName'
+    has_many :logger_styles,
+      through: :logger_style_names
+
+    #--------------------
+    # *** Validations ***
+    #++++++++++++++++++++
+
+    validates :name, uniqueness: true,
+                     presence: true
+
+  end
+end

data/app/models/naf/logger_style.rb

@@ -0,0 +1,58 @@
+module Naf
+  class LoggerStyle < NafBase
+    # Protect from mass-assignment issue
+    attr_accessible :name,
+                    :note,
+                    :logger_style_names_attributes
+
+    #---------------------
+    # *** Associations ***
+    #+++++++++++++++++++++
+
+    has_many :logger_style_names,
+      class_name: '::Naf::LoggerStyleName'
+    has_many :logger_names,
+      through: :logger_style_names
+
+    accepts_nested_attributes_for :logger_style_names
+
+    #--------------------
+    # *** Validations ***
+    #++++++++++++++++++++
+
+    validates :name, uniqueness: true,
+                     presence: true
+    validate :check_logger_style_names_attributes
+
+    before_save :check_blank_values
+
+    #-------------------------
+    # *** Instance Methods ***
+    #+++++++++++++++++++++++++
+
+    def _logger_names
+      logger_style_names.map do |lsn|
+        lsn.logger_name.name
+      end.join(', ')
+    end
+
+    def logger_levels
+      logger_style_names.map do |lsn|
+        lsn.logger_level.level
+      end.join(', ')
+    end
+
+    def check_logger_style_names_attributes
+      if logger_style_names.map{ |ln| ln.logger_name_id }.uniq!
+        errors.add(:logger_name_id, "should be an uniqueness")
+      end
+    end
+
+    private
+
+    def check_blank_values
+      self.note = nil if self.note.blank?
+    end
+
+  end
+end

data/app/models/naf/logger_style_name.rb

@@ -0,0 +1,28 @@
+module Naf
+  class LoggerStyleName < NafBase
+    # Protect from mass-assignment issue
+    attr_accessible :logger_style_id,
+                    :logger_name_id,
+                    :logger_level_id
+
+    #---------------------
+    # *** Associations ***
+    #+++++++++++++++++++++
+
+    belongs_to :logger_name,
+      class_name: '::Naf::LoggerName'
+    belongs_to :logger_style,
+      class_name: '::Naf::LoggerStyle'
+    belongs_to :logger_level,
+      class_name: '::Naf::LoggerLevel'
+
+    #--------------------
+    # *** Validations ***
+    #++++++++++++++++++++
+
+    validates :logger_name_id,
+              :logger_style_id, presence: true
+    validates :logger_style_id, uniqueness: { scope: :logger_name_id }
+
+  end
+end

data/app/models/naf/machine.rb

@@ -0,0 +1,257 @@
+require 'socket'
+
+module Naf
+  class Machine < NafBase
+    include ::Af::Application::SafeProxy
+    include PgAdvisoryLocker
+
+    # Protect from mass-assignment issue
+    attr_accessible :server_address,
+                    :server_name,
+                    :server_note,
+                    :enabled,
+                    :thread_pool_size,
+                    :log_level,
+                    :marked_down,
+                    :short_name,
+                    :deleted
+
+    IP_REGEX =  /^([01]?\d\d?|2[0-4]\d|25[0-5])\.([01]?\d\d?|2[0-4]\d|25[0-5])\.([01]?\d\d?|2[0-4]\d|25[0-5])\.([01]?\d\d?|2[0-4]\d|25[0-5])$/
+
+    #--------------------
+    # *** Validations ***
+    #++++++++++++++++++++
+
+    validates :server_address,
+              :thread_pool_size, presence: true
+    validates :short_name, uniqueness: true,
+                           allow_blank: true,
+                           allow_nil: true,
+                           format: {
+                             with: /^[a-zA-Z_][a-zA-Z0-9_]*$/,
+                             message: "letters should be first (use only letters and numbers)"
+                           }
+    validates :server_address, format: {
+                                 with: IP_REGEX,
+                                 message: "is not a valid IP address"
+                               },
+                               if: :server_address
+    validates :server_address, uniqueness: true,
+                               if: :correct_server_address?
+    validates :thread_pool_size, numericality: {
+                                   only_integer: true,
+                                   greater_than: -2147483648,
+                                   less_than: 2147483647
+                                 }
+    before_save :check_blank_values
+
+    #---------------------
+    # *** Associations ***
+    #+++++++++++++++++++++
+
+    has_many :machine_affinity_slots,
+      class_name: '::Naf::MachineAffinitySlot',
+      dependent: :destroy
+    has_many :affinities,
+      through: :machine_affinity_slots
+    has_many :machine_runners,
+      class_name: '::Naf::MachineRunner'
+
+    #----------------------
+    # *** Class Methods ***
+    #++++++++++++++++++++++
+
+    def self.enabled
+      where(enabled: true)
+    end
+
+    def self.up
+      where(marked_down: false)
+    end
+
+    def self.down
+      where(marked_down: true)
+    end
+
+    def self.with_affinity(affinity_short_name)
+      sql = <<-SQL
+         exists (select
+                   1
+                 from
+                   #{::Naf::MachineAffinitySlot.table_name} as mas
+                 inner join #{::Naf::Affinity.table_name} as a on
+                   a.id = mas.affinity_id and
+                   a.affinity_short_name = ?
+                 where
+                   mas.machine_id = #{::Naf::Machine.table_name}.id)
+      SQL
+      return where([sql, affinity_short_name])
+    end
+
+    def self.machine_ip_address
+      begin
+        Socket::getaddrinfo(hostname, "echo", Socket::AF_INET)[0][3]
+      rescue StandardError
+        "127.0.0.1"
+      end
+    end
+
+    def self.hostname
+      begin
+        Socket.gethostname
+      rescue StandardError
+        "local"
+      end
+    end
+
+    def self.local_machine
+      where(server_address: machine_ip_address).first
+    end
+
+    def self.current
+      local_machine
+    end
+
+    def self.last_time_schedules_were_checked
+      self.maximum(:last_checked_schedules_at)
+    end
+
+    def self.is_it_time_to_check_schedules?(check_period)
+      time = Naf::Machine.last_time_schedules_were_checked
+
+      time.nil? || time < (Time.zone.now - check_period)
+    end
+
+    def self.include_deleted(filter)
+      if !filter
+        where(deleted: false)
+      else
+        where({})
+      end
+    end
+
+    #-------------------------
+    # *** Instance Methods ***
+    #+++++++++++++++++++++++++
+
+    def lock_for_runner_use(&block)
+      advisory_lock(&block)
+    end
+
+    def unlock_for_runner_use
+      advisory_unlock
+    end
+
+    def machine_logger
+      af_logger(self.class.name)
+    end
+
+    def to_s
+      components = []
+      if enabled
+        components << "ENABLED"
+      else
+        components << "DISABLED"
+      end
+      components << "DOWN!" if marked_down
+      components << "id: #{id}"
+      components << "address: #{server_address}"
+      components << "name: \"#{server_name}\"" unless server_name.blank?
+      components << "pool size: #{thread_pool_size}"
+      components << "last checked schedules: #{last_checked_schedules_at}"
+      components << "last seen: #{last_seen_alive_at}"
+
+      return "::Naf::Machine<#{components.join(', ')}>"
+    end
+
+    def correct_server_address?
+      server_address.present? and IP_REGEX =~ server_address
+    end
+
+    def mark_checked_schedule
+      self.last_checked_schedules_at = Time.zone.now
+      save!
+    end
+
+    def mark_alive
+      self.last_seen_alive_at = Time.zone.now
+      save!
+    end
+
+    def mark_up
+      self.marked_down = false
+      save!
+    end
+
+    def mark_down(by_machine)
+      self.marked_down = true
+      self.marked_down_by_machine_id = by_machine.id
+      self.marked_down_at = Time.zone.now
+      save!
+    end
+
+    def is_stale?(period)
+      # if last_seen_alive_at is nil then the runner has not been started yet -- hold off
+      # claiming it is stale until the runner is run at least once.
+      self.last_seen_alive_at.present? && self.last_seen_alive_at < (Time.zone.now - period)
+    end
+
+    def mark_processes_as_dead(by_machine)
+      ::Naf::RunningJob.where(created_at: (Time.zone.now - Naf::HistoricalJob::JOB_STALE_TIME)..Time.zone.now).
+        where("request_to_terminate = false").
+        started_on(self).each do |job|
+
+        marking_at = Time.zone.now
+        machine_logger.alarm "#{by_machine.id} marking #{job} as dead at #{marking_at}"
+        job.request_to_terminate = job.historical_job.request_to_terminate = true
+        job.marked_dead_by_machine_id = job.historical_job.marked_dead_by_machine_id = by_machine.id
+        job.marked_dead_at = job.historical_job.marked_dead_at = marking_at
+        job.historical_job.finished_at = marking_at
+        job.save!
+        job.historical_job.save!
+      end
+    end
+
+    def mark_machine_down(by_machine)
+      marking_at = Time.zone.now
+      machine_logger.alarm "#{by_machine.id} marking #{self} as down at #{marking_at}"
+      self.marked_down = true
+      self.marked_down_by_machine_id = by_machine.id
+      self.marked_down_at = marking_at
+      save!
+      mark_processes_as_dead(by_machine)
+    end
+
+    def affinity
+      return ::Naf::Affinity.
+        where(affinity_classification_id: ::Naf::AffinityClassification.location.id,
+              affinity_name: self.id.to_s).first
+    end
+
+    def short_name_if_it_exist
+      short_name || server_name
+    end
+
+    def pickle(pickler)
+      pickler.generic_pickle(self, nil, [
+                                         :created_at,
+                                         :updated_at,
+                                         :last_checked_schedules_at,
+                                         :last_seen_alive_at,
+                                         :marked_down,
+                                         :marked_down_by_machine_id,
+                                         :marked_down_at
+                                        ])
+    end
+
+    private
+
+    def check_blank_values
+      self.short_name = nil if self.short_name.blank?
+      self.server_name = nil if self.server_name.blank?
+      self.server_note = nil if self.server_note.blank?
+      self.log_level = nil if self.log_level.blank?
+    end
+
+  end
+end