naf 1.1.0

data/app/models/logical/naf/job_creator.rb

@@ -0,0 +1,155 @@
+module Logical
+  module Naf
+    class JobCreator
+      def queue_application(application,
+                            application_run_group_restriction,
+                            application_run_group_name,
+                            application_run_group_limit = 1,
+                            priority = 0,
+                            affinities = [],
+                            prerequisites = [],
+                            enqueue = false)
+
+        # Before adding a job to the queue, check whether the number of
+        # jobs (running/queued) is equal to or greater than the application
+        # run group limit, or if enqueue_backlogs is set to false. If so,
+        # do not add the job to the queue
+        running_jobs = ::Naf::RunningJob.
+          select('application_run_group_limit, MAX(created_at) AS created_at, count(*)').
+          where('command = ? AND application_run_group_name = ?',
+            application.command, application_run_group_name).
+          group('application_run_group_name, application_run_group_limit').first
+
+        queued_jobs = ::Naf::QueuedJob.
+          select('application_run_group_limit, MAX(created_at) AS created_at, count(*)').
+          where('command = ? AND application_run_group_name = ?',
+            application.command, application_run_group_name).
+          group('application_run_group_name, application_run_group_limit').first
+
+        if enqueue == false && (running_jobs.present? || queued_jobs.present?)
+          group_limit = running_jobs.try(:application_run_group_limit).to_i + queued_jobs.try(:application_run_group_limit).to_i
+          total_jobs = running_jobs.try(:count).to_i + queued_jobs.try(:count).to_i
+
+          return if group_limit <= total_jobs
+        end
+
+        ::Naf::HistoricalJob.transaction do
+          historical_job = ::Naf::HistoricalJob.create!(application_id: application.id,
+                                                        application_type_id: application.application_type_id,
+                                                        command: application.command,
+                                                        application_run_group_restriction_id: application_run_group_restriction.id,
+                                                        application_run_group_name: application_run_group_name,
+                                                        application_run_group_limit: application_run_group_limit,
+                                                        priority: priority,
+                                                        log_level: application.log_level)
+          historical_job.add_tags([::Naf::HistoricalJob::SYSTEM_TAGS[:pre_work]])
+
+          # Create historical job affinity tabs for each affinity associated with the historical job
+          affinities.each do |affinity|
+            affinity_parameter = ::Naf::ApplicationScheduleAffinityTab.
+              where(affinity_id: affinity.id,
+                    application_schedule_id: application.application_schedule.try(:id)).
+              first.try(:affinity_parameter)
+            ::Naf::HistoricalJobAffinityTab.create(historical_job_id: historical_job.id,
+                                                   affinity_id: affinity.id,
+                                                   affinity_parameter: affinity_parameter)
+          end
+
+          historical_job.verify_prerequisites(prerequisites)
+          # Create historical job prerequisites for each prerequisite associated with the historical job
+          prerequisites.each do |prerequisite|
+            ::Naf::HistoricalJobPrerequisite.create(historical_job_id: historical_job.id,
+                                                    prerequisite_historical_job_id: prerequisite.id)
+          end
+
+          create_queue_job(historical_job)
+
+          return historical_job
+        end
+      end
+
+      def queue_application_schedule(application_schedule, schedules_queued_already = [])
+        prerequisite_jobs = []
+
+        # Check if schedule has been queued
+        if schedules_queued_already.include? application_schedule.id
+          raise ::Naf::HistoricalJob::JobPrerequisiteLoop.new(application_schedule)
+        end
+
+        # Keep track of queued schedules
+        schedules_queued_already << application_schedule.id
+        # Queue application schedule prerequisites
+        application_schedule.prerequisites.each do |application_schedule_prerequisite|
+          prerequisite_jobs << queue_application_schedule(application_schedule_prerequisite, schedules_queued_already)
+        end
+
+        # Queue the application
+        return queue_application(application_schedule.application,
+                                 application_schedule.application_run_group_restriction,
+                                 application_schedule.application_run_group_name,
+                                 application_schedule.application_run_group_limit,
+                                 application_schedule.priority,
+                                 application_schedule.affinities,
+                                 prerequisite_jobs,
+                                 application_schedule.enqueue_backlogs)
+      end
+
+      # This method act similar to queue_application but is used for testing purpose
+      def queue_rails_job(command,
+                          application_run_group_restriction = ::Naf::ApplicationRunGroupRestriction.limited_per_all_machines,
+                          application_run_group_name = :command,
+                          application_run_group_limit = 1,
+                          priority = 0,
+                          affinities = [],
+                          prerequisites = [])
+        application_run_group_name = command if application_run_group_name == :command
+        ::Naf::HistoricalJob.transaction do
+          historical_job = ::Naf::HistoricalJob.create!(application_type_id: 1,
+                                                        command: command,
+                                                        application_run_group_restriction_id: application_run_group_restriction.id,
+                                                        application_run_group_name: application_run_group_name,
+                                                        application_run_group_limit: application_run_group_limit,
+                                                        priority: priority)
+          affinities.each do |affinity|
+            ::Naf::HistoricalJobAffinityTab.create(historical_job_id: historical_job.id, affinity_id: affinity.id)
+          end
+
+          historical_job.verify_prerequisites(prerequisites)
+          prerequisites.each do |prerequisite|
+            ::Naf::HistoricalJobPrerequisite.create(historical_job_id: historical_job.id,
+                                                    prerequisite_historical_job_id: prerequisite.id)
+          end
+
+          create_queue_job(historical_job)
+
+          return historical_job
+        end
+      end
+
+      def create_queue_job(historical_job)
+        queued_job = ::Naf::QueuedJob.new(application_id: historical_job.application_id,
+                                          application_type_id: historical_job.application_type_id,
+                                          command: historical_job.command,
+                                          application_run_group_restriction_id: historical_job.application_run_group_restriction_id,
+                                          application_run_group_name: historical_job.application_run_group_name,
+                                          application_run_group_limit: historical_job.application_run_group_limit,
+                                          priority: historical_job.priority)
+        queued_job.id = historical_job.id
+        queued_job.save!
+      end
+
+      def queue_test
+        queue_rails_job("#{self.class.name}.test")
+      end
+
+      def self.test(*foo)
+        seconds = rand 120 + 15
+        puts "TEST CALLED: #{Time.zone.now}: #{foo.inspect}: sleeping for #{seconds} seconds"
+        sleep(seconds)
+        puts "TEST DONE: #{Time.zone.now}: #{foo.inspect}"
+      end
+    end
+
+    MiddleClass = JobCreator
+  end
+end

data/app/models/logical/naf/job_fetcher.rb

@@ -0,0 +1,167 @@
+module Logical
+  module Naf
+    class JobFetcher
+      include ::Af::Application::SafeProxy
+
+      attr_reader :machine
+
+      def initialize(machine)
+        @machine = machine
+      end
+
+      def fetch_next_job
+        fetch_possible_jobs.each do |possible_job|
+          running_job = nil
+          ::Naf::HistoricalJob.lock_for_job_queue do
+            running_job = start_job(possible_job)
+          end
+
+          if running_job.present?
+            # Update tags
+            running_job.historical_job.remove_tags([::Naf::HistoricalJob::SYSTEM_TAGS[:pre_work]])
+            running_job.historical_job.add_tags([::Naf::HistoricalJob::SYSTEM_TAGS[:work]])
+
+            # found a job
+            parse_log_level(running_job)
+
+            return running_job
+          end
+        end
+
+        # no jobs found
+        return nil
+      end
+
+      private
+
+      # This method fetches queued jobs that can be started. In order for a job to
+      # run on a machine, we need to check affinities for historical_job_affinity_tabs
+      # and machine_affinity_slots.
+      #
+      # A job without affinity tabs can be run on any machine
+      # A job with affinity tabs can only run on machines:
+      #   - that have matching affinity slot(s)
+      #
+      # A machine with non-required affinity slots can run jobs:
+      #   - that don't have any affinity tabs
+      #   - that have affinity tab(s) that match affinity slot(s)
+      # A machine with required affinity slots can run jobs:
+      #   - that have affinity tab(s) that match the required affinity slot(s)
+      #     and if other affinity tabs are present, they also need to match
+      #     afffinity slots
+      #
+      def fetch_possible_jobs
+        possible_jobs = nil
+        if machine.machine_affinity_slots.select(&:required).present?
+          # Retrieve the machine's required affinities in order to not
+          # compute it several times.
+          required_machine_affinities = ::Naf::Machine.
+            select("ARRAY(
+              SELECT affinity_id
+              FROM #{::Naf.schema_name}.machine_affinity_slots
+              WHERE machine_id = #{machine.id} AND required = true
+              ORDER BY affinity_id) AS required_affinities").
+            group("required_affinities").
+            first.required_affinities
+
+          # Choose queued jobs that can be run by the machine
+          possible_jobs = ::Naf::QueuedJob.
+            select("#{::Naf.schema_name}.queued_jobs.id, #{::Naf.schema_name}.queued_jobs.priority, #{::Naf.schema_name}.queued_jobs.created_at").
+            weight_available_on_machine(machine).
+            runnable_by_machine(machine).
+            is_not_restricted_by_run_group(machine).
+            prerequisites_finished.
+            group("#{::Naf.schema_name}.queued_jobs.id, #{::Naf.schema_name}.queued_jobs.priority, #{::Naf.schema_name}.queued_jobs.created_at").
+            having("array(
+              select affinity_id::integer
+              from #{::Naf.schema_name}.historical_job_affinity_tabs
+              where historical_job_id = queued_jobs.id and affinity_id in (
+               select affinity_id
+               from #{::Naf.schema_name}.machine_affinity_slots
+               where machine_id = #{machine.id} and required = true)
+              order by affinity_id) = '#{required_machine_affinities}'").
+            order_by_priority.
+            limit(100)
+        elsif machine.machine_affinity_slots.present?
+          # Choose queued jobs that can be run by the machine
+          possible_jobs = ::Naf::QueuedJob.
+            weight_available_on_machine(machine).
+            runnable_by_machine(machine).
+            is_not_restricted_by_run_group(machine).
+            prerequisites_finished.
+            order_by_priority.limit(100)
+        else
+          # Machine can run any queued job
+          possible_jobs = ::Naf::QueuedJob.
+            weight_available_on_machine(machine).
+            is_not_restricted_by_run_group(machine).
+            prerequisites_finished.
+            order_by_priority.limit(100)
+        end
+      end
+
+      def start_job(possible_job)
+        sql = <<-SQL
+          UPDATE
+            #{::Naf::HistoricalJob.partition_table_name(possible_job.id)}
+          SET
+            started_at = NOW(),
+            started_on_machine_id = ?
+          WHERE
+            id = ? AND started_at IS NULL
+          RETURNING
+            *
+        SQL
+
+        historical_job = ::Naf::HistoricalJob.find_by_sql([sql, machine.id, possible_job.id]).first
+        if historical_job.present?
+          ::Naf::QueuedJob.delete(historical_job.id)
+          running_job = ::Naf::RunningJob.new(application_id: historical_job.application_id,
+                                              application_type_id: historical_job.application_type_id,
+                                              command: historical_job.command,
+                                              application_run_group_restriction_id: historical_job.application_run_group_restriction_id,
+                                              application_run_group_name: historical_job.application_run_group_name,
+                                              application_run_group_limit: historical_job.application_run_group_limit,
+                                              started_on_machine_id: historical_job.started_on_machine_id,
+                                              started_at: historical_job.started_at,
+                                              log_level: historical_job.log_level)
+          running_job.id = historical_job.id
+          running_job.save!
+
+          running_job
+        else
+          nil
+        end
+      end
+
+      def logger
+        return af_logger(self.class.name)
+      end
+
+      def parse_log_level(running_job)
+        log_levels = {}
+        unless machine.log_level.blank?
+          begin
+            log_level_hash = JSON.parse(machine.log_level)
+            log_levels.merge!(log_level_hash)
+          rescue StandardError => e
+            logger.error "couldn't parse machine.log_level: #{machine.log_level}: (#{e.message})"
+          end
+        end
+
+        unless running_job.application.nil? || running_job.application.log_level.blank?
+          begin
+            log_level_hash = JSON.parse(running_job.application.log_level)
+            log_levels.merge!(log_level_hash)
+          rescue StandardError => e
+            logger.error "couldn't parse running_job.application.log_level: #{running_job.application.log_level}: (#{e.message})"
+          end
+        end
+
+        running_job.log_level = log_levels.to_json
+        running_job.save!
+      end
+
+    end
+  end
+end

data/app/models/logical/naf/job_statuses/errored.rb

@@ -0,0 +1,27 @@
+module Logical
+  module Naf
+    module JobStatuses
+      class Errored
+
+        def self.all(conditions)
+          <<-SQL
+            (
+              SELECT
+                j.*, null AS "historical_job_id"
+              FROM
+                "#{::Naf.schema_name}"."historical_jobs" AS j
+              WHERE
+                j.finished_at IS NOT NULL AND
+                (j.exit_status > 0 OR
+                j.request_to_terminate = true)
+                #{conditions}
+              ORDER BY
+                finished_at desc
+            )
+          SQL
+        end
+
+      end
+    end
+  end
+end

data/app/models/logical/naf/job_statuses/finished.rb

@@ -0,0 +1,26 @@
+module Logical
+  module Naf
+    module JobStatuses
+      class Finished
+
+        def self.all(conditions)
+          <<-SQL
+            (
+              SELECT
+                j.*, null AS "historical_job_id"
+              FROM
+                "#{::Naf.schema_name}"."historical_jobs" AS j
+              WHERE
+                (j.finished_at IS NOT NULL OR
+                 j.request_to_terminate = true)
+                #{conditions}
+              ORDER BY
+                finished_at DESC NULLS LAST
+            )
+          SQL
+        end
+
+      end
+    end
+  end
+end

data/app/models/logical/naf/job_statuses/finished_less_minute.rb

@@ -0,0 +1,25 @@
+module Logical
+  module Naf
+    module JobStatuses
+      class FinishedLessMinute
+
+        def self.all(conditions)
+          <<-SQL
+            (
+              SELECT
+                j.*, NULL AS "historical_job_id"
+              FROM
+                "#{::Naf.schema_name}"."historical_jobs" AS j
+              WHERE
+                j.finished_at > '#{Time.zone.now - 1.minute}'
+                #{conditions}
+              ORDER BY
+                finished_at DESC
+            )
+          SQL
+        end
+
+      end
+    end
+  end
+end

data/app/models/logical/naf/job_statuses/queued.rb

@@ -0,0 +1,32 @@
+module Logical
+  module Naf
+    module JobStatuses
+      class Queued
+
+        def self.all(conditions)
+          <<-SQL
+            (
+              SELECT DISTINCT
+                j.*, jp."historical_job_id"
+              FROM
+                "#{::Naf.schema_name}"."historical_jobs" AS j
+              LEFT JOIN
+                "#{::Naf.schema_name}"."historical_job_prerequisites" AS jp
+                ON j."id" = jp."historical_job_id"
+              WHERE
+                j.finished_at IS NULL AND
+                j.request_to_terminate = false AND
+                jp.historical_job_id IS NULL AND
+                j.started_at IS NULL
+              #{conditions}
+              ORDER BY
+                created_at desc
+            )
+          SQL
+        end
+
+      end
+    end
+  end
+end
+

data/app/models/logical/naf/job_statuses/running.rb

@@ -0,0 +1,34 @@
+module Logical
+  module Naf
+    module JobStatuses
+      class Running
+
+        def self.all(status = :running, conditions)
+          if status == :queued
+            order = "created_at"
+            direction = "desc"
+          else
+            order = "started_at"
+            direction = "desc"
+          end
+          <<-SQL
+            (
+              SELECT
+                j.*, null AS "historical_job_id"
+              FROM
+                "#{::Naf.schema_name}"."historical_jobs" AS j
+              WHERE
+                j.started_at IS NOT NULL AND
+                j.finished_at IS NULL AND
+                j.request_to_terminate = false
+              #{conditions}
+              ORDER BY
+                #{order} #{direction}
+            )
+          SQL
+        end
+
+      end
+    end
+  end
+end