que-scheduler 0.7.0 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a0e9f48536b268d1387f247f8a7ee52a55dbfef5
-  data.tar.gz: 148ab173c0eb5c22a6cfd6bf83fb7e846570c705
+  metadata.gz: 38355bcd24a4b7a7ec92449a833bcbf4977c0504
+  data.tar.gz: b01c3e96f925e9cceab9e3e1f8a6cac606fa8e25
 SHA512:
-  metadata.gz: 508e5271f7f35b62b6704a4f7633d026e149ede22f7f6497af8296aed7fd5c1bfcd8d1fbdd6b4ce4ec03adc59dedae195500830453038bfe0f33fb77c99f49ec
-  data.tar.gz: f18b35343f7a9075db5771b2c2f2917fb49df8efcd81fe055e5c048e2742d21623051291d81f5527af7b15664e178cd3eeac9516cc96d5c11060cf1e233e593f
+  metadata.gz: fb6dcde2cfd75d95c90dab1c5d72d678196ce6f5d288dbe9b86c45dbdfa78e6522c031e36959c5856399699c0327753d523e2de2938467695a367b50f8bbea18
+  data.tar.gz: 07e5303c3a9e0b41f9e8742c83e9bc9e68932875e0d70894679c343b8b342efa152d148b5c8398b0b68866fcd1c27e18576001f8b18f75e4e9b10d14ac580a0d

data/README.md

@@ -50,17 +50,17 @@ queue_documents_for_indexing:
   cron: "0 0 * * *"
   class: QueueDocuments
   
-# Specify job queues
+# Specify the job queue
 ReportOrders:
   cron: "0 0 * * *"
   queue: reporting
 
-# Specify job priority using Que's number system
+# Specify the job priority using Que's number system
 BatchOrders:
   cron: "0 0 * * *"
   priority: 25
   
-# Specify job queues
+# Specify job arguments
 SendOrders:
   cron: "0 0 * * *"
   args: ['open']
@@ -95,6 +95,12 @@ are no HA concerns to worry about and no namespace collisions between different
 
 Additionally, like Que, when your database is backed up, your scheduling state is stored too.
 
+## Multiple scheduler detection
+
+No matter how many tasks you have defined in your config, you will only ever need one que-scheduler
+job enqueued. que-scheduler knows this, and it will check before performing any operations that 
+there is only one of itself present.
+
 ## How it works
 
 que-scheduler is a job that reads a config file, enqueues any jobs it determines that need to be run,

data/lib/que/scheduler/defined_job.rb

@@ -1,14 +1,15 @@
 require 'hashie'
 require 'fugit'
 
-# This is the definition of one scheduled job in the yml file.
+# This is the definition of one scheduleable job in the que-scheduler config yml file.
 module Que
   module Scheduler
     class DefinedJob < Hashie::Dash
       include Hashie::Extensions::Dash::PropertyTranslation
 
       def self.err_field(f, v)
-        suffix = "in que-scheduler config #{QUE_SCHEDULER_CONFIG_LOCATION}"
+        suffix = 'in que-scheduler config ' \
+                 "#{Que::Scheduler::ScheduleParser::QUE_SCHEDULER_CONFIG_LOCATION}"
         raise "Invalid #{f} '#{v}' #{suffix}"
       end
 

data/lib/que/scheduler/enqueueing_calculator.rb

@@ -0,0 +1,76 @@
+require 'fugit'
+require 'backports/2.4.0/hash/compact'
+
+module Que
+  module Scheduler
+    EnqueueingCalculatorResult = Struct.new(:missed_jobs, :schedule_dictionary)
+
+    class EnqueueingCalculator
+      class << self
+        def parse(scheduler_config, scheduler_job_args)
+          missed_jobs = {}
+          schedule_dictionary = []
+
+          # For each scheduled item, we need not schedule a job it if it has no history, as it is
+          # new. Otherwise, check how many times we have missed the job since the last run time.
+          # If it is "unmissable" then we schedule all of them, with the missed time as an arg,
+          # otherwise just schedule it once.
+          scheduler_config.each do |desc|
+            job_name = desc.name
+            schedule_dictionary << job_name
+
+            # If we have never seen this job before, we don't want to scheduled any jobs for it.
+            # But we have added it to the dictionary, so it will be used to enqueue jobs next time.
+            next unless scheduler_job_args.job_dictionary.include?(job_name)
+
+            # This has been seen before. We should check if we have missed any executions.
+            missed = calculate_missed_runs(
+              desc, scheduler_job_args.last_run_time, scheduler_job_args.as_time
+            )
+            missed_jobs[desc.job_class] = missed unless missed.empty?
+          end
+
+          EnqueueingCalculatorResult.new(missed_jobs, schedule_dictionary)
+        end
+
+        private
+
+        # Given a job description, the last scheduler run time, and this run time, return all
+        # the instances that should be enqueued for that job class.
+        def calculate_missed_runs(desc, last_run_time, as_time)
+          missed_times = []
+          last_time = last_run_time
+          while (next_run = desc.next_run_time(last_time, as_time))
+            missed_times << next_run
+            last_time = next_run
+          end
+
+          generate_required_jobs_list(desc, missed_times)
+        end
+
+        # Given a job description, and the timestamps of the missed events, generate a list of jobs
+        # that can be enqueued as an array of arrays of args.
+        def generate_required_jobs_list(desc, missed_times)
+          jobs_for_class = []
+
+          unless missed_times.empty?
+            options = {
+              args: desc.args,
+              queue: desc.queue,
+              priority: desc.priority
+            }.compact
+
+            if desc.unmissable
+              missed_times.each do |time_missed|
+                jobs_for_class << options.merge(args: [time_missed] + (desc.args || []))
+              end
+            else
+              jobs_for_class << options
+            end
+          end
+          jobs_for_class
+        end
+      end
+    end
+  end
+end

data/lib/que/scheduler/schedule_parser.rb

@@ -1,74 +1,37 @@
-require 'fugit'
+require 'yaml'
 require 'backports/2.4.0/hash/compact'
 
+require_relative 'defined_job'
+
 module Que
   module Scheduler
-    ScheduleParserResult = Struct.new(:missed_jobs, :schedule_dictionary)
+    module ScheduleParser
+      QUE_SCHEDULER_CONFIG_LOCATION =
+        ENV.fetch('QUE_SCHEDULER_CONFIG_LOCATION', 'config/que_schedule.yml')
 
-    class ScheduleParser
       class << self
-        def parse(scheduler_config, scheduler_job_args)
-          missed_jobs = {}
-          schedule_dictionary = []
-
-          # For each scheduled item, we need not schedule a job it if it has no history, as it is
-          # new. Otherwise, check how many times we have missed the job since the last run time.
-          # If it is "unmissable" then we schedule all of them, with the missed time as an arg,
-          # otherwise just schedule it once.
-          scheduler_config.each do |desc|
-            job_name = desc.name
-            schedule_dictionary << job_name
-
-            # If we have never seen this job before, we don't want to scheduled any jobs for it.
-            # But we have added it to the dictionary, so it will be used to enqueue jobs next time.
-            next unless scheduler_job_args.job_dictionary.include?(job_name)
-
-            # This has been seen before. We should check if we have missed any executions.
-            missed = calculate_missed_runs(
-              desc, scheduler_job_args.last_run_time, scheduler_job_args.as_time
-            )
-            missed_jobs[desc.job_class] = missed unless missed.empty?
-          end
-
-          ScheduleParserResult.new(missed_jobs, schedule_dictionary)
-        end
-
-        private
-
-        # Given a job description, the last scheduler run time, and this run time, return all
-        # the instances that should be enqueued for that job class.
-        def calculate_missed_runs(desc, last_run_time, as_time)
-          missed_times = []
-          last_time = last_run_time
-          while (next_run = desc.next_run_time(last_time, as_time))
-            missed_times << next_run
-            last_time = next_run
+        def scheduler_config
+          @scheduler_config ||= begin
+            jobs_list(YAML.load_file(QUE_SCHEDULER_CONFIG_LOCATION))
           end
-
-          generate_required_jobs_list(desc, missed_times)
         end
 
-        # Given a job description, and the timestamps of the missed events, generate a list of jobs
-        # that can be enqueued as an array of arrays of args.
-        def generate_required_jobs_list(desc, missed_times)
-          jobs_for_class = []
-
-          unless missed_times.empty?
-            options = {
-              args: desc.args,
-              queue: desc.queue,
-              priority: desc.priority
-            }.compact
-
-            if desc.unmissable
-              missed_times.each do |time_missed|
-                jobs_for_class << options.merge(args: [time_missed] + (desc.args || []))
-              end
-            else
-              jobs_for_class << options
-            end
+        # Convert the config hash into a list of real classes and args, parsing the cron and
+        # "unmissable" parameters.
+        def jobs_list(schedule)
+          schedule.map do |k, v|
+            Que::Scheduler::DefinedJob.new(
+              {
+                name: k,
+                job_class: Object.const_get(v['class'] || k),
+                queue: v['queue'],
+                args: v['args'],
+                priority: v['priority'],
+                cron: v['cron'],
+                unmissable: v['unmissable']
+              }.compact
+            )
           end
-          jobs_for_class
         end
       end
     end

data/lib/que/scheduler/scheduler_job.rb

@@ -1,17 +1,13 @@
 require 'que'
-require 'yaml'
-require 'backports/2.4.0/hash/compact'
 
-require_relative 'defined_job'
 require_relative 'schedule_parser'
+require_relative 'enqueueing_calculator'
 require_relative 'scheduler_job_args'
 
 module Que
   module Scheduler
-    QUE_SCHEDULER_CONFIG_LOCATION =
-      ENV.fetch('QUE_SCHEDULER_CONFIG_LOCATION', 'config/que_schedule.yml')
-
     class SchedulerJob < Que::Job
+      SCHEDULER_COUNT_SQL = "SELECT COUNT(*) FROM que_jobs WHERE job_class = '#{name}'".freeze
       SCHEDULER_FREQUENCY = 60
 
       # Highest possible priority.
@@ -22,16 +18,17 @@ module Que
         options = { last_run_time: options, job_dictionary: oldarg } if oldarg.present?
 
         ::ActiveRecord::Base.transaction do
+          assert_one_scheduler_job
           scheduler_job_args = SchedulerJobArgs.prepare_scheduler_job_args(options)
           logs = ["que-scheduler last ran at #{scheduler_job_args.last_run_time}."]
 
           # It's possible one worker node has severe clock skew, and reports a time earlier than
           # the last run. If so, log, and rescheduled with the same last run at.
           if scheduler_job_args.as_time < scheduler_job_args.last_run_time
-            SchedulerJob.handle_clock_skew(scheduler_job_args, logs)
+            handle_clock_skew(scheduler_job_args, logs)
           else
             # Otherwise, run as normal
-            SchedulerJob.handle_normal_call(scheduler_job_args, logs)
+            handle_normal_call(scheduler_job_args, logs)
           end
 
           # Only now we're sure nothing errored, log the results
@@ -42,70 +39,50 @@ module Que
 
       private
 
-      class << self
-        def scheduler_config
-          @scheduler_config ||= begin
-            jobs_list(YAML.load_file(QUE_SCHEDULER_CONFIG_LOCATION))
-          end
-        end
-
-        # Convert the config hash into a list of real classes and args, parsing the cron and
-        # "unmissable" parameters.
-        def jobs_list(schedule)
-          schedule.map do |k, v|
-            Que::Scheduler::DefinedJob.new(
-              {
-                name: k,
-                job_class: Object.const_get(v['class'] || k),
-                queue: v['queue'],
-                args: v['args'],
-                priority: v['priority'],
-                cron: v['cron'],
-                unmissable: v['unmissable']
-              }.compact
-            )
-          end
-        end
+      def assert_one_scheduler_job
+        schedulers = ActiveRecord::Base.connection.execute(SCHEDULER_COUNT_SQL).first['count'].to_i
+        return if schedulers == 1
+        raise "Only one #{self.class.name} should be enqueued. #{schedulers} were found."
+      end
 
-        def handle_normal_call(scheduler_job_args, logs)
-          result = enqueue_required_jobs(scheduler_job_args, logs)
-          enqueue_self_again(
-            scheduler_job_args.as_time,
-            scheduler_job_args.as_time,
-            result.schedule_dictionary
-          )
-        end
+      def handle_normal_call(scheduler_job_args, logs)
+        result = enqueue_required_jobs(scheduler_job_args, logs)
+        enqueue_self_again(
+          scheduler_job_args.as_time,
+          scheduler_job_args.as_time,
+          result.schedule_dictionary
+        )
+      end
 
-        def enqueue_required_jobs(scheduler_job_args, logs)
-          # Obtain the hash of missed jobs. Keys are the job classes, and the values are arrays
-          # each containing more arrays for the arguments of that instance.
-          result = ScheduleParser.parse(SchedulerJob.scheduler_config, scheduler_job_args)
-          result.missed_jobs.each do |job_class, args_arrays|
-            args_arrays.each do |args|
-              logs << "que-scheduler enqueueing #{job_class} with options: #{args}"
-              job_class.enqueue(*args)
-            end
+      def enqueue_required_jobs(scheduler_job_args, logs)
+        # Obtain the hash of missed jobs. Keys are the job classes, and the values are arrays
+        # each containing more arrays for the arguments of that instance.
+        result = EnqueueingCalculator.parse(ScheduleParser.scheduler_config, scheduler_job_args)
+        result.missed_jobs.each do |job_class, args_arrays|
+          args_arrays.each do |args|
+            logs << "que-scheduler enqueueing #{job_class} with args: #{args}"
+            job_class.enqueue(*args)
           end
-          result
         end
+        result
+      end
 
-        def enqueue_self_again(last_full_execution, this_run_time, new_job_dictionary)
-          SchedulerJob.enqueue(
-            last_run_time: last_full_execution.iso8601,
-            job_dictionary: new_job_dictionary,
-            run_at: this_run_time.beginning_of_minute + SCHEDULER_FREQUENCY
-          )
-        end
+      def handle_clock_skew(scheduler_job_args, logs)
+        logs << 'que-scheduler detected worker with time older than last run. ' \
+                    'Rescheduling without enqueueing jobs.'
+        enqueue_self_again(
+          scheduler_job_args.last_run_time,
+          scheduler_job_args.as_time,
+          scheduler_job_args.job_dictionary
+        )
+      end
 
-        def handle_clock_skew(scheduler_job_args, logs)
-          logs << 'que-scheduler detected worker with time older than last run. ' \
-                      'Rescheduling without enqueueing jobs.'
-          enqueue_self_again(
-            scheduler_job_args.last_run_time,
-            scheduler_job_args.as_time,
-            scheduler_job_args.job_dictionary
-          )
-        end
+      def enqueue_self_again(last_full_execution, this_run_time, new_job_dictionary)
+        SchedulerJob.enqueue(
+          last_run_time: last_full_execution.iso8601,
+          job_dictionary: new_job_dictionary,
+          run_at: this_run_time.beginning_of_minute + SCHEDULER_FREQUENCY
+        )
       end
     end
   end

data/lib/que/scheduler/version.rb

@@ -1,5 +1,5 @@
 module Que
   module Scheduler
-    VERSION = '0.7.0'.freeze
+    VERSION = '0.8.0'.freeze
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: que-scheduler
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.8.0
 platform: ruby
 authors:
 - Harry Lascelles
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-11-05 00:00:00.000000000 Z
+date: 2017-11-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -90,6 +90,7 @@ files:
 - README.md
 - lib/que/scheduler.rb
 - lib/que/scheduler/defined_job.rb
+- lib/que/scheduler/enqueueing_calculator.rb
 - lib/que/scheduler/schedule_parser.rb
 - lib/que/scheduler/scheduler_job.rb
 - lib/que/scheduler/scheduler_job_args.rb