que-scheduler 0.5.0 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6e9d071dc4b8b0357483a4a8aa5aa0b48b95e9d5
-  data.tar.gz: 8b8090e1eb7d1106a3a861d6c3ae2361a58a3eaf
+  metadata.gz: 0e2783e68e146ba7ba266c283c887e256963b451
+  data.tar.gz: 55c52e08b23cab6ef75e3b9c8e8d0cc368f123c2
 SHA512:
-  metadata.gz: 222d36133bf344092ad2ddff06031eecab82ee522f0da8da89d9f659f604b0770601acf676ba895003336a8076ef853538c554743f99db7571ac392e56548b92
-  data.tar.gz: 7cc612d46143d4fe9338e4d534c2b796c85692e2be77ed9204e02e8063e1430b6e07c5d40ee9624d93da2bcb5aa9f379911fd6fb52ae76aa70e264a6b01bddae
+  metadata.gz: 551365361fc0f13a520c396243577d972fed0b8b3dfe798fb92990f2d1b2779bf79544b9dfb6a258439bcd5bc17a773c45bb142fa39f3b9930eec8b7487fba5d
+  data.tar.gz: 363661214b0d6375a813ecc68b9dbd04e1cf367dca2ff978a9047a7e2aa99825df1ae64ebda7f533919a628845f804cea4f8ea9c32a6cb805754c61a80e10f8e

data/README.md

@@ -45,20 +45,35 @@ For example:
 CancelAbandonedOrders:
   cron: "*/5 * * * *"
 
+# Specify the job_class, using any name for the key.
 queue_documents_for_indexing:
   cron: "0 0 * * *"
-  # By default the job name (hash key) will be taken as worker class name.
-  # If you want to have a different job name and class name, provide the 'class' option
-  class: "QueueDocuments"
-  queue: high
-  args: ['reports', 'expenses']
-
-clear_leaderboards_contributors:
-  cron: "30 6 * * 1"
-  class: "ClearLeaderboards"
-  queue: low
-  args: contributors
+  class: QueueDocuments
   
+# Specify job queues
+ReportOrders:
+  cron: "0 0 * * *"
+  queue: reporting
+
+# Specify job priority using Que's number system
+BatchOrders:
+  cron: "0 0 * * *"
+  priority: 25
+  
+# Specify job queues
+SendOrders:
+  cron: "0 0 * * *"
+  args: ['open']
+  
+# Altogether now
+all_args_job:
+  cron: "0 0 * * *"
+  class: QueueDocuments
+  queue: reporting
+  priority: 25
+  args: ['open']
+  
+# Ensure you never miss a job, even after downtime
 DailyBatchReport:
   cron: "0 3 * * *"
   # This job will be run every day, and if workers are offline for several days, then the backlog

data/lib/que/scheduler/defined_job.rb

@@ -0,0 +1,32 @@
+require 'hashie'
+require 'fugit'
+
+# This is the definition of one scheduled job in the yml file.
+module Que
+  module Scheduler
+    class DefinedJob < Hashie::Dash
+      include Hashie::Extensions::Dash::PropertyTranslation
+
+      def self.err_field(f, v)
+        suffix = "in que-scheduler config #{QUE_SCHEDULER_CONFIG_LOCATION}"
+        raise "Invalid #{f} '#{v}' #{suffix}"
+      end
+
+      property :name, required: true
+      property :job_class, required: true
+      property :cron, transform_with: ->(v) { Fugit::Cron.new(v) || err_field(:cron, v) }
+      property :queue, transform_with: ->(v) { v.is_a?(String) ? v : err_field(:queue, v) }
+      property :priority, transform_with: ->(v) { v.is_a?(Integer) ? v : err_field(:priority, v) }
+      property :args
+      property :unmissable, default: false
+
+      # Given a "last time", return the next Time the event will occur, or nil if it
+      # is after "to".
+      def next_run_time(from, to)
+        next_time = cron.next_time(from)
+        next_run = next_time.to_local_time.in_time_zone(next_time.zone)
+        next_run <= to ? next_run : nil
+      end
+    end
+  end
+end

data/lib/que/scheduler/schedule_parser.rb

@@ -1,4 +1,5 @@
 require 'fugit'
+require 'backports/2.4.0/hash/compact'
 
 module Que
   module Scheduler
@@ -15,7 +16,7 @@ module Que
           # If it is "unmissable" then we schedule all of them, with the missed time as an arg,
           # otherwise just schedule it once.
           scheduler_config.each do |desc|
-            job_name = desc[:name]
+            job_name = desc.name
             schedule_dictionary << job_name
 
             # If we have never seen this job before, we don't want to scheduled any jobs for it.
@@ -26,7 +27,7 @@ module Que
             missed = calculate_missed_runs(
               desc, scheduler_job_args.last_run_time, scheduler_job_args.as_time
             )
-            missed_jobs[desc[:clazz]] = missed unless missed.empty?
+            missed_jobs[desc.job_class] = missed unless missed.empty?
           end
 
           ScheduleParserResult.new(missed_jobs, schedule_dictionary)
@@ -39,7 +40,7 @@ module Que
         def calculate_missed_runs(desc, last_run_time, as_time)
           missed_times = []
           last_time = last_run_time
-          while (next_run = next_run_time(desc[:cron], last_time, as_time))
+          while (next_run = desc.next_run_time(last_time, as_time))
             missed_times << next_run
             last_time = next_run
           end
@@ -47,28 +48,24 @@ module Que
           generate_required_jobs_list(desc, missed_times)
         end
 
-        # Given a cron, and a "last time", return the next Time the event will occur, or nil if it
-        # is after "to".
-        def next_run_time(cron, from, to)
-          fugit_cron = Fugit::Cron.parse(cron)
-          next_time = fugit_cron.next_time(from)
-          next_run = next_time.to_local_time.in_time_zone(next_time.zone)
-          next_run <= to ? next_run : nil
-        end
-
         # Given a job description, and the timestamps of the missed events, generate a list of jobs
         # that can be enqueued as an array of arrays of args.
         def generate_required_jobs_list(desc, missed_times)
           jobs_for_class = []
+
           unless missed_times.empty?
-            job_args = desc[:args]
+            options = {
+              args: desc.args,
+              queue: desc.queue,
+              priority: desc.priority
+            }.compact
 
-            if desc[:unmissable]
+            if desc.unmissable
               missed_times.each do |time_missed|
-                jobs_for_class << [time_missed] + job_args
+                jobs_for_class << options.merge(args: [time_missed] + (desc.args || []))
               end
             else
-              jobs_for_class << job_args
+              jobs_for_class << options
             end
           end
           jobs_for_class

data/lib/que/scheduler/scheduler_job.rb

@@ -1,10 +1,16 @@
 require 'que'
 require 'yaml'
+require 'backports/2.4.0/hash/compact'
+
+require_relative 'defined_job'
 require_relative 'schedule_parser'
 require_relative 'scheduler_job_args'
 
 module Que
   module Scheduler
+    QUE_SCHEDULER_CONFIG_LOCATION =
+      ENV.fetch('QUE_SCHEDULER_CONFIG_LOCATION', 'config/que_schedule.yml')
+
     class SchedulerJob < Que::Job
       SCHEDULER_FREQUENCY = 60
 
@@ -17,22 +23,25 @@ module Que
 
         ::ActiveRecord::Base.transaction do
           scheduler_job_args = SchedulerJobArgs.prepare_scheduler_job_args(options)
-          Que.log(message: "que-scheduler last ran at #{scheduler_job_args.last_run_time}.")
-          result = enqueue_required_jobs(scheduler_job_args)
+          logs = ["que-scheduler last ran at #{scheduler_job_args.last_run_time}."]
+          result = enqueue_required_jobs(scheduler_job_args, logs)
           enqueue_self_again(scheduler_job_args, result.schedule_dictionary)
+
+          # Only now we're sure nothing errored, log the results
+          logs.each { |str| Que.log(message: str) }
           destroy
         end
       end
 
       private
 
-      def enqueue_required_jobs(scheduler_job_args)
+      def enqueue_required_jobs(scheduler_job_args, logs)
         # Obtain the hash of missed jobs. Keys are the job classes, and the values are arrays
         # each containing more arrays for the arguments of that instance.
         result = ScheduleParser.parse(SchedulerJob.scheduler_config, scheduler_job_args)
         result.missed_jobs.each do |job_class, args_arrays|
           args_arrays.each do |args|
-            Que.log(message: "que-scheduler enqueueing #{job_class} with args: #{args}")
+            logs << "que-scheduler enqueueing #{job_class} with options: #{args}"
             job_class.enqueue(*args)
           end
         end
@@ -50,8 +59,7 @@ module Que
       class << self
         def scheduler_config
           @scheduler_config ||= begin
-            location = ENV.fetch('QUE_SCHEDULER_CONFIG_LOCATION', 'config/que_schedule.yml')
-            jobs_list(YAML.load_file(location))
+            jobs_list(YAML.load_file(QUE_SCHEDULER_CONFIG_LOCATION))
           end
         end
 
@@ -59,16 +67,17 @@ module Que
         # "unmissable" parameters.
         def jobs_list(schedule)
           schedule.map do |k, v|
-            clazz = Object.const_get(v['class'] || k)
-            args = v.key?('args') ? v.fetch('args') : []
-            unmissable = v['unmissable'] == true
-            {
-              name: k,
-              clazz: clazz,
-              args: args,
-              cron: v.fetch('cron'),
-              unmissable: unmissable
-            }
+            Que::Scheduler::DefinedJob.new(
+              {
+                name: k,
+                job_class: Object.const_get(v['class'] || k),
+                queue: v['queue'],
+                args: v['args'],
+                priority: v['priority'],
+                cron: v['cron'],
+                unmissable: v['unmissable']
+              }.compact
+            )
           end
         end
       end

data/lib/que/scheduler/scheduler_job_args.rb

@@ -1,5 +1,6 @@
 require 'hashie'
 
+# These are the args that are used for this particular run of the scheduler.
 module Que
   module Scheduler
     class SchedulerJobArgs < Hashie::Dash

data/lib/que/scheduler/version.rb

@@ -1,5 +1,5 @@
 module Que
   module Scheduler
-    VERSION = '0.5.0'.freeze
+    VERSION = '0.6.0'.freeze
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: que-scheduler
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.6.0
 platform: ruby
 authors:
 - Harry Lascelles
@@ -66,6 +66,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.10'
+- !ruby/object:Gem::Dependency
+  name: backports
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.10'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.10'
 description: A lightweight cron scheduler for the async job worker Que
 email:
 - harry@harrylascelles.com
@@ -75,6 +89,7 @@ extra_rdoc_files: []
 files:
 - README.md
 - lib/que/scheduler.rb
+- lib/que/scheduler/defined_job.rb
 - lib/que/scheduler/schedule_parser.rb
 - lib/que/scheduler/scheduler_job.rb
 - lib/que/scheduler/scheduler_job_args.rb