que-scheduler 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 34c186335cf0386ab154de58d4ca489dbb186ce5
+  data.tar.gz: 8dea367b4b83245873e6657ec4191fa778d4698b
+SHA512:
+  metadata.gz: e5fa939bcc60786ac68de29c384165a7cdb21a054095e185f748bbd2732e24fd70d2f9ae49ee1b20251034600268567610df7e148867ec5e47fce84dbc69efd0
+  data.tar.gz: bc60257fed3c568e362a030277f32a033a36cbdb1a4cf7b1ab3b1e7307d49935d0d25615b0ad215b4edf4633805e3b65e7a5d78d1118dcc48fdf7ddeffa41e46

data/README.md

@@ -0,0 +1,90 @@
+que-scheduler
+================
+
+[![Build Status](https://travis-ci.org/resque/que-scheduler.svg?branch=master)](https://travis-ci.org/hlascelles/que-scheduler)
+[![Code Climate](https://codeclimate.com/github/hlascelles/sque-scheduler/badges/gpa.svg)](https://codeclimate.com/github/hlascelles/que-scheduler)
+
+### Description
+
+que-scheduler is an extension to [Que](https://github.com/chanks/que) that adds support for scheduling 
+items using a cron style configuration file. It works by running as a que job itself, determining what 
+needs to be run, enqueueing those jobs, then enqueueing itself to check again later.
+
+### Installation
+
+To install, add the gem to your Gemfile:
+
+```ruby
+gem 'que-scheduler'
+```
+
+You will need to specify a schedule config (see below). The default location that que-scheduler will 
+look for it is `config/que_schedule.yml`
+
+Finally, add a migration to start the job scheduler.
+
+```ruby
+Que::Scheduler::SchedulerJob.enqueue
+```
+
+### Environment Variables
+
+You can configure some aspects of the gem with environment variables.
+
+* `QUE_SCHEDULER_CONFIG_LOCATION` - The location of the schedule configuration (default config/que_schedule.yml)
+
+#### Schedule configuration
+
+The schedule file is a list of que job classes with arguments and a schedule frequency (in crontab 
+syntax). The format is a superset of the resque-scheduler config format, so it they can be used
+as-is with no modification, assuming the job classes are migrated from Resque to Que.
+
+It has one additional feature, `unmissable: true`. This is set on a job that must be run for every 
+single matching cron time that goes by, even if the system is offline over more than one match. To better process these unmissable jobs, they are always enqueued with the first 
+argument being the time that they were supposed to be processed.  
+ 
+For example:
+
+```yaml
+CancelAbandonedOrders:
+  cron: "*/5 * * * *"
+
+queue_documents_for_indexing:
+  cron: "0 0 * * *"
+  # By default the job name (hash key) will be taken as worker class name.
+  # If you want to have a different job name and class name, provide the 'class' option
+  class: "QueueDocuments"
+  queue: high
+  args:
+
+clear_leaderboards_contributors:
+  cron: "30 6 * * 1"
+  class: "ClearLeaderboards"
+  queue: low
+  args: contributors
+  
+DailyBatchReport:
+  cron: "0 3 * * *"
+  # This job will be run every day, and if workers are offline for several days, then the backlog
+  # will all be scheduled when they are restored, each with that events timestamp. 
+  unmissable: true
+```
+
+### Redundancy and Fail-Over
+
+Because of the way que-scheduler works, it requires no additional processes. It is, itself, a Que job.
+As long as there are Que workers functioning, then jobs will continue to be scheduled correctly. 
+
+### How it works
+
+que-scheduler is a job that reads a config file, then schedules itself endlessly on a delay, enqueueing 
+any jobs it determines that need to be run. The flow is as follows
+
+1. The job for the very first time.
+1. que-scheduler loads the config file, and notices it is new. It will not schedule any other jobs, except itself.
+1. Some time later it runs again. It knows what jobs it should be monitoring, and notices that some have are due. It enqueues those jobs and itself.
+1. After a deploy that changes the config, it notices a new job to monitor, and one to forget.
+
+### Thanks
+
+This gem was inspired by the makers of the excellent [Que](https://github.com/chanks/que) job scheduler gem. 

data/lib/que/scheduler.rb

@@ -0,0 +1,2 @@
+require 'que/scheduler/version'
+require 'que/scheduler/scheduler_job'

data/lib/que/scheduler/schedule_parser.rb

@@ -0,0 +1,64 @@
+require 'fugit'
+
+module Que
+  module Scheduler
+    ScheduleParserResult = Struct.new(:missed_jobs, :schedule_dictionary, :seconds_until_next_job)
+
+    class ScheduleParser
+      SCHEDULER_FREQUENCY = 60
+
+      class << self
+        def parse(jobs_list, as_time, last_time, known_jobs)
+          missed_jobs = {}
+          schedule_dictionary = []
+
+          # For each scheduled item, we need not schedule a job it if it has no history, as it is
+          # new. Otherwise, check how many times we have missed the job since the last run time.
+          # If it is "unmissable" then we schedule all of them, with the missed time as an arg,
+          # otherwise just schedule it once.
+          jobs_list.each do |desc|
+            schedule_dictionary << desc[:name]
+
+            next unless known_jobs.include?(desc[:name])
+            # This has been seen before. We should check if we have missed any executions.
+            missed = calculate_missed_runs(desc, last_time, as_time)
+            missed_jobs[desc[:clazz]] = missed unless missed.empty?
+          end
+
+          seconds_until_next_job = SCHEDULER_FREQUENCY # TODO: make it 1 sec after next known run
+          ScheduleParserResult.new(missed_jobs, schedule_dictionary, seconds_until_next_job)
+        end
+
+        private
+
+        def calculate_missed_runs(desc, last_scheduler_run_time, as_time)
+          jobs_for_class = []
+          missed_times = []
+          last_time = last_scheduler_run_time
+          while (next_run = next_run_time(desc[:cron], last_time, as_time))
+            missed_times << next_run
+            last_time = next_run
+          end
+
+          unless missed_times.empty?
+            if desc[:unmissable]
+              missed_times.each do |time_missed|
+                jobs_for_class << [time_missed] + desc[:args]
+              end
+            else
+              jobs_for_class << desc[:args]
+            end
+          end
+          jobs_for_class
+        end
+
+        def next_run_time(cron, from, to)
+          fugit_cron = Fugit::Cron.parse(cron)
+          next_time = fugit_cron.next_time(from)
+          next_run = next_time.to_local_time
+          next_run <= to ? next_run : nil
+        end
+      end
+    end
+  end
+end

data/lib/que/scheduler/scheduler_job.rb

@@ -0,0 +1,53 @@
+require 'que'
+require 'yaml'
+require_relative 'schedule_parser'
+
+module Que
+  module Scheduler
+    class SchedulerJob < Que::Job
+      # Highest possible priority.
+      @priority = 0
+
+      def run(last_time = nil, known_jobs = [])
+        ::ActiveRecord::Base.transaction do
+          last_time = last_time.nil? ? Time.now : Time.parse(last_time)
+          as_time = Time.now
+
+          result =
+            ScheduleParser.parse(SchedulerJob.scheduler_config, as_time, last_time, known_jobs)
+          result.missed_jobs.each do |job_class, args_arrays|
+            args_arrays.each { |args| job_class.enqueue(*args) }
+          end
+          SchedulerJob.enqueue(as_time + result.seconds_until_next_job, result.schedule_dictionary)
+          destroy
+        end
+      end
+
+      class << self
+        def scheduler_config
+          @scheduler_config ||= begin
+            location = ENV.fetch('QUE_SCHEDULER_CONFIG_LOCATION', 'config/que_schedule.yml')
+            jobs_list(YAML.load_file(location))
+          end
+        end
+
+        # Convert the config hash into a list of real classes and args, parsing the cron and
+        # unmissable parameters.
+        def jobs_list(schedule)
+          schedule.map do |k, v|
+            clazz = Object.const_get(v['class'] || k)
+            args = v.key?('args') ? v.fetch('args') : []
+            unmissable = v['unmissable'] == true
+            {
+              name: k,
+              clazz: clazz,
+              args: args,
+              cron: v.fetch('cron'),
+              unmissable: unmissable
+            }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/que/scheduler/version.rb

@@ -0,0 +1,5 @@
+module Que
+  module Scheduler
+    VERSION = '0.1.0'.freeze
+  end
+end

metadata

@@ -0,0 +1,203 @@
+--- !ruby/object:Gem::Specification
+name: que-scheduler
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Harry Lascelles
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-10-29 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: fugit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: que
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.12'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.12'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '4.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '4.0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.15'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.15'
+- !ruby/object:Gem::Dependency
+  name: pry-byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: que-testing
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.51'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.51'
+- !ruby/object:Gem::Dependency
+  name: timecop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.7'
+description: A lightweight cron scheduler for the async job worker Que
+email:
+- harry@harrylascelles.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- lib/que/scheduler.rb
+- lib/que/scheduler/schedule_parser.rb
+- lib/que/scheduler/scheduler_job.rb
+- lib/que/scheduler/version.rb
+homepage: https://rubygems.org/gems/que-scheduler
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.14
+signing_key: 
+specification_version: 4
+summary: A cron scheduler for Que
+test_files: []