mini_scheduler 0.11.0 → 0.13.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d84546928a92e57ef72f4af8a5125c6d1ac12cddd32cd3b4203f819754b8c754
-  data.tar.gz: a5a8406b9c15ef74ea8729991137ebcc37f80ea4a367e7f9ce5da11cc6620ab9
+  metadata.gz: ba1f84f21c6f32b3085b9632348bd401ffbc0f5920bf1b850c645a7b1599335d
+  data.tar.gz: 0c79d4c284be049977dec995de59bc020c108fcb0a7e300bbb975a801787df94
 SHA512:
-  metadata.gz: c8b0a9e24bc8b785ffb0a4a01b4cef24a403604ec92851d862533b306ceab23008dad5419a7833e7c4f625d32206604f3308360db8942f61b7c254b0c2603a1d
-  data.tar.gz: e2f99830e4ede44cd66fba4d206cb6d2c16e712e0df30888a8677ec8b5ac6db78d3538da92769985cc84e8f211025f1682c51aa41716a034e39a5b49cdeeb56a
+  metadata.gz: 5fa187b74ece7225aab2ddf186699cc2c4e1ce58980660e7d1caa823580f24fb9283d79f2d58a99fc4f460d451b9b85ca66a76a617a8f84f08f51c70a00edd5a
+  data.tar.gz: 2a75aaae5394830ad9e613494f9edbd42bf83180b1e27e24bf1428f2f70e9b35d6bc314329f97681f230dd7e586db7e881d678d3c83d748fe7e8ab5451ce5564

data/CHANGELOG.md

@@ -1,13 +1,34 @@
-# 0.11.0 - 24-06-2019
+# 0.13.0 - 2020-11-30
+
+- Fix exception code so it has parity with Sidekiq 4.2.3 and up, version bump cause
+minimum version of Sikekiq changed.
+
+# 0.12.3 - 2020-10-15
+
+- Fixes a problem where scheduler didn't recover from Redis flush
+
+# 0.12.2 - 2019-09-11
+
+- Allow sorting schedule history by schedule name
+
+# 0.12.1 - 2019-08-30
+
+- Jobs that change family from per host to non per host can cause a tight loop
+
+# 0.12.0 - 2019-08-29
+
+- Add support for multiple workers which allows avoiding queue starvation
+
+# 0.11.0 - 2019-06-24
 
 - Correct situation where distributed mutex could end in a tight loop when
  redis could not be contacted
 
-# 0.9.2 - 26-04-2019
+# 0.9.2 - 2019-04-26
 
 - Correct UI so it displays durations that are longer than a minute
 
-# 0.9.1 - 21-01-2019
+# 0.9.1 - 2019-01-21
 
 - Remove dependency on ActiveSupport and add proper dependency for Sidekiq
 - Remove Discourse specific bits from Sidekiq web scheduler tab.

data/Gemfile

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 source 'https://rubygems.org'
 
 git_source(:github) { 'https://github.com/discourse/mini_scheduler' }

data/Guardfile

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 # A sample Guardfile
 # More info at https://github.com/guard/guard#readme
 

data/README.md

@@ -1,4 +1,4 @@
-[![Build Status](https://travis-ci.org/discourse/mini_scheduler.svg?branch=master)](https://travis-ci.org/discourse/mini_scheduler)
+[![Build Status](https://github.com/discourse/mini_scheduler/workflows/CI/badge.svg)](https://github.com/discourse/mini_scheduler/actions)
 [![Gem Version](https://badge.fury.io/rb/mini_scheduler.svg)](https://rubygems.org/gems/mini_scheduler)
 
 # mini_scheduler
@@ -30,6 +30,20 @@ In a Rails application, create files needed in your application to configure min
 
 An initializer is created named `config/initializers/mini_scheduler.rb` which lists all the configuration options.
 
+## Configuring MiniScheduler
+
+By default each instance of MiniScheduler will run with a single worker. To amend this behavior:
+
+```
+if Sidekiq.server? && defined?(Rails)
+  Rails.application.config.after_initialize do
+    MiniScheduler.start(workers: 5)
+  end
+end
+```
+
+This is useful for cases where you have extremely long running tasks that you would prefer did not starve.
+
 ## Usage
 
 Create jobs with a recurring schedule like this:

data/Rakefile

@@ -1,6 +1,19 @@
-require "bundler/gem_tasks"
+#!/usr/bin/env rake
+# frozen_string_literal: true
+
 require "rspec/core/rake_task"
+require 'bundler'
+
+begin
+  Bundler.setup :default, :development
+  Bundler::GemHelper.install_tasks
+rescue Bundler::BundlerError => error
+  $stderr.puts error.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit error.status_code
+end
 
 RSpec::Core::RakeTask.new(:spec)
 
-task default: :spec
+desc "Default: run tests"
+task default: [ :spec ]

data/lib/mini_scheduler.rb

@@ -52,11 +52,11 @@ module MiniScheduler
     @skip_schedule
   end
 
-  def self.start
+  def self.start(workers: 1)
     schedules = Manager.discover_schedules
 
     Manager.discover_queues.each do |queue|
-      manager = Manager.new(queue: queue)
+      manager = Manager.new(queue: queue, workers: workers)
 
       schedules.each do |schedule|
         if schedule.queue == queue

data/lib/mini_scheduler/manager.rb

@@ -2,7 +2,7 @@
 
 module MiniScheduler
   class Manager
-    attr_accessor :random_ratio, :redis, :enable_stats, :queue
+    attr_accessor :random_ratio, :redis, :enable_stats, :queue, :workers
 
     class Runner
       def initialize(manager)
@@ -12,11 +12,12 @@ module MiniScheduler
         @manager = manager
         @hostname = manager.hostname
 
-        @reschedule_orphans_thread = Thread.new do
+        @recovery_thread = Thread.new do
           while !@stopped
             sleep 60
 
             @mutex.synchronize do
+              repair_queue
               reschedule_orphans
             end
           end
@@ -29,9 +30,12 @@ module MiniScheduler
             sleep (@manager.keep_alive_duration / 2)
           end
         end
-        @thread = Thread.new do
-          while !@stopped
-            process_queue
+        @threads = []
+        manager.workers.times do
+          @threads << Thread.new do
+            while !@stopped
+              process_queue
+            end
           end
         end
       end
@@ -42,6 +46,12 @@ module MiniScheduler
         MiniScheduler.handle_job_exception(ex, message: "Scheduling manager keep-alive")
       end
 
+      def repair_queue
+        @manager.repair_queue
+      rescue => ex
+        MiniScheduler.handle_job_exception(ex, message: "Scheduling manager queue repair")
+      end
+
       def reschedule_orphans
         @manager.reschedule_orphans!
       rescue => ex
@@ -82,7 +92,7 @@ module MiniScheduler
 
           klass.new.perform
         rescue => e
-          MiniScheduler.handle_job_exception(e, message: "Running a scheduled job", job: klass)
+          MiniScheduler.handle_job_exception(e, message: "Running a scheduled job", job: { "class" => klass })
 
           error = "#{e.class}: #{e.message} #{e.backtrace.join("\n")}"
           failed = true
@@ -119,19 +129,19 @@ module MiniScheduler
           @stopped = true
 
           @keep_alive_thread.kill
-          @reschedule_orphans_thread.kill
+          @recovery_thread.kill
 
           @keep_alive_thread.join
-          @reschedule_orphans_thread.join
+          @recovery_thread.join
 
           enq(nil)
 
           kill_thread = Thread.new do
             sleep 0.5
-            @thread.kill
+            @threads.each(&:kill)
           end
 
-          @thread.join
+          @threads.each(&:join)
           kill_thread.kill
           kill_thread.join
         end
@@ -171,6 +181,7 @@ module MiniScheduler
 
     def initialize(options = nil)
       @queue = options && options[:queue] || "default"
+      @workers = options && options[:workers] || 1
       @redis = MiniScheduler.redis
       @random_ratio = 0.1
       unless options && options[:skip_runner]
@@ -248,6 +259,15 @@ module MiniScheduler
       nil
     end
 
+    def repair_queue
+      return if redis.exists?(self.class.queue_key(queue)) ||
+        redis.exists?(self.class.queue_key(queue, hostname))
+
+      self.class.discover_schedules
+        .select { |schedule| schedule.queue == queue }
+        .each { |schedule| ensure_schedule!(schedule) }
+    end
+
     def tick
       lock do
         schedule_next_job
@@ -261,11 +281,14 @@ module MiniScheduler
 
       if due.to_i <= Time.now.to_i
         klass = get_klass(key)
-        unless klass
+        if !klass || (
+          (klass.is_per_host && !hostname) || (hostname && !klass.is_per_host)
+        )
           # corrupt key, nuke it (renamed job or something)
           redis.zrem Manager.queue_key(queue, hostname), key
           return
         end
+
         info = schedule_info(klass)
         info.prev_run = Time.now.to_i
         info.prev_result = "QUEUED"

data/lib/mini_scheduler/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module MiniScheduler
-  VERSION = "0.11.0"
+  VERSION = "0.13.0"
 end

data/lib/mini_scheduler/views/history.erb

@@ -2,7 +2,20 @@
   <div class="col-sm-12">
     <h3>Scheduler History</h3>
   </div>
+
+  <div class="col-sm-12">
+    <form>
+      <select name="filter">
+        <option value="All" <%= "selected" if !@filter %>>All</option>
+        <% @schedules.each do |schedule| %>
+          <option value="<%=schedule%>" <%= "selected" if @filter == schedule.to_s%>><%=schedule%></option>
+        <% end %>
+      </select>
+      <input type="submit" value="Filter">
+    </form>
+  </div>
 </header>
+<br>
 
 <div class="container">
   <div class="row">

data/lib/mini_scheduler/views/scheduler.erb

@@ -39,7 +39,7 @@
             <td>
               <%= sane_duration @info.prev_duration %>
             </td>
-            <td>
+            <td style="word-wrap: break-word">
               <%= @info.current_owner %>
             </td>
             <td>

data/lib/mini_scheduler/web.rb

@@ -4,6 +4,20 @@ module MiniScheduler
   module Web
     VIEWS = File.expand_path('views', File.dirname(__FILE__)) unless defined? VIEWS
 
+    def self.find_schedules_by_time
+      Manager.discover_schedules.sort do |a, b|
+        a_next = a.schedule_info.next_run
+        b_next = b.schedule_info.next_run
+        if a_next && b_next
+          a_next <=> b_next
+        elsif a_next
+          -1
+        else
+          1
+        end
+      end
+    end
+
     def self.registered(app)
 
       app.helpers do
@@ -24,23 +38,24 @@ module MiniScheduler
 
       app.get "/scheduler" do
         MiniScheduler.before_sidekiq_web_request&.call
-        @schedules = Manager.discover_schedules.sort do |a, b|
-          a_next = a.schedule_info.next_run
-          b_next = b.schedule_info.next_run
-          if a_next && b_next
-            a_next <=> b_next
-          elsif a_next
-            -1
-          else
-            1
-          end
-        end
+        @schedules = Web.find_schedules_by_time
         erb File.read(File.join(VIEWS, 'scheduler.erb')), locals: { view_path: VIEWS }
       end
 
       app.get "/scheduler/history" do
         MiniScheduler.before_sidekiq_web_request&.call
-        @scheduler_stats = Stat.order('started_at desc').limit(200)
+        @schedules = Manager.discover_schedules
+        @schedules.sort_by!(&:to_s)
+        @scheduler_stats = Stat.order('started_at desc')
+
+        @filter = params[:filter]
+        names = @schedules.map(&:to_s)
+        @filter = nil if !names.include?(@filter)
+        if @filter
+          @scheduler_stats = @scheduler_stats.where(name: @filter)
+        end
+
+        @scheduler_stats = @scheduler_stats.limit(200)
         erb File.read(File.join(VIEWS, 'history.erb')), locals: { view_path: VIEWS }
       end
 

data/mini_scheduler.gemspec

@@ -15,14 +15,10 @@ Gem::Specification.new do |spec|
   spec.homepage      = "https://github.com/discourse/mini_scheduler"
   spec.license       = "MIT"
 
-  # Specify which files should be added to the gem when it is released.
-  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
-  spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
-    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
-  end
+  spec.files = `git ls-files`.split($/).reject { |s| s =~ /^(spec|\.)/ }
   spec.require_paths = ["lib"]
 
-  spec.add_dependency "sidekiq"
+  spec.add_dependency "sidekiq", ">= 4.2.3"
 
   spec.add_development_dependency "pg", ">= 1.0"
   spec.add_development_dependency "activesupport", ">= 5.2"
@@ -32,4 +28,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "guard-rspec"
   spec.add_development_dependency "mock_redis"
   spec.add_development_dependency "rake"
+  spec.add_development_dependency 'rubocop-discourse'
 end

metadata

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: mini_scheduler
 version: !ruby/object:Gem::Version
-  version: 0.11.0
+  version: 0.13.0
 platform: ruby
 authors:
 - Sam Saffron
 - Neil Lalonde
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-06-24 00:00:00.000000000 Z
+date: 2020-11-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sidekiq
@@ -17,14 +17,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 4.2.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 4.2.3
 - !ruby/object:Gem::Dependency
   name: pg
   requirement: !ruby/object:Gem::Requirement
@@ -137,6 +137,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: rubocop-discourse
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Adds recurring jobs for Sidekiq
 email:
 - neil.lalonde@discourse.org
@@ -144,10 +158,6 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- ".gitignore"
-- ".rspec"
-- ".rubocop.yml"
-- ".travis.yml"
 - CHANGELOG.md
 - Gemfile
 - Guardfile
@@ -172,7 +182,7 @@ homepage: https://github.com/discourse/mini_scheduler
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -187,8 +197,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
-signing_key: 
+rubygems_version: 3.1.4
+signing_key:
 specification_version: 4
 summary: Adds recurring jobs for Sidekiq
 test_files: []

data/.gitignore

@@ -1,13 +0,0 @@
-/.bundle/
-/.yardoc
-/_yardoc/
-/coverage/
-/doc/
-/pkg/
-/spec/reports/
-/tmp/
-Gemfile.lock
-.DS_Store
-*.swp
-
-.rubocop-https---raw-githubusercontent-com-discourse-discourse-master--rubocop-yml

data/.rspec

@@ -1 +0,0 @@
---require spec_helper --color

data/.rubocop.yml

@@ -1 +0,0 @@
-inherit_from: https://raw.githubusercontent.com/discourse/discourse/master/.rubocop.yml

data/.travis.yml

@@ -1,18 +0,0 @@
-language: ruby
-rvm:
-  - ruby-head
-  - 2.5
-  - 2.6
-
-before_install:
-  - gem install bundler
-
-cache: bundler
-sudo: false
-
-services:
-  - redis-server
-
-matrix:
-  allow_failures:
-    - rvm: ruby-head