workhorse 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 720021fa190163c2d0d37f76b6bd0589698a7f3e
-  data.tar.gz: 817a622805d27e32a9e733c7a74e42ae9b7d2ce7
+  metadata.gz: 0cf16c2d211462d7875c56595c5d624ff30b8cfe
+  data.tar.gz: 4772dce847b5b7aa51f1931baa7a878d34bc0e06
 SHA512:
-  metadata.gz: 38e10cf259f6d02fd07f0ef2a568c69a913dcab9a0b720ec8db2730ebb03f3161ede58467c2f11c88106c5bd688f33d18e0c87294060e62833c3880b6e6e22cd
-  data.tar.gz: 86b461b572b35b5f6345b5026cadcbe6a01eadcb99befb7d51d10b118889254e88daf883aa57754ebf85a90382b7df8cced56167b971ca273845f169b42e7d52
+  metadata.gz: ca5aec71e437a2e151fa0b3356185c44bc12cb60cf09d817097e15307320247083eab40553da836e3dff911cfb98747c1a991755a72d30bb84f7bbd714544a55
+  data.tar.gz: 1dcfe3bda9f7c1a6d7dac78bf6a795cadb0a59248ec1da72e3a0fba89b91745b7053938f0f2a4570bf48155ae233968399b01a48e6620a53927f59b54444add2

data/CHANGELOG.md

@@ -1,5 +1,9 @@
 # Workhorse Change log
 
-## 0.1.0 - 2017-12-08
+## 0.2.0 - 2017-12-19
+
+* Adds support for job-level priorities
+
+## 0.1.0 - 2017-12-18
 
 * First feature-complete production release

data/FAQ.md

@@ -57,3 +57,15 @@ Workhorse::Worker.start_and_wait(pool_size: 5)
 ```
 
 This can then be started and "daemonized" using standard linux tools.
+
+## I'm getting random autoloading exeptions
+
+Make sure to always start the worker in *production mode*, i.e.:
+
+```bash
+RAILS_ENV=production bin/workhorse.rb start
+```
+
+## Why does workhorse not support timeouts?
+
+Generic timeout implementations are [a dangerous thing](http://www.mikeperham.com/2015/05/08/timeout-rubys-most-dangerous-api/) in Ruby. This is why we decided against providing this feature in Workhorse and recommend to implement the timeouts inside of your jobs - i.e. via network timeouts.

data/README.md

@@ -15,6 +15,8 @@ How it works:
 * Each worker is configurable as to which queue(s) it processes. Jobs in the
   same queue never run simultaneously. Jobs with no queue can always run in
   parallel.
+* Each job has a priority, the default being 0. Jobs with higher priorities
+  (lower is higher, 0 the highest) get processed first.
 * Each worker polls the database and spawns a number of threads to execute jobs
   of different queues simultaneously.
 
@@ -66,7 +68,7 @@ What it does not do:
 
 Workhorse can handle any jobs that support the `perform` method and are
 serializable. To queue a basic job, use the static method `Workhorse.enqueue`.
-You can optionally pass a queue name.
+You can optionally pass a queue name and a priority.
 
 ```ruby
 class MyJob
@@ -79,7 +81,7 @@ class MyJob
   end
 end
 
-Workhorse.enqueue MyJob.new('John'), queue: :test
+Workhorse.enqueue MyJob.new('John'), queue: :test, priority: 2
 ```
 
 ### RailsOps operations
@@ -89,7 +91,7 @@ Workhorse allows you to easily queue
 method `Workhorse.enqueue_op`:
 
 ```ruby
-Workhorse.enqueue_op Operations::Jobs::CleanUpDatabase, { quiet: true }, queue: :maintenance
+Workhorse.enqueue_op Operations::Jobs::CleanUpDatabase, { quiet: true }, queue: :maintenance, priority: 2
 ```
 
 Params passed using the second argument will be used for operation instantiation
@@ -98,7 +100,8 @@ at job execution.
 If you do not want to pass any params to the operation, just omit the second hash:
 
 ```ruby
-Workhorse.enqueue_op Operations::Jobs::CleanUpDatabase, queue: :maintenance
+Workhorse.enqueue_op Operations::Jobs::CleanUpDatabase, queue: :maintenance,
+priority: 2
 ```
 
 ## Configuring and starting workers
@@ -140,7 +143,7 @@ For this case, the workhorse install routine automatically creates the file
 The script can be called as follows:
 
 ```bash
-bin/workhorse.rb start|stop|status|watch|restart|usage
+RAILS_ENV=production bin/workhorse.rb start|stop|status|watch|restart|usage
 ```
 
 #### Background and customization
@@ -163,8 +166,6 @@ Please consult the [FAQ](FAQ.md).
 ## Roadmap
 
 * [ ] ActiveJob integration for Rails
-* [ ] Job timeouts
-* [ ] Job priorities
 
 ## Copyright
 

data/VERSION

@@ -1 +1 @@
-0.1.0
+0.2.0

data/lib/generators/workhorse/templates/create_table_jobs.rb

@@ -1,4 +1,4 @@
-class CreateTableJobs < ActiveRecord::Migration
+class CreateTableJobs < ActiveRecord::Migration[4.2]
   def change
     create_table :jobs, force: true do |t|
       t.string :state, null: false, default: 'waiting'
@@ -14,6 +14,8 @@ class CreateTableJobs < ActiveRecord::Migration
       t.datetime :failed_at
       t.text :last_error, limit: 4_294_967_295
 
+      t.integer :priority, null: false
+
       t.timestamps null: false
     end
 

data/lib/workhorse/enqueuer.rb

@@ -1,16 +1,17 @@
 module Workhorse
   module Enqueuer
     # Enqueue any object that is serializable and has a `perform` method
-    def enqueue(job, queue: nil)
+    def enqueue(job, queue: nil, priority: 0)
       return DbJob.create!(
         queue: queue,
+        priority: priority,
         handler: Marshal.dump(job)
       )
     end
 
     # Enqueue an ActiveJob job
     def enqueue_active_job(job)
-      enqueue job, queue: job.queue_name
+      enqueue job, queue: job.queue_name, priority: job.priority
     end
 
     # Enqueue the execution of an operation by its class and params

data/lib/workhorse/poller.rb

@@ -18,8 +18,8 @@ module Workhorse
       @thread = Thread.new do
         begin
           loop do
-            poll
             break unless running?
+            poll
             sleep
           end
         rescue => e
@@ -113,7 +113,7 @@ module Workhorse
       end
 
       # Order by creation date
-      select = select.order(table[:created_at].asc)
+      select = select.order(table[:priority].asc).order(table[:created_at].asc)
 
       # Limit number of records
       if is_oracle

data/lib/workhorse/worker.rb

@@ -52,6 +52,8 @@ module Workhorse
       @logger = logger
 
       fail 'Polling interval must be an integer.' unless @polling_interval.is_a?(Integer)
+
+      check_rails_env if defined?(Rails)
     end
 
     def log(text, level = :info)
@@ -134,6 +136,12 @@ module Workhorse
 
     private
 
+    def check_rails_env
+      unless Rails.env.production?
+        warn 'WARNING: Always run workhorse workers in production environment. Other environments can lead to unexpected behavior.'
+      end
+    end
+
     def trap_termination
       SHUTDOWN_SIGNALS.each do |signal|
         Signal.trap(signal) do

data/test/lib/db_schema.rb

@@ -4,7 +4,7 @@ ActiveRecord::Schema.define do
   create_table :jobs, force: true do |t|
     t.string :state, null: false, default: 'waiting'
     t.string :queue, null: true
-    t.text :handler, null: false
+    t.text :handler, null: false, limit: 4_294_967_295
 
     t.string :locked_by
     t.datetime :locked_at
@@ -13,7 +13,9 @@ ActiveRecord::Schema.define do
 
     t.datetime :succeeded_at
     t.datetime :failed_at
-    t.text :last_error
+    t.text :last_error, limit: 4_294_967_295
+
+    t.integer :priority, null: false
 
     t.timestamps null: false
   end

data/test/lib/jobs.rb

@@ -9,7 +9,7 @@ class BasicJob
 
   def perform
     results << @some_param
-    sleep @sleep_time
+    sleep @sleep_time if @sleep_time > 0
   end
 end
 

data/test/lib/test_helper.rb

@@ -15,10 +15,19 @@ class WorkhorseTest < ActiveSupport::TestCase
     options[:pool_size] ||= 5
     options[:polling_interval] ||= 1
 
+    with_worker(options) do
+      sleep time
+    end
+  end
+
+  def with_worker(options = {})
     w = Workhorse::Worker.new(options)
     w.start
-    sleep time
-    w.shutdown
+    begin
+      yield(w)
+    ensure
+      w.shutdown
+    end
   end
 end
 

data/test/workhorse/enqueuer_test.rb

@@ -25,6 +25,12 @@ class Workhorse::EnqueuerTest < WorkhorseTest
 
     db_job = Workhorse::DbJob.first
     assert_equal 'q1', db_job.queue
+    assert_equal 0, db_job.priority
+  end
+
+  def test_with_priority
+    Workhorse.enqueue BasicJob.new, priority: 1
+    assert_equal 1, Workhorse::DbJob.first.priority
   end
 
   def test_op

data/test/workhorse/performer_test.rb

@@ -5,14 +5,14 @@ class Workhorse::WorkerTest < WorkhorseTest
   # connections.
   def test_db_connections
     w = Workhorse::Worker.new polling_interval: 1, pool_size: 5
-    5.times do
+    2.times do
       Workhorse.enqueue DbConnectionTestJob.new
     end
     w.start
     sleep 1
     w.shutdown
 
-    assert_equal 5, DbConnectionTestJob.db_connections.count
-    assert_equal 5, DbConnectionTestJob.db_connections.uniq.count
+    assert_equal 2, DbConnectionTestJob.db_connections.count
+    assert_equal 2, DbConnectionTestJob.db_connections.uniq.count
   end
 end

data/test/workhorse/worker_test.rb

@@ -2,44 +2,41 @@ require 'test_helper'
 
 class Workhorse::WorkerTest < WorkhorseTest
   def test_idle
-    w = Workhorse::Worker.new(pool_size: 5, polling_interval: 1)
-    w.start
-    assert_equal 5, w.idle
+    with_worker(pool_size: 5, polling_interval: 1) do |w|
+      assert_equal 5, w.idle
 
-    Workhorse.enqueue BasicJob.new(sleep_time: 0.5)
+      sleep 0.5
+      Workhorse.enqueue BasicJob.new(sleep_time: 1)
 
-    sleep 0.1
-    assert_equal 4, w.idle
+      sleep 1
+      assert_equal 4, w.idle
 
-    sleep 0.5
-    assert_equal 5, w.idle
-
-    w.shutdown
+      sleep 1
+      assert_equal 5, w.idle
+    end
   end
 
   def test_start_and_shutdown
-    w = Workhorse::Worker.new
-    w.start
-    w.assert_state! :running
-
-    assert_raises RuntimeError do
-      w.start
-    end
+    with_worker do |w|
+      w.assert_state! :running
 
-    w.shutdown
-    w.shutdown # Should be ignored
+      assert_raises RuntimeError do
+        w.start
+      end
 
-    Workhorse.enqueue BasicJob.new
+      w.shutdown
+      w.shutdown # Should be ignored
+    end
   end
 
   def test_perform
-    w = Workhorse::Worker.new polling_interval: 1
-    Workhorse.enqueue BasicJob.new(sleep_time: 0.1)
-    assert_equal 'waiting', Workhorse::DbJob.first.state
+    with_worker(polling_interval: 1) do
+      sleep 0.1
+      Workhorse.enqueue BasicJob.new(sleep_time: 0.1)
+      assert_equal 'waiting', Workhorse::DbJob.first.state
 
-    w.start
-    sleep 1
-    w.shutdown
+      sleep 1
+    end
 
     assert_equal 'succeeded', Workhorse::DbJob.first.state
   end
@@ -48,10 +45,7 @@ class Workhorse::WorkerTest < WorkhorseTest
     BasicJob.results.clear
 
     Workhorse.enqueue BasicJob.new(some_param: 5, sleep_time: 0)
-    w = Workhorse::Worker.new polling_interval: 1
-    w.start
-    sleep 0.5
-    w.shutdown
+    work 0.5
 
     assert_equal 'succeeded', Workhorse::DbJob.first.state
 
@@ -60,21 +54,19 @@ class Workhorse::WorkerTest < WorkhorseTest
   end
 
   def test_term
-    w = Workhorse::Worker.new
-    w.start
-    Process.kill 'TERM', Process.pid
-    sleep 1
-    w.assert_state! :shutdown
-    w.shutdown
+    with_worker do |w|
+      Process.kill 'TERM', Process.pid
+      sleep 1
+      w.assert_state! :shutdown
+    end
   end
 
   def test_int
-    w = Workhorse::Worker.new
-    w.start
-    Process.kill 'INT', Process.pid
-    sleep 1
-    w.assert_state! :shutdown
-    w.shutdown
+    with_worker do |w|
+      Process.kill 'INT', Process.pid
+      sleep 1
+      w.assert_state! :shutdown
+    end
   end
 
   def test_no_queues
@@ -117,6 +109,19 @@ class Workhorse::WorkerTest < WorkhorseTest
     assert_equal 'succeeded', jobs[2].state
   end
 
+  def test_order_with_priorities
+    Workhorse.enqueue BasicJob.new(some_param: 6, sleep_time: 0), priority: 4
+    Workhorse.enqueue BasicJob.new(some_param: 4, sleep_time: 0), priority: 3
+    Workhorse.enqueue BasicJob.new(some_param: 5, sleep_time: 0), priority: 3
+    Workhorse.enqueue BasicJob.new(some_param: 3, sleep_time: 0), priority: 2
+    Workhorse.enqueue BasicJob.new(some_param: 2, sleep_time: 0), priority: 1
+    Workhorse.enqueue BasicJob.new(some_param: 1, sleep_time: 0), priority: 0
+
+    BasicJob.results.clear
+    work 6.5, pool_size: 1
+    assert_equal (1..6).to_a, BasicJob.results
+  end
+
   private
 
   def enqueue_in_multiple_queues

data/workhorse.gemspec

@@ -1,54 +1,55 @@
 # -*- encoding: utf-8 -*-
+# stub: workhorse 0.2.0 ruby lib
 
 Gem::Specification.new do |s|
-  s.name = "workhorse"
-  s.version = "0.1.0"
+  s.name = "workhorse".freeze
+  s.version = "0.2.0"
 
-  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
-  s.authors = ["Sitrox"]
-  s.date = "2017-12-18"
-  s.files = [".gitignore", ".releaser_config", ".rubocop.yml", ".travis.yml", "CHANGELOG.md", "FAQ.md", "Gemfile", "LICENSE", "README.md", "RUBY_VERSION", "Rakefile", "VERSION", "bin/rubocop", "lib/generators/workhorse/install_generator.rb", "lib/generators/workhorse/templates/bin/workhorse.rb", "lib/generators/workhorse/templates/config/initializers/workhorse.rb", "lib/generators/workhorse/templates/create_table_jobs.rb", "lib/workhorse.rb", "lib/workhorse/daemon.rb", "lib/workhorse/daemon/shell_handler.rb", "lib/workhorse/db_job.rb", "lib/workhorse/enqueuer.rb", "lib/workhorse/jobs/run_rails_op.rb", "lib/workhorse/performer.rb", "lib/workhorse/poller.rb", "lib/workhorse/pool.rb", "lib/workhorse/worker.rb", "test/lib/db_schema.rb", "test/lib/jobs.rb", "test/lib/test_helper.rb", "test/workhorse/enqueuer_test.rb", "test/workhorse/performer_test.rb", "test/workhorse/poller_test.rb", "test/workhorse/pool_test.rb", "test/workhorse/worker_test.rb", "workhorse.gemspec"]
-  s.require_paths = ["lib"]
-  s.rubygems_version = "2.0.14.1"
-  s.summary = "Multi-threaded job backend with database queuing for ruby."
-  s.test_files = ["test/lib/db_schema.rb", "test/lib/jobs.rb", "test/lib/test_helper.rb", "test/workhorse/enqueuer_test.rb", "test/workhorse/performer_test.rb", "test/workhorse/poller_test.rb", "test/workhorse/pool_test.rb", "test/workhorse/worker_test.rb"]
+  s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
+  s.require_paths = ["lib".freeze]
+  s.authors = ["Sitrox".freeze]
+  s.date = "2017-12-19"
+  s.files = [".gitignore".freeze, ".releaser_config".freeze, ".rubocop.yml".freeze, ".travis.yml".freeze, "CHANGELOG.md".freeze, "FAQ.md".freeze, "Gemfile".freeze, "LICENSE".freeze, "README.md".freeze, "RUBY_VERSION".freeze, "Rakefile".freeze, "VERSION".freeze, "bin/rubocop".freeze, "lib/generators/workhorse/install_generator.rb".freeze, "lib/generators/workhorse/templates/bin/workhorse.rb".freeze, "lib/generators/workhorse/templates/config/initializers/workhorse.rb".freeze, "lib/generators/workhorse/templates/create_table_jobs.rb".freeze, "lib/workhorse.rb".freeze, "lib/workhorse/daemon.rb".freeze, "lib/workhorse/daemon/shell_handler.rb".freeze, "lib/workhorse/db_job.rb".freeze, "lib/workhorse/enqueuer.rb".freeze, "lib/workhorse/jobs/run_rails_op.rb".freeze, "lib/workhorse/performer.rb".freeze, "lib/workhorse/poller.rb".freeze, "lib/workhorse/pool.rb".freeze, "lib/workhorse/worker.rb".freeze, "test/lib/db_schema.rb".freeze, "test/lib/jobs.rb".freeze, "test/lib/test_helper.rb".freeze, "test/workhorse/enqueuer_test.rb".freeze, "test/workhorse/performer_test.rb".freeze, "test/workhorse/poller_test.rb".freeze, "test/workhorse/pool_test.rb".freeze, "test/workhorse/worker_test.rb".freeze, "workhorse.gemspec".freeze]
+  s.rubygems_version = "2.6.14".freeze
+  s.summary = "Multi-threaded job backend with database queuing for ruby.".freeze
+  s.test_files = ["test/lib/db_schema.rb".freeze, "test/lib/jobs.rb".freeze, "test/lib/test_helper.rb".freeze, "test/workhorse/enqueuer_test.rb".freeze, "test/workhorse/performer_test.rb".freeze, "test/workhorse/poller_test.rb".freeze, "test/workhorse/pool_test.rb".freeze, "test/workhorse/worker_test.rb".freeze]
 
   if s.respond_to? :specification_version then
     s.specification_version = 4
 
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
-      s.add_development_dependency(%q<bundler>, ["~> 1.3"])
-      s.add_development_dependency(%q<rake>, [">= 0"])
-      s.add_development_dependency(%q<rubocop>, ["= 0.51.0"])
-      s.add_development_dependency(%q<minitest>, [">= 0"])
-      s.add_development_dependency(%q<mysql2>, ["~> 0.3.13"])
-      s.add_development_dependency(%q<benchmark-ips>, [">= 0"])
-      s.add_runtime_dependency(%q<activesupport>, [">= 0"])
-      s.add_runtime_dependency(%q<activerecord>, [">= 0"])
-      s.add_runtime_dependency(%q<schemacop>, ["~> 2.0"])
-      s.add_runtime_dependency(%q<concurrent-ruby>, [">= 0"])
+      s.add_development_dependency(%q<bundler>.freeze, ["~> 1.3"])
+      s.add_development_dependency(%q<rake>.freeze, [">= 0"])
+      s.add_development_dependency(%q<rubocop>.freeze, ["= 0.51.0"])
+      s.add_development_dependency(%q<minitest>.freeze, [">= 0"])
+      s.add_development_dependency(%q<mysql2>.freeze, ["~> 0.3.13"])
+      s.add_development_dependency(%q<benchmark-ips>.freeze, [">= 0"])
+      s.add_runtime_dependency(%q<activesupport>.freeze, [">= 0"])
+      s.add_runtime_dependency(%q<activerecord>.freeze, [">= 0"])
+      s.add_runtime_dependency(%q<schemacop>.freeze, ["~> 2.0"])
+      s.add_runtime_dependency(%q<concurrent-ruby>.freeze, [">= 0"])
     else
-      s.add_dependency(%q<bundler>, ["~> 1.3"])
-      s.add_dependency(%q<rake>, [">= 0"])
-      s.add_dependency(%q<rubocop>, ["= 0.51.0"])
-      s.add_dependency(%q<minitest>, [">= 0"])
-      s.add_dependency(%q<mysql2>, ["~> 0.3.13"])
-      s.add_dependency(%q<benchmark-ips>, [">= 0"])
-      s.add_dependency(%q<activesupport>, [">= 0"])
-      s.add_dependency(%q<activerecord>, [">= 0"])
-      s.add_dependency(%q<schemacop>, ["~> 2.0"])
-      s.add_dependency(%q<concurrent-ruby>, [">= 0"])
+      s.add_dependency(%q<bundler>.freeze, ["~> 1.3"])
+      s.add_dependency(%q<rake>.freeze, [">= 0"])
+      s.add_dependency(%q<rubocop>.freeze, ["= 0.51.0"])
+      s.add_dependency(%q<minitest>.freeze, [">= 0"])
+      s.add_dependency(%q<mysql2>.freeze, ["~> 0.3.13"])
+      s.add_dependency(%q<benchmark-ips>.freeze, [">= 0"])
+      s.add_dependency(%q<activesupport>.freeze, [">= 0"])
+      s.add_dependency(%q<activerecord>.freeze, [">= 0"])
+      s.add_dependency(%q<schemacop>.freeze, ["~> 2.0"])
+      s.add_dependency(%q<concurrent-ruby>.freeze, [">= 0"])
     end
   else
-    s.add_dependency(%q<bundler>, ["~> 1.3"])
-    s.add_dependency(%q<rake>, [">= 0"])
-    s.add_dependency(%q<rubocop>, ["= 0.51.0"])
-    s.add_dependency(%q<minitest>, [">= 0"])
-    s.add_dependency(%q<mysql2>, ["~> 0.3.13"])
-    s.add_dependency(%q<benchmark-ips>, [">= 0"])
-    s.add_dependency(%q<activesupport>, [">= 0"])
-    s.add_dependency(%q<activerecord>, [">= 0"])
-    s.add_dependency(%q<schemacop>, ["~> 2.0"])
-    s.add_dependency(%q<concurrent-ruby>, [">= 0"])
+    s.add_dependency(%q<bundler>.freeze, ["~> 1.3"])
+    s.add_dependency(%q<rake>.freeze, [">= 0"])
+    s.add_dependency(%q<rubocop>.freeze, ["= 0.51.0"])
+    s.add_dependency(%q<minitest>.freeze, [">= 0"])
+    s.add_dependency(%q<mysql2>.freeze, ["~> 0.3.13"])
+    s.add_dependency(%q<benchmark-ips>.freeze, [">= 0"])
+    s.add_dependency(%q<activesupport>.freeze, [">= 0"])
+    s.add_dependency(%q<activerecord>.freeze, [">= 0"])
+    s.add_dependency(%q<schemacop>.freeze, ["~> 2.0"])
+    s.add_dependency(%q<concurrent-ruby>.freeze, [">= 0"])
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: workhorse
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Sitrox
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-12-18 00:00:00.000000000 Z
+date: 2017-12-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler