workhorse 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 76af1f05b3bf26fa8ecf78fdef9bd16d651f4c21
-  data.tar.gz: 71b70edd2c90872a13f5c27c9f512a9b61808b63
+  metadata.gz: cf7da416379ae17a4defae96289beca3c4dd1d58
+  data.tar.gz: 5b4e9cb886632f4a78982506747e1bcbdef1a7f3
 SHA512:
-  metadata.gz: 9f29aaab73b1c7e52adcbb1ab224a8c13c2223760e52d95ee458166a8cb7d252db9d294dfeab2487c481b40be4bd4fd449beef475ad70293385501f9432dc8c5
-  data.tar.gz: e1c5c223f41f677c65d3e932719833c2771f394c2132c406d2f10621b1f45edbf729a13e46690f16c51ebde996d364855291fe1755e901f25f2b94e181f8835a
+  metadata.gz: c7f8b1730ada8f2613b63ee87bf570acae14fe057bfd2650ba1ea4daa479f641a99f6d77e74b2f9484cd62aedc2899a2fd5fed8b138b09b52a2191855439d884
+  data.tar.gz: ee19f3e7a11eafda737333a7c41b6a83444e927c37033d8230adb26987c8bbd60c17346da5e4abfa61521500991b9856dc5fba91f5f37368fc72941a8fce60d6

data/.gitignore

@@ -1,5 +1,6 @@
 *.gem
 *.rbc
+*.log
 /.config
 /coverage/
 /InstalledFiles
@@ -42,7 +43,7 @@ build-iPhoneSimulator/
 
 # for a library or gem, you might want to ignore these files since the code is
 # intended to run in multiple environments; otherwise, check them in:
-# Gemfile.lock
+Gemfile.lock
 # .ruby-version
 # .ruby-gemset
 

data/.releaser_config

@@ -0,0 +1,3 @@
+version_file: VERSION
+always_from_master: true
+gem_style: github

data/.rubocop.yml

@@ -0,0 +1,84 @@
+AllCops:
+  TargetRubyVersion: 2.3
+
+  Exclude:
+    - 'vendor/**/*'
+    - 'tmp/**/*'
+    - 'log/**/*'
+    - '*.gemspec'
+
+  DisplayCopNames: true
+
+Style/FrozenStringLiteralComment:
+  Enabled: false
+
+Style/DoubleNegation:
+  Enabled: false
+
+Style/SignalException:
+  EnforcedStyle: only_fail
+
+Lint/RescueWithoutErrorClass:
+  Enabled: False
+
+Style/ConditionalAssignment:
+  Enabled: false
+
+Layout/IndentArray:
+  EnforcedStyle: consistent
+
+Metrics/MethodLength:
+  Enabled: false
+
+Metrics/ClassLength:
+  Enabled: false
+
+Metrics/ModuleLength:
+  Enabled: false
+
+Metrics/ParameterLists:
+  Max: 5
+  CountKeywordArgs: false
+
+Metrics/AbcSize:
+  Enabled: False
+
+Metrics/CyclomaticComplexity:
+  Enabled: False
+
+Metrics/PerceivedComplexity:
+  Enabled: False
+
+Metrics/LineLength:
+  Max: 160
+
+Metrics/BlockNesting:
+  Enabled: false
+
+Metrics/BlockLength:
+  Enabled: false
+
+Style/IfUnlessModifier:
+  Enabled: false
+
+Style/Documentation:
+  Enabled: false
+
+Style/RedundantReturn:
+  Enabled: false
+
+Style/AsciiComments:
+  Enabled: false
+
+Style/GuardClause:
+  Enabled: false
+
+Style/ClassAndModuleChildren:
+  Enabled: false
+  EnforcedStyle: compact
+  SupportedStyles:
+    - nested
+    - compact
+
+Style/NumericPredicate:
+  Enabled: false

data/.travis.yml

@@ -0,0 +1,11 @@
+language: ruby
+rvm:
+ - 2.3.0
+services:
+ - mysql
+before_install:
+  - mysql -e 'CREATE DATABASE workhorse;'
+script:
+ - bundle install
+ - bundle exec rake test
+ - bundle exec rubocop

data/README.md

@@ -1,20 +1,172 @@
 [![Build Status](https://travis-ci.org/sitrox/workhorse.svg?branch=master)](https://travis-ci.org/sitrox/workhorse)
 [![Gem Version](https://badge.fury.io/rb/workhorse.svg)](https://badge.fury.io/rb/workhorse)
 
-**This gem is considered work in progress. Do not use this in production yet.**
-
 # Workhorse
 
+**This Gem is still in an early stage of development. Please not use this in production yet.**
+
 Multi-threaded job backend with database queuing for ruby.
 
+## Introduction
+
+What it is:
+
+* Jobs are instances of classes that support the `perform` method.
+* Jobs are persisted in the database using ActiveRecord.
+* You can start one or more worker processes.
+* Each worker is configurable as to which queue(s) it processes.
+* Each worker polls the database and spawns a number of threads to execute jobs
+  of different queues simultaneously.
+
+What it isn't:
+
+* It cannot spawn new processes. Jobs are run in separate threads but not in
+  separate processes (unless you start multiple worker processes manually).
+* It does not support retries.
+
 ## Installation
 
-To install this gem using bundler, add it to your `Gemfile`:
+### Requirements
+
+* A database and table handler that properly supports row-level locking (such as
+  MySQL with InnoDB, PostgreSQL or Oracle).
+* An operating system and file system that supports file locking.
+
+### Installing under Rails
+
+1. Add `workhorse` to your `Gemfile`:
+
+   ```ruby
+   gem 'workhorse'
+   ```
+
+2. Run the install generator:
+
+   ```bash
+   bin/rails generate workhorse:install
+   ```
+
+   This generates:
+
+   * A database migration for creating the `jobs` table
+   * An initializer `config/initializers/workhorse.rb` for global configuration
+   * A daemon worker script under `bin/workhorse.rb`
+
+   Please customize the configuration files to your liking.
+
+## Queuing jobs
+
+### Basic jobs
+
+Workhorse can handle any jobs that support the `perform` method and are
+serializable. To queue a basic job, use the static method `Workhorse.enqueue`:
+
+```ruby
+class MyJob
+  def initialize(name)
+    @name = name
+  end
+
+  def perform
+    puts "Hello #{@name}"
+  end
+end
+
+Workhorse.enqueue MyJob.new('John'), queue: :test
+```
+
+In the above example, we also specify a queue named `:test`. This means that
+this job will never run simoultaneously with other jobs in the same queue. If no
+queue is given, the job can always be executed simoultaneously with any other
+job.
+
+### RailsOps operations
+
+Workhorse allows you to easily queue
+[RailsOps](https://github.com/sitrox/rails_ops) operations using the static
+method `Workhorse.enqueue_op`:
 
 ```ruby
-gem 'workhorse'
+Workhorse.enqueue Operations::Jobs::CleanUpDatabase, quiet: true
 ```
 
+Params passed using the second argument will be used for operation instantiation
+at job execution.
+
+You can also specify a queue:
+
+```ruby
+Workhorse.enqueue Operations::Jobs::CleanUpDatabase, { quiet: true }, queue: :maintenance
+```
+
+## Configuring and starting workers
+
+Workers poll the database for new jobs and execute them in one or more threads.
+Typically, one worker is started per process. While you can start workers
+manually, either in your main application process(es) or in a separate one,
+workhorse also provides you with a convenient way of starting one or multiple
+worker processes as a daemon.
+
+### Start workers manually
+
+Workers are created by instantiatating, configuring and starting a new
+`Workhorse::Worker` instance.
+
+```ruby
+Workhorse::Worker.start_and_wait(
+  pool_size: 5,                           # Processes 5 jobs concurrently
+  quiet:     false,                       # Logs to STDOUT
+  logger:    Rails.logger                 # Logs to Rails log. You can also
+                                          # provide any custom logger.
+)
+```
+
+See [code
+documentation](http://www.rubydoc.info/github/sitrox/workhorse/Workhorse%2FWorker:initialize)
+for more information on the arguments.
+
+### Start workers using a daemon script
+
+Using `Workhorse::Daemon` (`Workhorse::Daemon::ShellHandler`), you can spawn one
+or multiple worker processes automatically. This is useful for cases where you
+want the workers to exist in separate processes as opposed to in your main
+application process(es).
+
+For this case, the workhorse install routine automatically creates a file called
+`bin/workhorse.rb` which can be used to start one or more worker processes.
+
+To start the daemon:
+
+```bash
+bin/workhorse.rb start[|stop|status|watch|restart|usage]
+```
+
+#### Background and customization
+
+The daemon-part allows you to run arbitrary code as a daemon:
+
+```ruby
+Workhorse::Daemon::ShellHandler.run count: 5 do
+  # This runs as a daemon and will be started 5 times
+end
+```
+
+Within this shell handler, you can now instantiate, configure and start a worker
+as described under *Start workers manually*:
+
+```ruby
+Workhorse::Daemon::ShellHandler.run count: 5 do
+  # This will be run 5 times, each time in a separate process. Per process, it
+  # will be able to process 3 jobs concurrently.
+  Workhorse::Worker.start_and_wait(pool_size: 3, logger: Rails.logger)
+end
+```
+
+## Roadmap
+
+* [ ] ActiveJob integration for Rails
+* [ ] Job timeouts
+
 ## Copyright
 
 Copyright (c) 2017 Sitrox. See `LICENSE` for further details.

data/Rakefile

@@ -0,0 +1,35 @@
+task :gemspec do
+  gemspec = Gem::Specification.new do |spec|
+    spec.name          = 'workhorse'
+    spec.version       = IO.read('VERSION').chomp
+    spec.authors       = ['Sitrox']
+    spec.summary       = %(
+      Multi-threaded job backend with database queuing for ruby.
+    )
+    spec.files         = `git ls-files`.split($INPUT_RECORD_SEPARATOR)
+    spec.executables   = []
+    spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+    spec.require_paths = ['lib']
+
+    spec.add_development_dependency 'bundler', '~> 1.3'
+    spec.add_development_dependency 'rake'
+    spec.add_development_dependency 'rubocop', '0.51.0'
+    spec.add_development_dependency 'minitest'
+    spec.add_development_dependency 'mysql2', '~> 0.3.13'
+    spec.add_development_dependency 'benchmark-ips'
+    spec.add_dependency 'activesupport'
+    spec.add_dependency 'activerecord'
+    spec.add_dependency 'schemacop', '~> 2.0'
+    spec.add_dependency 'concurrent-ruby'
+  end
+
+  File.open('workhorse.gemspec', 'w') { |f| f.write(gemspec.to_ruby.strip) }
+end
+
+require 'rake/testtask'
+
+Rake::TestTask.new do |t|
+  t.pattern = 'test/workhorse/**/*_test.rb'
+  t.verbose = false
+  t.libs << 'test/lib'
+end

data/VERSION

@@ -1 +1 @@
-0.0.1
+0.0.2

data/bin/rubocop

@@ -0,0 +1 @@
+bundle exec rubocop "$@"

data/lib/generators/workhorse/install_generator.rb

@@ -0,0 +1,24 @@
+module Workhorse
+  class InstallGenerator < Rails::Generators::Base
+    include Rails::Generators::Migration
+
+    source_root File.expand_path('../templates', __FILE__)
+
+    def self.next_migration_number(_dir)
+      Time.now.utc.strftime('%Y%m%d%H%M%S')
+    end
+
+    def install_migration
+      migration_template 'create_table_jobs.rb', 'db/migrate/create_table_jobs.rb'
+    end
+
+    def install_daemon_script
+      template 'bin/workhorse.rb'
+      chmod 'bin/workhorse.rb', 0o755
+    end
+
+    def install_initializer
+      template 'config/initializers/workhorse.rb'
+    end
+  end
+end

data/lib/generators/workhorse/templates/bin/workhorse.rb

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+
+require './config/environment'
+
+Workhorse::Daemon::ShellHandler.run do
+  Workhorse::Worker.start_and_wait(pool_size: 5, logger: Rails.logger)
+end

data/lib/generators/workhorse/templates/config/initializers/workhorse.rb

@@ -0,0 +1,11 @@
+Workhorse.setup do |config|
+  # Set this to false in order to prevent jobs from being automatically
+  # wrapped into a transaction. The built-in workhorse logic will still run
+  # in transactions.
+  config.perform_jobs_in_tx = true
+
+  # Enable this to specify an alternative callback for handling transactions.
+  # config.tx_callback = proc do |&block|
+  #   ActiveRecord::Base.transaction&(&block)
+  # end
+end

data/lib/generators/workhorse/templates/create_table_jobs.rb

@@ -0,0 +1,23 @@
+class CreateTableJobs < ActiveRecord::Migration
+  def change
+    create_table :jobs, force: true do |t|
+      t.string :state, null: false, default: 'waiting'
+      t.string :queue, null: true
+      t.text :handler, null: false, limit: 4_294_967_295
+
+      t.string :locked_by
+      t.datetime :locked_at
+
+      t.datetime :started_at
+
+      t.datetime :succeeded_at
+      t.datetime :failed_at
+      t.text :last_error, limit: 4_294_967_295
+
+      t.timestamps null: false
+    end
+
+    add_index :jobs, :queue
+    add_index :jobs, :state
+  end
+end

data/lib/workhorse.rb

@@ -0,0 +1,42 @@
+require 'socket'
+require 'active_support/all'
+require 'active_record'
+
+module Workhorse
+  @set_up = false
+
+  # Returns the performer currently performing the active job. This can only be
+  # called from within a job and the same thread.
+  def self.performer
+    Thread.current[:workhorse_current_performer]\
+      || fail('No performer is associated with the current thread. This method must always be called inside of a job.')
+  end
+
+  cattr_accessor :tx_callback
+  self.tx_callback = proc do |&block|
+    ActiveRecord::Base.transaction(&block)
+  end
+
+  cattr_accessor :perform_jobs_in_tx
+  self.perform_jobs_in_tx = true
+
+  def self.setup
+    fail 'Workhorse is already set up.' if @set_up
+    yield self
+    @set_up = true
+  end
+end
+
+require 'workhorse/db_job'
+require 'workhorse/enqueuer'
+require 'workhorse/performer'
+require 'workhorse/poller'
+require 'workhorse/pool'
+require 'workhorse/worker'
+require 'workhorse/jobs/run_rails_op'
+
+# Daemon functionality is not available on java platforms
+if RUBY_PLATFORM != 'java'
+  require 'workhorse/daemon'
+  require 'workhorse/daemon/shell_handler'
+end