delayed_job_worker_pool 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 79d4a1eeecbbf23ff19a1811faf6a8d35d9616c5
+  data.tar.gz: 99b6e576ac2ef89486b12614f171dd2db7d4588c
+SHA512:
+  metadata.gz: 303a786f4bbe3c5e96a23e6cd63582f321289b4b8e63489071b9502059d503cf12f2667f8891c11a2955947bf2a426831840bdd28377a7e57dec5afac4179205
+  data.tar.gz: 824d5d7e04193c7a4ed6fbddec598f378dc6f5f2ad57e39bc32700f3eb0a000e1881778aaf0fedf371607012812b10859adafd7b0606723154066ce1fc73a7c0

data/.gitignore

@@ -0,0 +1,11 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+.rspec
+*.sqlite3

data/.travis.yml

@@ -0,0 +1,4 @@
+language: ruby
+rvm:
+  - 2.2.2
+before_install: gem install bundler -v 1.10.6

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 Joel Turkel
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,86 @@
+# Delayed Job Worker Pool
+
+[![Gem Version](https://badge.fury.io/rb/delayed_job_worker_pool.png)][gem]
+[![Build Status](https://secure.travis-ci.org/salsify/delayed_job_worker_pool.png?branch=master)][travis]
+[![Code Climate](https://codeclimate.com/github/salsify/delayed_job_worker_pool.png)][codeclimate]
+
+[gem]: https://rubygems.org/gems/delayed_job_worker_pool
+[travis]: http://travis-ci.org/salsify/delayed_job_worker_pool
+[codeclimate]: https://codeclimate.com/github/salsify/delayed_job_worker_pool
+
+[Delayed Job's](https://github.com/collectiveidea/delayed_job) built-in worker pooling daemonizes all worker processes. This is great for certain environments but not so great for environments like Heroku that really want your processes to run in the foreground. Delayed Job Worker Pool runs a pool of Delayed Job workers **without** daemonizing them. [Salsify](http://salsify.com) is currently using this to run multiple Delayed Job workers on a single Heroku PX dyno.
+ 
+This gem only works with MRI on Linux/MacOS X.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'delayed_job_worker_pool'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install delayed_job_worker_pool
+
+## Usage
+
+From your Rails root directory run:
+
+```
+delayed_job_worker_pool <config file>
+```
+
+The config file is a Ruby DSL inspired by the [Puma](https://github.com/puma/puma) configuration DSL. Here's an example:
+
+```
+workers Integer(ENV['NUM_WORKERS'] || 1)
+queues (ENV['QUEUES'] || ENV['QUEUE'] || '').split(',')
+sleep_delay ENV['WORKER_SLEEP_DELAY']
+
+preload_app
+
+before_worker_boot do
+  puts 'Master about to start forking children'
+end
+
+before on_worker_boot do
+  puts "Worker #{Process.pid} started"
+  
+  # Reconnect to the database
+  ActiveRecord::Base.establish_connection
+end
+
+after_worker_boot do
+  puts 'Master booted children'
+  
+  # Don't hang on to database connections from the master after we've completed initialization
+  ActiveRecord::Base.connection_pool.disconnect!
+end
+```
+
+Here's more information on each setting:
+
+* `workers` - The number of Delayed Job worker processes to fork. The master process will relaunch workers that fail.
+* Delayed Job worker settings (`queues`, `min_priority`, `max_priority`, `sleep_delay`, `read_ahead`) - These are passed through to the Delayed Job worker.
+* `preload_app` - This forces the master process to load Rails before forking worker processes causing the memory consumed by the code to be shared between workers. **If you use this setting make sure you re-establish any necessary connections in the on_worker_boot callback.**
+* `before_worker_boot` - A callback that runs in the master process before forking any workers.
+* `on_worker_boot` - A callback that runs in the worker process after it has been forked.
+* `after_worker_boot` - A callback that runs in the master process after the initial set of workers have been forked.
+
+All settings are optional and nil values are ignored. 
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/salsify/delayed_job_worker_pool.
+
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
+

data/Rakefile

@@ -0,0 +1,6 @@
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+
+task default: :spec

data/bin/delayed_job_worker_pool

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+
+unless ARGV.size == 1
+  puts "usage: #{File.basename(__FILE__)} <config>"
+  exit(1)
+end
+
+require 'delayed_job_worker_pool'
+
+options = DelayedJobWorkerPool::DSL.load(ARGV[0])
+DelayedJobWorkerPool::WorkerPool.new(options).run

data/delayed_job_worker_pool.gemspec

@@ -0,0 +1,28 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'delayed_job_worker_pool/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'delayed_job_worker_pool'
+  spec.version       = DelayedJobWorkerPool::VERSION
+  spec.authors       = ['Joel Turkel']
+  spec.email         = ['jturkel@salsify.com']
+
+  spec.summary       = 'Worker process pooling for Delayed Job'
+  spec.homepage      = 'https://github.com/salsify/delayed_job_worker_pool'
+  spec.license       = 'MIT'
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.executables   = ['delayed_job_worker_pool']
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'delayed_job', ['>= 3.0', '< 4.1']
+
+  spec.add_development_dependency 'delayed_job_active_record'
+  spec.add_development_dependency 'bundler', '~> 1.10'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rspec', '>= 3.3'
+  spec.add_development_dependency 'sqlite3', '>= 1.3'
+  spec.add_development_dependency 'rails', '>= 4.2'
+end

data/lib/delayed_job_worker_pool.rb

@@ -0,0 +1,5 @@
+require 'delayed_job_worker_pool/application'
+require 'delayed_job_worker_pool/dsl'
+require 'delayed_job_worker_pool/worker'
+require 'delayed_job_worker_pool/worker_pool'
+require 'delayed_job_worker_pool/version'

data/lib/delayed_job_worker_pool/application.rb

@@ -0,0 +1,22 @@
+module DelayedJobWorkerPool
+  module Application
+    extend self
+
+    def load
+      require(base_application_filename)
+    rescue LoadError
+      raise "Could not find Rails initialization file #{full_application_filename}. " \
+            "Make sure delayed_job_worker_pool is run from the Rails root directory."
+    end
+
+    private
+
+    def base_application_filename
+      "#{Dir.pwd}/config/environment"
+    end
+
+    def full_application_filename
+      "#{base_application_filename}.rb"
+    end
+  end
+end

data/lib/delayed_job_worker_pool/dsl.rb

@@ -0,0 +1,35 @@
+module DelayedJobWorkerPool
+  class DSL
+    SIMPLE_SETTINGS = [:workers, :queues, :min_priority, :max_priority, :sleep_delay, :read_ahead].freeze
+    CALLBACK_SETTINGS = [:before_worker_boot, :on_worker_boot, :after_worker_boot].freeze
+
+    def self.load(path)
+      options = {}
+
+      dsl = new(options)
+      dsl.instance_eval(File.read(path), path, 1)
+
+      options
+    end
+
+    def initialize(options)
+      @options = options
+    end
+
+    SIMPLE_SETTINGS.each do |option_name|
+      define_method(option_name) do |option_value|
+        @options[option_name] = option_value unless option_value.nil?
+      end
+    end
+
+    def preload_app(preload_app = true)
+      @options[:preload_app] = preload_app
+    end
+
+    CALLBACK_SETTINGS.each do |option_name|
+      define_method(option_name) do |&block|
+        @options[option_name] = block
+      end
+    end
+  end
+end

data/lib/delayed_job_worker_pool/version.rb

@@ -0,0 +1,3 @@
+module DelayedJobWorkerPool
+  VERSION = '0.1.0'.freeze
+end

data/lib/delayed_job_worker_pool/worker.rb

@@ -0,0 +1,10 @@
+module DelayedJobWorkerPool
+  module Worker
+    extend self
+    
+    def run(options = {})
+      dj_worker = Delayed::Worker.new(options)
+      dj_worker.start
+    end
+  end
+end

data/lib/delayed_job_worker_pool/worker_pool.rb

@@ -0,0 +1,133 @@
+module DelayedJobWorkerPool
+  class WorkerPool
+    def initialize(options = {})
+      @options = options
+      @worker_pids = []
+      self.shutting_down = false
+    end
+
+    def run
+      log("Starting master #{Process.pid}")
+
+      install_signal_handlers
+
+      load_app if preload_app?
+
+      invoke_callback(:before_worker_boot)
+
+      log_uninheritable_threads
+
+      create_master_alive_pipe
+      num_workers.times { fork_worker }
+
+      invoke_callback(:after_worker_boot)
+
+      monitor_workers
+
+      exit
+    ensure
+      master_alive_write_pipe.close if master_alive_write_pipe
+      master_alive_read_pipe.close if master_alive_read_pipe
+    end
+
+    private
+
+    attr_reader :options, :worker_pids, :master_alive_read_pipe, :master_alive_write_pipe
+    attr_accessor :shutting_down
+
+    def install_signal_handlers
+      trap('TERM') do
+        shutdown('TERM')
+      end
+
+      trap('INT') do
+        shutdown('INT')
+      end
+    end
+
+    def log_uninheritable_threads
+      Thread.list.each do |t|
+        next if t == Thread.current
+        if t.respond_to?(:backtrace)
+          log("WARNING: Thread will not be inherited by workers: #{t.inspect} - #{t.backtrace ? t.backtrace.first : ''}")
+        else
+          log("WARNING: Thread will not be inherited by workers: #{t.inspect}")
+        end
+      end
+    end
+
+    def create_master_alive_pipe
+      @master_alive_read_pipe, @master_alive_write_pipe = IO.pipe
+    end
+
+    def load_app
+      DelayedJobWorkerPool::Application.load
+    end
+
+    def shutdown(signal)
+      log("Shutting down master #{Process.pid} with signal #{signal}")
+      self.shutting_down = true
+      worker_pids.each do |child_pid|
+        log("Telling worker #{child_pid} to shutdown with signal #{signal}")
+        Process.kill(signal, child_pid)
+      end
+    end
+
+    def monitor_workers
+      until worker_pids.empty?
+        worker_pid, status = Process.wait2
+
+        next unless worker_pids.include?(worker_pid)
+
+        log("Worker #{worker_pid} exited with status #{status.to_i}")
+        worker_pids.delete(worker_pid)
+        fork_worker unless shutting_down
+      end
+    end
+
+    def invoke_callback(callback_name)
+      options[callback_name].call if options[callback_name]
+    end
+
+    def fork_worker
+      worker_pid = Kernel.fork { run_worker }
+      worker_pids << worker_pid
+      log("Started worker #{worker_pid}")
+    end
+
+    def run_worker
+      master_alive_write_pipe.close
+
+      Thread.new do
+        IO.select([master_alive_read_pipe])
+        log('Detected dead master. Shutting down worker.')
+        exit(1)
+      end
+
+      load_app unless preload_app?
+
+      invoke_callback(:on_worker_boot)
+
+      DelayedJobWorkerPool::Worker.run(worker_options)
+    rescue => e
+      log("Worker failed with error: #{e.message}\n#{e.backtrace.join("\n")}")
+      exit(1)
+    end
+
+    def num_workers
+      options.fetch(:workers, 1)
+    end
+
+    def preload_app?
+      options.fetch(:preload_app, false)
+    end
+
+    def worker_options
+      options.except(:workers, :preload_app, :before_worker_boot, :on_worker_boot, :after_worker_boot)
+    end
+
+    def log(message)
+      puts(message)
+    end
+  end
+end

metadata

@@ -0,0 +1,163 @@
+--- !ruby/object:Gem::Specification
+name: delayed_job_worker_pool
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Joel Turkel
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-08-17 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: delayed_job
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '4.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '4.1'
+- !ruby/object:Gem::Dependency
+  name: delayed_job_active_record
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3.3'
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '4.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '4.2'
+description: 
+email:
+- jturkel@salsify.com
+executables:
+- delayed_job_worker_pool
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/delayed_job_worker_pool
+- delayed_job_worker_pool.gemspec
+- lib/delayed_job_worker_pool.rb
+- lib/delayed_job_worker_pool/application.rb
+- lib/delayed_job_worker_pool/dsl.rb
+- lib/delayed_job_worker_pool/version.rb
+- lib/delayed_job_worker_pool/worker.rb
+- lib/delayed_job_worker_pool/worker_pool.rb
+homepage: https://github.com/salsify/delayed_job_worker_pool
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.7
+signing_key: 
+specification_version: 4
+summary: Worker process pooling for Delayed Job
+test_files: []