job_dispatch 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: b05fed659c186b001b926e08650b3a281160668f
+  data.tar.gz: 6ed34896dfbe1a5dcc1382a36b39e7c3e73f2f2f
+SHA512:
+  metadata.gz: 4a211880bafb99581f6a4d8bd884ddcc37d68170f5ba28fde9943d0cdf2eb9735a6cd3c78f683ef4c923f02bc4c19d5479461904d8b25d060088c088048e2747
+  data.tar.gz: 30656712d978383424f23945daa8ae77e3f61dd6c4314d324b8063203e6ecf604347588074b9db0f9ea39fed4b0289869d55d20dc5e3dfaed901caea7d5ad674

data/.gitignore

@@ -0,0 +1,19 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+.idea
+

data/.rspec

@@ -0,0 +1 @@
+--color

data/.travis.yml

@@ -0,0 +1,13 @@
+language: ruby
+rvm:
+  - 2.1.1
+  - 2.0.0
+  - 1.9.3
+  - rbx
+gemfile:
+  - Gemfile
+matrix:
+  allow_failures:
+    - rvm: rbx
+env: RUBYOPT="-Ku"
+

data/Gemfile

@@ -0,0 +1,20 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in job_dispatch.gemspec
+gemspec
+
+# required for testing rubinius on travis:
+platforms :rbx do
+  gem 'rubysl', '~> 2.0'
+  gem 'rubinius', '~> 2.0'
+  gem "rubinius-coverage", github: "rubinius/rubinius-coverage"
+end
+
+group :test do
+  gem 'guard'
+  group :mac do
+    gem 'growl'
+    gem 'guard-rspec'
+    gem 'terminal-notifier-guard'
+  end
+end

data/Guardfile

@@ -0,0 +1,13 @@
+# A sample Guardfile
+# More info at https://github.com/guard/guard#readme
+
+# To use zeus, cmd: 'zeus rspec'
+guard 'rspec', cmd: 'rspec' do
+
+  notification :growl, :sticky => false #, :path => '/usr/local/bin/growlnotify'
+  notification :terminal_notifier
+
+  watch(%r{^spec/.+_spec\.rb$})
+  watch(%r{^lib/(.+)\.rb$})     { |m| "spec/lib/#{m[1]}_spec.rb" }
+  watch('spec/spec_helper.rb')  { "spec" }
+end

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Matt Connolly
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,85 @@
+# JobDispatch
+
+[![Build Status](https://travis-ci.org/mobiledataanywhere/job_dispatch.png)](https://travis-ci.org/mobiledataanywhere/job_dispatch)
+
+Job Dispatch is a gem for dispatching jobs to workers in an asynchronous manner. 
+Job Dispatch does not require any specific database, deliberately separating the storage of jobs
+from the dispatching of jobs to workers.
+Jobs are dispatched using ZeroMQ messages and workers can be implemented in any language.
+
+Only the dispatcher needs to access the database. This decouples the workers from the database.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'job_dispatch'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install job_dispatch
+
+
+## Requirements
+
+### Dispatcher
+
+This gem uses [rbczmq](https://github.com/methodmissing/rbczmq) for sending messages, so its use is limited to
+platforms supporting this gem, namely a posix OS and MRI Ruby or Rubinius.
+
+JRuby is not supported. Nor is Windows.
+
+### Workers
+
+The built in Ruby worker has the same ruby requirements as the broker. However, additional workers can be implemented in any
+language on any platform provided they have access to ZeroMQ for messaging and a JSON library for reading/writing message payloads.
+
+
+## Usage
+
+TODO: Write usage instructions here.
+
+JobDispatch will work with any Job model class, provided it fulfils the following:
+
+Attributes:
+
+ * `queue`
+ * `status`
+ * `target`
+ * `method`
+ * `enqueued_at`
+ * `scheduled_at`
+ * `expire_execution_at`
+ * `completed_at`
+ * `timeout`
+ * `retry_count`
+ * `retry_delay`
+ * `result`
+
+Class Methods:
+
+    dequeue_job_for_queue(queue, time=nil)
+
+This method is for retrieving a single PENDING job from the database and atomically
+marking it as being IN PROGRESS so that it can not be received again. It is the dispatcher's
+Responsibility to update its status to completed/failed and schedule a retry if the
+retry_count > 0.
+
+See the example model classes in the examples/ directory.
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request
+
+## License
+
+Licensed under MIT license, see LICENSE.txt.
+

data/Rakefile

@@ -0,0 +1,10 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+task :default => [:spec]
+
+desc "Run the specs."
+RSpec::Core::RakeTask.new do |t|
+  t.pattern = "spec/**/*_spec.rb"
+end
+

data/bin/job-dispatcher

@@ -0,0 +1,34 @@
+#!/usr/bin/env ruby
+#
+# This command is expected to be run in the root of a Rails application which has a Job model class
+# and has an initializer to configure JobDispatch.
+
+ROOT_DIR = Dir.pwd
+APP_PATH = File.expand_path('config/application.rb', ROOT_DIR)
+
+unless File.exist?('config/job_dispatch.yml')
+  $stderr.puts "Configuration file at 'config/job_dispatch.yml' not found."
+  exit 1
+end
+
+if ENV['BUNDLE_GEMFILE'] || File.exists('Gemfile')
+  require 'bundler/setup'
+end
+
+require 'job_dispatch'
+
+# boot the rails app so we can access the Rails stored job queue.
+require File.expand_path('config/boot.rb', ROOT_DIR)
+
+ENV["RAILS_ENV"] ||= "development"
+JobDispatch.load_config_from_yml('config/job_dispatch.yml', ENV["RAILS_ENV"])
+
+# TODO: Find a way to start the dispatcher with only ActiveRecord/Mongoid skipping the rest of rails
+# as it uses ~100MB RAM.
+
+require File.expand_path('config/environment.rb', ROOT_DIR)
+
+JobDispatch.logger = Rails.logger
+
+broker = JobDispatch::Broker.new(JobDispatch.config.broker[:bind], JobDispatch.config.signaller[:bind])
+broker.run

data/bin/job-status

@@ -0,0 +1,69 @@
+#!/usr/bin/env ruby
+#
+# This command is expected to be run in the root of a Rails application which has a Job model class
+# and has an initializer to configure JobDispatch.
+
+ROOT_DIR = Dir.pwd
+APP_PATH = File.expand_path('config/application.rb', ROOT_DIR)
+
+unless File.exist?('config/job_dispatch.yml')
+  $stderr.puts "Configuration file at 'config/job_dispatch.yml' not found."
+  exit 1
+end
+
+if ENV['BUNDLE_GEMFILE'] || File.exists('Gemfile')
+  require 'bundler/setup'
+end
+
+require 'job_dispatch'
+
+# boot the rails app so we can access the Rails stored job queue.
+require File.expand_path('config/boot.rb', ROOT_DIR)
+
+ENV["RAILS_ENV"] ||= "development"
+JobDispatch.load_config_from_yml('config/job_dispatch.yml', ENV["RAILS_ENV"])
+
+$:.unshift(File.join(ROOT_DIR, "lib"))
+
+
+require 'job_dispatch/status'
+require 'json'
+
+repeat = 0
+
+if ARGV.count > 0
+  repeat = ARGV.first.to_i
+end
+
+endpoint = JobDispatch.config.broker[:connect]
+if endpoint.nil? || endpoint.empty?
+  $stderr.puts "No Job Dispatch broker connect address has been specified."
+  exit 1
+end
+
+status = JobDispatch::Status.new(endpoint)
+status.connect
+loop do
+  begin
+    Timeout.timeout(5) do
+      status.fetch
+      status.print
+    end
+
+    if repeat > 0
+      sleep(repeat)
+    else
+      break
+    end
+
+  rescue TimeoutError
+    # message may have been sent and we might be waiting for a reply that never comes, so
+    # close socket and reconnect.
+    puts "Job Dispatcher status: No response from broker, reconnecting..."
+    status.disconnect
+    status.connect
+  rescue Interrupt
+    exit 0
+  end
+end
+status.disconnect

data/bin/job-worker

@@ -0,0 +1,40 @@
+#!/usr/bin/env ruby
+#
+# This command is expected to be run in the root of a Rails application which has a Job model class
+# and has an initializer to configure JobDispatch.
+
+ROOT_DIR = Dir.pwd
+APP_PATH = File.expand_path('config/application.rb', ROOT_DIR)
+
+unless File.exist?('config/job_dispatch.yml')
+  $stderr.puts "Configuration file at 'config/job_dispatch.yml' not found."
+  exit 1
+end
+
+if ENV['BUNDLE_GEMFILE'] || File.exists('Gemfile')
+  require 'bundler/setup'
+end
+
+require 'job_dispatch'
+
+# boot the rails app so we can access the Rails stored job queue.
+require File.expand_path('config/boot.rb', ROOT_DIR)
+
+ENV["RAILS_ENV"] ||= "development"
+JobDispatch.load_config_from_yml('config/job_dispatch.yml', ENV["RAILS_ENV"])
+
+# TODO: Find a way to start the dispatcher with only ActiveRecord/Mongoid skipping the rest of rails
+# as it uses ~100MB RAM.
+
+require File.expand_path('config/environment.rb', ROOT_DIR)
+
+JobDispatch.logger = Rails.logger
+
+endpoint = JobDispatch.config.broker[:connect]
+if endpoint.nil? || endpoint.empty?
+  $stderr.puts "No Job Dispatch broker connect address has been specified."
+  exit 1
+end
+
+worker = JobDispatch::Worker.new(endpoint, JobDispatch.config.worker_options)
+worker.run

data/examples/mongoid-job.rb

@@ -0,0 +1,43 @@
+# JobDispatch compliant job.
+#
+class Job
+
+  include Mongoid::Document
+
+  # important: Need this to include the success! and fail! methods.
+  include JobDispatch::Job
+
+  # required fields:
+  field :queue, :type => String, :default => 'default'
+  field :status, :type => Integer, :default => PENDING
+
+  field :parameters, :type => Array # list of parameters for the action
+  field :target, :type => String
+  field :method, :type => String
+
+  field :enqueued_at, :type => Time #, :default => -> { Time.now.to_i }
+  field :scheduled_at, :type => Time, :default => 0
+  field :expire_execution_at, :type => Time
+  field :timeout, :type => Integer
+  field :retry_count, :type => Integer, :default => 0
+  field :retry_delay, :type => Integer, :default => 10
+  field :completed_at, :type => Time
+
+  # If the job completed successfully, this is the result. If it failed,
+  # this is the exception error message
+  field :result, :type => Object
+
+  # any indexes:
+  index({queue: 1, status: 1, scheduled_at: 1})
+
+  # dequeue a job from the database.
+  # This is an atomic operation that also marks the job as being in the pending state.
+  def self.dequeue_job_for_queue(queue, time=nil)
+    time ||= Time.now
+    self.
+        where(:queue => queue, :status => PENDING).
+        where(:scheduled_at.lte => time).
+        sort(:enqueued_at => 1).
+        find_and_modify({"$set" => {:status => IN_PROGRESS}}, new: true)
+  end
+end

data/job_dispatch.gemspec

@@ -0,0 +1,33 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'job_dispatch/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "job_dispatch"
+  spec.version       = JobDispatch::VERSION
+  spec.authors       = ["Matt Connolly"]
+  spec.email         = ["matt@mobiledataanywhere.com"]
+  spec.description   = %q{Job Dispatch to workers via ZeroMQ}
+  spec.summary       = %q{Job Dispatch is a gem for dispatching jobs to workers in an asynchronous manner. Job Dispatch does not require any specific database. Jobs are dispatched using ZeroMQ messages and workers can be implemented in any language.}
+  spec.homepage      = ""
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_runtime_dependency "rbczmq", "~> 1.7"
+  spec.add_runtime_dependency 'nullobject'
+  spec.add_runtime_dependency 'json'
+  spec.add_runtime_dependency 'text-table', '~> 1.2.3'
+  spec.add_runtime_dependency 'activesupport', '>= 3.0.0'
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rspec", "~> 2.14"
+  spec.add_development_dependency "simplecov", "~> 0.8"
+  spec.add_development_dependency "timecop", "~> 0.7"
+  spec.add_development_dependency "factory_girl", "~> 4.3.0"
+  spec.add_development_dependency "rake"
+end

data/lib/job_dispatch/broker/command.rb

@@ -0,0 +1,45 @@
+require 'json'
+
+module JobDispatch
+  # A very simple representation of a worker and a message received from or to be sent to that worker.
+  # this represents job messages from the Broker's ROUTER socket perspective. ie: messages are wrapped
+  # by the worker ID so that they can be routed to the client.
+  #
+  # The parameters object is expected to be a Hash with indifferent access
+  class Broker
+    class Command
+      attr_accessor :worker_id
+      attr_accessor :parameters
+
+      def initialize(worker_id=nil, parameters=nil)
+        @worker_id, @parameters = worker_id, parameters
+      end
+
+      def worker_ready?
+        @parameters[:command] == "ready" || @parameters[:ready]
+      end
+
+      def queue
+        (@parameters[:queue] || :default).to_sym
+      end
+
+      def command
+        @parameters[:command]
+      end
+
+      def status
+        @parameters[:status] && @parameters[:status].to_sym
+      end
+
+      def success?
+        status == :success
+      end
+
+      # the name of the worker
+      def worker_name
+        @parameters[:worker_name]
+      end
+    end
+  end
+
+end

data/lib/job_dispatch/broker/internal_job.rb

@@ -0,0 +1,32 @@
+module JobDispatch
+  class Broker
+
+    # a class to represent a worker processing a broker command that is not a job. It is still tracked by
+    # the broker as if it was a job, though. If the worker does not reply, they will be timed out. And
+    # status report will show the worker state as executing this command.
+    class InternalJob
+      attr :id, :command, :queue
+
+      def initialize(command, queue)
+        @id = SecureRandom.uuid
+        @command = command
+        @timeout_at = Time.now + JobDispatch::Job::DEFAULT_EXECUTION_TIMEOUT
+        @queue = queue
+      end
+
+      def timed_out?
+        Time.now > @timeout_at
+      end
+
+      def as_json
+        {
+            command: command,
+            id: id,
+            queue: queue,
+            target: "JobDispatch",
+            method: command
+        }
+      end
+    end
+  end
+end

data/lib/job_dispatch/broker/socket.rb

@@ -0,0 +1,85 @@
+module JobDispatch
+  class Broker
+    #
+    # This class represents a ZMQ socket where workers can phone in and ask for work to do.
+    # If there is no work to do, an idle command is returned
+    #
+    # The response will be a single JSON encoded message such as:
+    #
+    # { "command":"idle" }
+    #
+    # or
+    #
+    # { "target":"MyMailer", "method":"deliver", "parameters":[14, "Subject", "Some other parameter"]}
+    #
+    # In which the worker should execute the code `MyMailer.deliver(14, 'Subject', 'Some other parameter')`
+    #
+    #
+    # Other commands can be sent to the broker, such as:
+    #
+    # { "command":"status" } => returns status of the broker, number of idle workers per queue, etc.
+    #
+    class Socket
+      attr_reader :socket
+
+      def initialize(bind_address)
+        @bind_address = bind_address
+        @socket = nil
+        @queues = {}
+      end
+
+      def connect
+        unless @socket
+          @socket = JobDispatch.context.socket(ZMQ::ROUTER)
+          @socket.bind(@bind_address)
+        end
+        @socket
+      end
+
+      def disconnect
+        if @socket
+          @socket.close
+          @socket = nil
+        end
+      end
+
+      def poll_item
+        @poll_item ||= ZMQ::Pollitem(@socket, ZMQ::POLLIN)
+      end
+
+      def readable?
+        poller = ZMQ::Poller.new
+        poller.register(poll_item)
+        poller.poll(0) > 0
+      end
+
+      # Process a message received on the broker socket. The returned Command object contains the identity of the
+      # requester (worker) so that replies can be sent to them.
+      #
+      # @return [Command]
+      def read_command
+        message = socket.recv_message #=> ZMQ::Message
+        worker_id = Identity.new(message.unwrap.data.to_s.to_sym)
+        json = message.first.data
+        parameters = begin
+          JSON.parse(json).with_indifferent_access
+        rescue
+          JobDispatch.logger.error("Received invalid json data: #{json.inspect} from socket id '#{worker_id}'")
+          {error: "Invalid JSON"}
+        end
+        Command.new(worker_id, parameters)
+      end
+
+      # Send a command message to a worker.
+      # @param [Command command] the command to send to the worker.
+      def send_command(command)
+        message = ZMQ::Message.new
+        message.addstr(JSON.dump(command.parameters))
+        message.wrap(ZMQ::Frame(command.worker_id.to_s))
+        socket.send_message message
+      end
+
+    end
+  end
+end
+