eq 0.0.1

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--format progress

data/.travis.yml

@@ -0,0 +1,3 @@
+rvm:
+  - 1.9.3
+  - rbx-19mode

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in eq.gemspec
+gemspec

data/Guardfile

@@ -0,0 +1,24 @@
+# A sample Guardfile
+# More info at https://github.com/guard/guard#readme
+
+guard 'rspec', :version => 2 do
+  watch(%r{^spec/.+_spec\.rb$})
+  watch(%r{^lib/(.+)\.rb$})     { |m| "spec/lib/#{m[1]}_spec.rb" }
+  watch('spec/spec_helper.rb')  { "spec" }
+
+  # Rails example
+  watch(%r{^app/(.+)\.rb$})                           { |m| "spec/#{m[1]}_spec.rb" }
+  watch(%r{^app/(.*)(\.erb|\.haml)$})                 { |m| "spec/#{m[1]}#{m[2]}_spec.rb" }
+  watch(%r{^app/controllers/(.+)_(controller)\.rb$})  { |m| ["spec/routing/#{m[1]}_routing_spec.rb", "spec/#{m[2]}s/#{m[1]}_#{m[2]}_spec.rb", "spec/acceptance/#{m[1]}_spec.rb"] }
+  watch(%r{^spec/support/(.+)\.rb$})                  { "spec" }
+  watch('config/routes.rb')                           { "spec/routing" }
+  watch('app/controllers/application_controller.rb')  { "spec/controllers" }
+  
+  # Capybara request specs
+  watch(%r{^app/views/(.+)/.*\.(erb|haml)$})          { |m| "spec/requests/#{m[1]}_spec.rb" }
+  
+  # Turnip features and steps
+  watch(%r{^spec/acceptance/(.+)\.feature$})
+  watch(%r{^spec/acceptance/steps/(.+)_steps\.rb$})   { |m| Dir[File.join("**/#{m[1]}.feature")][0] || 'spec/acceptance' }
+end
+

data/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2012 Jens Bissinger
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,79 @@
+# EXPERIMENTAL FOO, THE DEVILS RIDE...
+
+# EQ - Embedded Queueing
+
+EQ is a little framework to queue tasks within a process.
+
+[![Travis-CI Build Status](https://secure.travis-ci.org/dpree/eq.png)](https://secure.travis-ci.org/dpree/eq)
+[![Code Climate](https://codeclimate.com/badge.png)](https://codeclimate.com/github/dpree/eq)
+
+## Installation
+
+Install it yourself using Rubygems.
+
+    $ gem install eq
+
+Or use something like [Bundler](http://gembundler.com/).
+
+## Example
+
+If you want to execute a simple example you can just run [examples/simple_usage.rb](./examples/simple_usage.rb) from your commandline.
+
+**1. Define a Job class with a perform method.**
+
+	class MyJob
+	  def self.perform *some_args
+	    # do some long running stuff here
+	  end
+	end
+
+**2. Start the EQ system.**
+
+	EQ.boot
+	
+**3. Let EQ do some work you.**
+
+	EQ.queue.push MyJob, 'foo'
+	EQ.queue.push MyJob, 'bar'
+	…	
+
+## Configuration
+
+Right now there is only one queueing backend available that is based on the Sequel gem. Therefore, basically any SQL database supported by Sequel might be used.
+
+The default SQL database that is used is a . You can change it using any argument that Sequel.connect method would accept.
+	
+	# SQLite3 in-memory (default) using String syntax
+	EQ.config.sequel = 'sqlite:/'
+	
+	# SQLite3 file using Hash syntax
+	EQ.config.sequel = {adapter: 'sqlite', database: 'my_db.sqlite3'}
+	
+	# Postgres
+	EQ.config.sequel = 'postgres://user:password@host:port/my_db'
+	
+
+### Logging
+
+EQ uses the logging mechanism of the underlying Celluloid (`Celluloid.logger`) framework. Basically you can just bind it to your application logger or re-configure it (see the Documentation of the `Logger` class from Ruby Standard Library).
+
+**Changing the Logger:**
+
+	# Use the logger of your Rails application.
+	Celluloid.logger = Rails.logger
+	
+	# No more logging at all.
+	Celluloid.logger = Logger.new('/dev/null')
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Added some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request
+
+# LICENSE
+
+Copyright (c) 2012 Jens Bissinger. See [LICENSE](LICENSE).
+

data/Rakefile

@@ -0,0 +1,6 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec)
+task :default => :spec

data/benchmarks/queueing.rb

@@ -0,0 +1,33 @@
+#!/usr/bin/env ruby
+
+require File.join(File.dirname(__FILE__), '..', 'lib', 'eq', 'boot', 'all')
+
+require 'benchmark'
+require 'tempfile'
+
+EQ.logger.level = Logger::Severity::ERROR
+
+class MyJob
+  def self.perform stuff
+  end
+end
+
+n = 1000
+Benchmark.bm(50) do |b|
+  EQ.boot
+  b.report('memory-based sqlite') do
+    n.times { |i| EQ.queue.push! MyJob, i }
+    EQ.queue.waiting_count # block
+  end
+  EQ.shutdown
+
+  EQ.config do |config|
+    config[:sqlite] = Tempfile.new('').path
+  end
+  EQ.boot
+  b.report('file-based sqlite') do
+    n.times { |i| EQ.queue.push! MyJob, i }
+    EQ.queue.waiting_count # block
+  end
+  EQ.shutdown
+end

data/benchmarks/working.rb

@@ -0,0 +1,35 @@
+#!/usr/bin/env ruby
+
+require File.join(File.dirname(__FILE__), '..', 'lib', 'eq', 'boot', 'all')
+
+require 'benchmark'
+require 'tempfile'
+
+EQ.logger.level = Logger::Severity::ERROR
+
+class MyJob
+  def self.perform stuff
+  end
+end
+
+n = 500
+Benchmark.bm(50) do |b|
+  EQ.boot_queueing
+  n.times { |i| EQ.queue.push! MyJob, i }
+  b.report('memory-based sqlite') do
+    EQ.boot_working
+    sleep 0.01 until EQ.queue.waiting_count == 0
+  end
+  EQ.shutdown
+
+  EQ.config do |config|
+    config[:sqlite] = Tempfile.new('').path
+  end
+  EQ.boot_queueing
+  n.times { |i| EQ.queue.push! MyJob, i }
+  b.report('file-based sqlite') do
+    EQ.boot_working
+    sleep 0.01 until EQ.queue.waiting_count == 0
+  end
+  EQ.shutdown
+end

data/eq.gemspec

@@ -0,0 +1,26 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/eq/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["Jens Bissinger"]
+  gem.email         = ["mail@jens-bissinger.de"]
+  gem.description   = %q{Embedded Queueing. Background processing within a single process using multi-threading and a SQL database.}
+  gem.summary       = %q{Based on Celluloid (multi-threading) and Sequel (SQLite3, MySQL, PostgreSQL, ...).}
+  gem.homepage      = "https://github.com/dpree/eq"
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name          = "eq"
+  gem.require_paths = ["lib"]
+  gem.version       = EQ::VERSION
+
+  gem.add_dependency "sqlite3"
+  gem.add_dependency "sequel"
+  gem.add_dependency "celluloid"
+  gem.add_development_dependency "guard"
+  gem.add_development_dependency "guard-rspec"
+  gem.add_development_dependency "rspec"
+  gem.add_development_dependency "rake"
+  gem.add_development_dependency "timecop"
+end

data/examples/queueing.rb

@@ -0,0 +1,30 @@
+#!/usr/bin/env ruby
+require File.join(File.dirname(__FILE__), '..', 'lib', 'eq', 'boot', 'queueing')
+
+EQ.logger.level = Logger::Severity::INFO
+EQ.config do |config|
+  config[:sqlite] = "foo.sqlite"
+end
+
+def say words; EQ.logger.info(words); end
+
+class SingleJob
+  def self.perform
+    sleep 0.05
+  end
+end
+
+require 'timeout'
+begin
+  Timeout.timeout(120) do
+    EQ.boot
+    loop do
+      say "pushed!"
+      sleep 0.05
+      EQ.queue.push! SingleJob
+    end
+  end
+rescue Timeout::Error
+  say "shutdown: #{EQ.shutdown}"
+end
+

data/examples/simple_usage.rb

@@ -0,0 +1,35 @@
+#!/usr/bin/env ruby
+require File.join(File.dirname(__FILE__), '..', 'lib', 'eq', 'boot', 'all')
+
+# Define a Job class with a perform method.
+class MyJob
+  RESULT_PATH = 'my_job_result.txt'
+
+  def self.perform enqueued_at, workload_in_seconds
+    # do some long running stuff here
+    sleep workload_in_seconds
+    File.open RESULT_PATH, 'a' do |file|
+      finished_at = Time.now
+      file.puts "Processed a job with workload of #{workload_in_seconds}s:\n"\
+                " - enqueued at     = #{enqueued_at}\n"\
+                " - finished at     = #{finished_at}\n"\
+                " - actual workload = #{finished_at - enqueued_at}s"
+    end
+  end
+end
+
+# Cleanup results file.
+File.delete MyJob::RESULT_PATH if File.exists? MyJob::RESULT_PATH
+
+# Start the EQ system.
+EQ.boot
+
+# Enqueue some work.
+EQ.queue.push MyJob, Time.now, 1
+EQ.queue.push MyJob, Time.now, 2
+
+# Wait some time to get the work done.
+sleep 3
+
+# Read the results file.
+puts File.read MyJob::RESULT_PATH

data/examples/working.rb

@@ -0,0 +1,27 @@
+#!/usr/bin/env ruby
+require File.join(File.dirname(__FILE__), '..', 'lib', 'eq', 'boot', 'all')
+
+EQ.logger.level = Logger::Severity::INFO
+EQ.config do |config|
+  config[:sqlite] = "foo.sqlite"
+end
+
+def say words; EQ.logger.info(words); end
+
+class SingleJob
+  def self.perform
+    say "worked!"
+    sleep 0.05
+  end
+end
+
+require 'timeout'
+begin
+  Timeout.timeout(120) do
+    EQ.boot
+    sleep 0.5 while EQ.working?
+  end
+rescue Timeout::Error
+  say "shutdown: #{EQ.shutdown}"
+end
+

data/lib/eq/boot/all.rb

@@ -0,0 +1,2 @@
+require File.join(File.dirname(__FILE__), 'queueing')
+require File.join(File.dirname(__FILE__), 'working')

data/lib/eq/boot/queueing.rb

@@ -0,0 +1 @@
+require File.join(File.dirname(__FILE__), '..', '..', 'eq-queueing')

data/lib/eq/boot/working.rb

@@ -0,0 +1 @@
+require File.join(File.dirname(__FILE__), '..', '..', 'eq-working')

data/lib/eq/job.rb

@@ -0,0 +1,25 @@
+module EQ
+  class Job < Struct.new(:id, :serialized_payload)
+    class << self
+      def dump *unserialized_payload
+        Marshal.dump(unserialized_payload.flatten)
+      end
+
+      def load id, serialized_payload
+        Job.new id, serialized_payload
+      end
+    end
+
+    # unmarshals the serialized_payload
+    def unpack
+      #[const_name.split("::").inject(Kernel){|res,current| res.const_get(current)}, *payload]
+      Marshal.load(serialized_payload)
+    end
+
+    # calls MyJobClass.perform(*payload)
+    def perform
+      const, *payload = unpack
+      const.perform *payload
+    end
+  end
+end

data/lib/eq/logging.rb

@@ -0,0 +1,15 @@
+module EQ
+  module Logging
+    def debug message
+      EQ.logger.debug message
+    end
+
+    def info message
+      EQ.logger.info message
+    end
+
+    def log_error message
+      EQ.logger.error message
+    end
+  end
+end

data/lib/eq/version.rb

@@ -0,0 +1,3 @@
+module EQ
+  VERSION = '0.0.1'
+end

data/lib/eq-queueing/backends/sequel.rb

@@ -0,0 +1,133 @@
+require 'sequel'
+
+module EQ::Queueing::Backends
+
+  # this class provides a queueing backend via Sequel ORM mapper
+  # basically any database adapter known by Sequel is supported
+  # configure via EQ::conig[:sequel]
+  class Sequel
+    include EQ::Logging
+
+    TABLE_NAME = :jobs
+
+    attr_reader :db
+
+    # establishes the connection to the database and ensures that
+    # the jobs table is created
+    def initialize config
+      connect config
+      create_table_if_not_exists!
+    end
+
+    # @param [#to_sequel_block] payload
+    # @return [Fixnum] id of the job
+    def push payload
+      jobs.insert payload: payload.to_sequel_blob, created_at: Time.now
+    rescue ::Sequel::DatabaseError => e
+      retry if on_error e
+    end
+
+    # pulls a job from the waiting stack and moves it to the
+    # working stack. sets a timestamp :started_working_at so that
+    # the working duration can be tracked.
+    # @param [Time] now
+    # @return [Array<Fixnum, String>] job data consisting of id and payload
+    def reserve
+      db.transaction do
+        if job = waiting.order(:id.asc).limit(1).first
+          job[:started_working_at] = Time.now
+          update_job!(job)
+          [job[:id], job[:payload]]
+        end
+      end
+    rescue ::Sequel::DatabaseError => e
+      retry if on_error e
+    end
+
+    # finishes a job in the working queue
+    # @param [Fixnum] id of the job
+    # @return [TrueClass, FalseClass] true, when there was a job that could be deleted
+    def pop id
+      jobs.where(id: id).delete == 1
+    rescue ::Sequel::DatabaseError => e
+      retry if on_error e
+    end
+
+    # list of jobs waiting to be worked on
+    def waiting
+      jobs.where(started_working_at: nil)
+    rescue ::Sequel::DatabaseError => e
+      retry if on_error e
+    end
+
+    # list of jobs currentyl being worked on
+    def working
+      waiting.invert
+    rescue ::Sequel::DatabaseError => e
+      retry if on_error e
+    end
+
+    # list of all jobs
+    def jobs
+      db[TABLE_NAME]
+    rescue ::Sequel::DatabaseError => e
+      retry if on_error e
+    end
+
+    # updates a changed job object, uses the :id key to identify the job
+    # @param [Hash] changed job
+    def update_job! changed_job
+      jobs.where(id: changed_job[:id]).update(changed_job)
+    rescue ::Sequel::DatabaseError => e
+      retry if on_error e
+    end
+
+    # this re-enqueues jobs that timed out
+    # @return [Fixnum] number of jobs that were re-enqueued
+    def requeue_timed_out_jobs
+      # 10 seconds ago
+      jobs.where{started_working_at <= (Time.now - EQ.config.job_timeout)}\
+          .update(started_working_at: nil)
+    end
+
+    # statistics:
+    #   - #job_count
+    #   - #working_count
+    #   - #waiting_count
+    %w[ job working waiting ].each do |stats_name|
+      define_method "#{stats_name}_count" do
+        begin
+          send(stats_name).send(:count)
+        rescue ::Sequel::DatabaseError => e
+          retry if on_error e
+        end
+      end
+    end
+
+  private
+
+    # connects to the given database config
+    def connect config
+      @db = ::Sequel.connect config
+    rescue ::Sequel::DatabaseError => e
+      retry if on_error e
+    end
+
+    def create_table_if_not_exists!
+      db.create_table? TABLE_NAME do
+        primary_key :id
+        Timestamp :created_at
+        Timestamp :started_working_at
+        Blob :payload
+      end
+    rescue ::Sequel::DatabaseError => e
+      retry if on_error e
+    end
+
+    def on_error error
+      log_error error.inspect
+      sleep 0.05
+      true
+    end
+  end
+end

data/lib/eq-queueing/backends.rb

@@ -0,0 +1,6 @@
+module EQ::Queueing
+  module Backends
+  end
+end
+
+require File.join(File.dirname(__FILE__), 'backends', 'sequel')

data/lib/eq-queueing/queue.rb

@@ -0,0 +1,54 @@
+module EQ::Queueing
+  
+  # this class basically provides a API that wraps the low-level calls
+  # to the queueing backend that is configured and passed to the #initialize method
+  # furthermore this class adds some functionality to serialize / deserialze
+  # using the Job class
+  class Queue
+    include Celluloid
+    include EQ::Logging
+
+    %w[ job_count waiting_count working_count waiting working ].each do |method_name|
+      define_method method_name do
+        queue.send(method_name)
+      end
+    end
+    alias :size :job_count
+    
+    # @param [Object] queue_backend
+    def initialize queue_backend
+      @queue = queue_backend
+    end
+
+    # @param [Array<Class, *payload>] unserialized_payload
+    # @return [Fixnum] job_id 
+    def push *unserialized_payload
+      debug "enqueing #{unserialized_payload.inspect} ..."
+      queue.push EQ::Job.dump(unserialized_payload)
+    end
+
+    # @return [EQ::Job, nilClass] job instance
+    def reserve
+      requeue_timed_out_jobs
+      if serialized_job = queue.reserve
+        job_id, serialized_payload = *serialized_job
+        job = EQ::Job.load job_id, serialized_payload
+        debug "dequeud #{job.inspect}"
+        job
+      end
+    end
+
+    # @return [TrueClass, FalseClass]
+    def pop job_id
+      queue.pop job_id
+    end
+
+    # re-enqueues jobs that timed out
+    def requeue_timed_out_jobs
+      queue.requeue_timed_out_jobs
+    end
+
+    attr_reader :queue
+
+  end
+end

data/lib/eq-queueing.rb

@@ -0,0 +1,30 @@
+require File.join(File.dirname(__FILE__), 'eq')
+require File.join(File.dirname(__FILE__), 'eq-queueing', 'backends')
+require File.join(File.dirname(__FILE__), 'eq-queueing', 'queue')
+
+module EQ::Queueing
+  module_function
+
+  def boot
+    EQ::Queueing::Queue.supervise_as :_eq_queueing, initialize_queueing_backend
+  end
+
+  def shutdown
+    queue.terminate! if queue
+  end
+
+  def queue
+    Celluloid::Actor[:_eq_queueing]
+  end
+
+  # @raise ConfigurationError when EQ.config.queue is not supported
+  def initialize_queueing_backend
+    queue_config = EQ.config.send(EQ.config.queue)
+    case EQ.config.queue
+    when 'sequel'
+      EQ::Queueing::Backends::Sequel.new queue_config
+    else
+      raise EQ::ConfigurationError, "EQ.config.queue = '#{EQ.config.queue}' is not supported!"
+    end
+  end
+end

data/lib/eq-working/manager.rb

@@ -0,0 +1,31 @@
+module EQ::Working
+  class Manager
+    include Celluloid
+    include EQ::Logging
+
+    def initialize
+      run!
+    end
+
+    # polls the EQ.queue via EQ.queue.reserve
+    def run
+      debug "worker manager running"
+      loop do
+        if EQ.queue && job = EQ.queue.reserve
+          debug "got #{job.inspect}"
+          if worker = EQ::Working.worker_pool
+            debug ' - found worker'
+            worker.process! job
+          else
+            debug ' - no worker'
+          end
+        else
+          # currently no job
+        end
+        sleep 0.01
+      end
+    rescue Celluloid::DeadActorError
+      retry
+    end
+  end
+end

data/lib/eq-working/system.rb

@@ -0,0 +1,10 @@
+module EQ::Working
+  class System < Celluloid::SupervisionGroup
+    include EQ::Logging
+
+    # TODO celluloid: replace this with the following in the next version
+    # pool Worker, as: _eq_working_pool
+    supervise EQ::Working::Worker, as: :_eq_working_pool, method: 'pool_link'
+    supervise EQ::Working::Manager, as: :_eq_working_manager
+  end
+end

data/lib/eq-working/worker.rb

@@ -0,0 +1,18 @@
+module EQ::Working
+  class Worker
+    include Celluloid
+    include EQ::Logging
+
+    def initialize
+      debug "initialized worker"
+    end
+
+    # @param [EQ::Job] job instance
+    # @return [TrueClass, FalseClass] true when job is done and deleted
+    def process job
+      debug "processing #{job.inspect}"
+      job.perform
+      EQ.queue.pop job.id
+    end
+  end
+end

data/lib/eq-working.rb

@@ -0,0 +1,24 @@
+require File.join(File.dirname(__FILE__), 'eq')
+require File.join(File.dirname(__FILE__), 'eq-working', 'worker')
+require File.join(File.dirname(__FILE__), 'eq-working', 'manager')
+require File.join(File.dirname(__FILE__), 'eq-working', 'system')
+
+module EQ::Working
+  module_function
+
+  def boot
+    Celluloid::Actor[:_eq_working] = EQ::Working::System.run!
+  end
+
+  def shutdown
+    worker.finalize! if worker
+  end
+
+  def worker
+    Celluloid::Actor[:_eq_working]
+  end
+
+  def worker_pool
+    Celluloid::Actor[:_eq_working_pool]
+  end
+end

data/lib/eq.rb

@@ -0,0 +1,65 @@
+require 'ostruct'
+require 'celluloid'
+
+require File.join(File.dirname(__FILE__), 'eq', 'version')
+require File.join(File.dirname(__FILE__), 'eq', 'logging')
+require File.join(File.dirname(__FILE__), 'eq', 'job')
+
+module EQ
+  class ConfigurationError < ArgumentError; end
+
+  DEFAULT_CONFIG = {
+    queue: 'sequel',
+    sequel: 'sqlite:/',
+    job_timeout: 5 # in seconds
+  }.freeze
+
+  module_function
+
+  def config
+    @config ||= OpenStruct.new DEFAULT_CONFIG
+    yield @config if block_given?
+    @config
+  end
+
+  # this boots queuing and working
+  # optional: to use another queuing or working subsystem just do
+  # require 'eq/working' or require 'eq/queueing' instead of require 'eq/all'
+  def boot
+    boot_queueing if defined? EQ::Queueing
+    boot_working if defined? EQ::Working
+  end
+
+  def shutdown
+    EQ::Working.shutdown if defined? EQ::Working
+    EQ::Queueing.shutdown if defined? EQ::Queueing
+  end
+
+  def boot_queueing
+    EQ::Queueing.boot
+  end
+
+  def boot_working
+    EQ::Working.boot
+  end
+
+  def queue
+    EQ::Queueing.queue
+  end
+
+  def worker
+    EQ::Working.worker
+  end
+
+  def queueing?
+    queue.alive?
+  end
+
+  def working?
+    worker.alive?
+  end
+
+  def logger
+    Celluloid.logger
+  end
+end

data/spec/lib/eq/job_spec.rb

@@ -0,0 +1,26 @@
+require 'spec_helper'
+
+describe EQ::Job do |variable|
+  it 'dumps const and payload' do
+    payload = EQ::Job.dump([EQ, 'bar', 'baz'])
+    Marshal.load(payload).should == [EQ, 'bar', 'baz']
+  end
+
+  it 'loads const and payload' do
+    serialized_payload = Marshal.dump [EQ, 'foo', 'bar']
+    job = EQ::Job.load(1, serialized_payload)
+    job.unpack.should == [EQ, 'foo', 'bar']
+  end
+
+  it 'performs using const.perform(*payload)' do
+    class MyJob
+      def self.perform(*args)
+        {result: args}
+      end
+    end
+    my_job_args = [1,2,3]
+    serialized_payload = EQ::Job.dump(MyJob, *my_job_args)
+    job = EQ::Job.load(1, serialized_payload)
+    job.perform.should == {result: my_job_args}
+  end
+end

data/spec/lib/eq-queueing/backends/sequel_spec.rb

@@ -0,0 +1,23 @@
+require 'spec_helper'
+
+describe EQ::Queueing::Backends::Sequel do
+  subject { EQ::Queueing::Backends::Sequel.new 'sqlite:/' }
+  it_behaves_like 'abstract queue'
+  it_behaves_like 'queue backend'
+
+  it 'handles ::Sequel::DatabaseError with retry' do
+    db_method = subject.instance_eval('method(:jobs)')
+    raised = false
+    subject.stub(:jobs).and_return do |arg|
+      if raised
+        db_method.call
+      else
+        raised = true
+        raise ::Sequel::DatabaseError, "failed"
+      end
+    end
+    subject.waiting_count.should == 0
+    subject.push "foo"
+    subject.waiting_count.should == 1
+  end
+end

data/spec/lib/eq-queueing/queue_spec.rb

@@ -0,0 +1,67 @@
+require 'spec_helper'
+
+describe EQ::Queueing::Queue do
+  let(:queue_backend) do
+    Class.new(Struct.new(:waiting, :working)) do
+      def push payload
+        raise ArgumentError, "queue_backend mock only supports one waiting job at a time" if waiting
+        self.waiting = [1, payload]
+        1
+      end
+
+      def reserve
+        raise ArgumentError, "queue_backend mock only supports one working job at a time" if working
+        if self.working = waiting
+          self.working << Time.now
+          self.waiting = nil
+          return working
+        end
+      end
+
+      def requeue_timed_out_jobs
+        raise ArgumentError, "queue_backend mock only supports on waiting job at a time" if waiting && working
+        # timeout after EQ.config.job_timeout seconds
+        if working && working.last <= (Time.now - EQ.config.job_timeout)
+          working.pop
+          self.waiting = working
+          self.working = nil
+          1
+        else
+          0
+        end
+      end
+
+      def pop id
+        result = false
+        
+        if waiting && id == waiting.first
+          self.waiting = nil
+          result = true
+        end
+
+        if working && id == working.first
+          self.working = nil
+          result = true
+        end
+        
+        result
+      end
+
+      def waiting_count; waiting ? 1 : 0; end
+      def working_count; working ? 1 : 0; end
+    end.new
+  end
+  subject { EQ::Queueing::Queue.new(queue_backend) }
+  it_behaves_like 'abstract queue'
+
+  it 'serializes jobs' do
+    EQ::Job.should_receive(:dump).with(["foo"])
+    subject.push "foo"
+  end
+
+  it 'deserializes jobs' do
+    subject.push "foo"
+    EQ::Job.should_receive(:load).with(1, EQ::Job.dump(["foo"]))
+    subject.reserve
+  end
+end

data/spec/lib/eq_spec.rb

@@ -0,0 +1,5 @@
+require 'spec_helper'
+
+describe EQ do
+
+end

data/spec/spec_helper.rb

@@ -0,0 +1,22 @@
+# This file was generated by the `rspec --init` command. Conventionally, all
+# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
+# Require this file using `require "spec_helper"` to ensure that it is only
+# loaded once.
+#
+# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
+require File.join(File.dirname(__FILE__), '..', 'lib', 'eq', 'boot', 'all')
+Dir[File.join(File.dirname(__FILE__), '/support/**/*.rb')].each {|f| require f; puts f}
+
+RSpec.configure do |config|
+  config.treat_symbols_as_metadata_keys_with_true_values = true
+  config.run_all_when_everything_filtered = true
+  config.filter_run :focus
+
+  # Run specs in random order to surface order dependencies. If you find an
+  # order dependency and want to debug it, you can fix the order by providing
+  # the seed, which is printed after each run.
+  #     --seed 1234
+  config.order = 'random'
+end
+
+require "timecop"

data/spec/support/shared_examples_for_queue.rb

@@ -0,0 +1,75 @@
+shared_examples_for 'queue backend' do
+  it 'pushes and pops' do
+    subject.push "foo"
+    job_id, payload = *subject.reserve
+    job_id.should == 1
+    payload.should == "foo"
+  end
+end
+
+shared_examples_for 'abstract queue' do
+  it 'pushes jobs' do
+    subject.waiting_count.should == 0
+    subject.working_count.should == 0
+    subject.push("foo").should == 1 # job id
+    subject.waiting_count.should == 1
+    subject.working_count.should == 0
+  end
+
+  it 'reserves jobs' do
+    id = subject.push "foo"
+    subject.reserve
+    subject.waiting_count.should == 0
+    subject.working_count.should == 1
+    subject.pop id
+    subject.waiting_count.should == 0
+    subject.working_count.should == 0
+  end
+
+  it 'pops jobs' do
+    subject.pop(1).should be_false # no job
+    subject.push "foo"
+    subject.pop(1).should be_true # one job
+    subject.pop(1).should be_false # again no job"
+  end
+
+  it 'puts working job back on waiting when they timeout via #requeue_timed_out_jobs' do
+    # freeze time on start of 1986
+    Timecop.freeze(Time.new(1986)) do
+    
+      # create a job
+      id = subject.push "foo"
+
+      # start working
+      data = subject.reserve
+
+      # no on working at the beginning
+      subject.waiting_count.should == 0
+      subject.working_count.should == 1
+
+      # no one will be re-enqueued
+      subject.requeue_timed_out_jobs.should == 0
+
+      # no on working after senseless re-enqueueing
+      subject.waiting_count.should == 0
+      subject.working_count.should == 1
+
+    end
+
+    # freeze the time to 10s in the future
+    Timecop.freeze(Time.new(1986, 01, 01, 00, 00, EQ.config.job_timeout)) do  
+      
+      # nothing happened yet...
+      subject.waiting_count.should == 0
+      subject.working_count.should == 1
+
+      # this time one will be re-enqueued
+      subject.requeue_timed_out_jobs.should == 1
+    
+      # now the old job is available again
+      subject.waiting_count.should == 1
+      subject.working_count.should == 0
+
+    end
+  end
+end

metadata

@@ -0,0 +1,217 @@
+--- !ruby/object:Gem::Specification
+name: eq
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Jens Bissinger
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-08-23 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sequel
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: celluloid
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: guard
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: guard-rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: timecop
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Embedded Queueing. Background processing within a single process using
+  multi-threading and a SQL database.
+email:
+- mail@jens-bissinger.de
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .rspec
+- .travis.yml
+- Gemfile
+- Guardfile
+- LICENSE
+- README.md
+- Rakefile
+- benchmarks/queueing.rb
+- benchmarks/working.rb
+- eq.gemspec
+- examples/queueing.rb
+- examples/simple_usage.rb
+- examples/working.rb
+- lib/eq-queueing.rb
+- lib/eq-queueing/backends.rb
+- lib/eq-queueing/backends/sequel.rb
+- lib/eq-queueing/queue.rb
+- lib/eq-working.rb
+- lib/eq-working/manager.rb
+- lib/eq-working/system.rb
+- lib/eq-working/worker.rb
+- lib/eq.rb
+- lib/eq/boot/all.rb
+- lib/eq/boot/queueing.rb
+- lib/eq/boot/working.rb
+- lib/eq/job.rb
+- lib/eq/logging.rb
+- lib/eq/version.rb
+- spec/lib/eq-queueing/backends/sequel_spec.rb
+- spec/lib/eq-queueing/queue_spec.rb
+- spec/lib/eq/job_spec.rb
+- spec/lib/eq_spec.rb
+- spec/spec_helper.rb
+- spec/support/shared_examples_for_queue.rb
+homepage: https://github.com/dpree/eq
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: Based on Celluloid (multi-threading) and Sequel (SQLite3, MySQL, PostgreSQL,
+  ...).
+test_files:
+- spec/lib/eq-queueing/backends/sequel_spec.rb
+- spec/lib/eq-queueing/queue_spec.rb
+- spec/lib/eq/job_spec.rb
+- spec/lib/eq_spec.rb
+- spec/spec_helper.rb
+- spec/support/shared_examples_for_queue.rb
+has_rdoc: