eq 0.0.1 → 0.1.0

data/.gitignore

@@ -15,3 +15,4 @@ spec/reports
 test/tmp
 test/version_tmp
 tmp
+log/*.log

data/README.md

@@ -2,7 +2,7 @@
 
 # EQ - Embedded Queueing
 
-EQ is a little framework to queue tasks within a process.
+EQ is a little framework to queue and perform background tasks within a single-process ruby application. It uses the Celluloid actor framework to do the work in the background. Its queue backends persist your jobs. So your jobs will survive application stop/restart. 
 
 [![Travis-CI Build Status](https://secure.travis-ci.org/dpree/eq.png)](https://secure.travis-ci.org/dpree/eq)
 [![Code Climate](https://codeclimate.com/badge.png)](https://codeclimate.com/github/dpree/eq)
@@ -31,27 +31,67 @@ If you want to execute a simple example you can just run [examples/simple_usage.
 
 	EQ.boot
 	
-**3. Let EQ do some work you.**
+**3. Enqueue some jobs in the EQ queue.**
 
-	EQ.queue.push MyJob, 'foo'
-	EQ.queue.push MyJob, 'bar'
+	EQ.push MyJob, 'foo'
+	EQ.push MyJob, 'bar'
 	…	
 
+**5. Let EQ do your work.**
+
+	# EQ will spawn and maintain worker threads that execute the following for you:
+	MyJob.perform 'foo'
+	MyJob.perform 'bar'
+
+**6. Shutdown EQ gracefully when you're application is done.***
+
+	EQ.shutdown
+
+### Optional: Schedule jobs using Clockwork
+
+	module Clockwork
+	  every(5.seconds, MyJob)
+	  every(1.day, MyJob, :at => '00:00')
+	end
+
 ## Configuration
 
-Right now there is only one queueing backend available that is based on the Sequel gem. Therefore, basically any SQL database supported by Sequel might be used.
+Right now there are two queueing backends available, one that is based on the Sequel gem and one based on LevelDB. With Sequel basically any SQL database might be used. Just make sure that you install the Backend before running the application.
+
+### Sequel
 
-The default SQL database that is used is a . You can change it using any argument that Sequel.connect method would accept.
+Gemfile
+
+	gem 'sequel'
+	gem 'sqlite3'
+
+Configuration
+
+	EQ.config.queue = 'sequel'
 	
-	# SQLite3 in-memory (default) using String syntax
-	EQ.config.sequel = 'sqlite:/'
+	# With SQLite3 in-memory (default) using String syntax
+	# Caution: This won't persist your jobs!
+	EQ.config.sequel = 'sqlite:/' 
 	
-	# SQLite3 file using Hash syntax
+	# With SQLite3 file using Hash syntax
 	EQ.config.sequel = {adapter: 'sqlite', database: 'my_db.sqlite3'}
 	
-	# Postgres
+	# With Postgres
 	EQ.config.sequel = 'postgres://user:password@host:port/my_db'
 	
+	# Mysql, Oracle, etc.
+	# ...
+
+# LevelDB
+
+Gemfile
+
+	gem 'leveldb-ruby'
+
+Configuration
+
+	EQ.config.queue = 'leveldb'
+	EQ.config.leveldb = 'path/to/my/queue.leveldb'
 
 ### Logging
 
@@ -62,7 +102,7 @@ EQ uses the logging mechanism of the underlying Celluloid (`Celluloid.logger`) f
 	# Use the logger of your Rails application.
 	Celluloid.logger = Rails.logger
 	
-	# No more logging at all.
+	# No logging at all.
 	Celluloid.logger = Logger.new('/dev/null')
 
 ## Contributing

data/TODO.md

@@ -0,0 +1,10 @@
+# TODO
+
+* tests
+* job strategies
+** retry
+** drop
+** exponential backoff
+* stats / each iterator
+* delete all
+* configurable timeout per job

data/benchmarks/all.rb

@@ -0,0 +1,13 @@
+#!/usr/bin/env ruby
+
+puts ""
+puts "QUEUEING"
+system "#{File.join(File.dirname(__FILE__), 'queueing.rb')}"
+
+puts ""
+puts "WORKING"
+system "#{File.join(File.dirname(__FILE__), 'working.rb')}"
+
+puts ""
+puts "BOTH"
+system "#{File.join(File.dirname(__FILE__), 'parallel.rb')}"

data/benchmarks/parallel.rb

@@ -0,0 +1,23 @@
+#!/usr/bin/env ruby
+
+require File.join(File.dirname(__FILE__), 'queue_backend_benchmark')
+
+class MyJob
+  def self.perform stuff
+  end
+end
+
+class Executor < Struct.new(:n, :benchmark)
+  def report name, &configure
+    EQ.config &configure
+    EQ.boot
+    benchmark.report name do
+      n.times { |i| EQ.push MyJob, i }
+      sleep 0.01 until EQ.count(:waiting) == 0
+    end  
+    EQ.shutdown
+    sleep 0.05
+  end
+end
+
+QueueBackendBenchmark.new(Executor.new(100)).run

data/benchmarks/queue_backend_benchmark.rb

@@ -0,0 +1,27 @@
+require 'benchmark'
+require 'tmpdir'
+require File.join(File.dirname(__FILE__), '..', 'lib', 'eq', 'boot', 'all')
+
+EQ.logger.level = Logger::Severity::ERROR
+
+class QueueBackendBenchmark < Struct.new(:executor)
+  def run
+    Benchmark.bm(50) do |benchmark|
+      executor.benchmark = benchmark
+
+      executor.report 'sequel with sqlite3 (in-memory)' do |config|
+        config.queue = 'sequel'
+      end
+
+      executor.report 'sequel with sqlite3 (file)' do |config|
+        config.queue = 'sequel'
+        config.sequel = "sqlite://#{Dir.mktmpdir}/benchmark.sqlite3"
+      end
+
+      executor.report 'leveldb' do |config|
+        config.queue = 'leveldb'
+        config.leveldb = Dir.mktmpdir
+      end
+    end
+  end
+end

data/benchmarks/queueing.rb

@@ -1,33 +1,23 @@
 #!/usr/bin/env ruby
 
-require File.join(File.dirname(__FILE__), '..', 'lib', 'eq', 'boot', 'all')
-
-require 'benchmark'
-require 'tempfile'
-
-EQ.logger.level = Logger::Severity::ERROR
+require File.join(File.dirname(__FILE__), 'queue_backend_benchmark')
 
 class MyJob
   def self.perform stuff
   end
 end
 
-n = 1000
-Benchmark.bm(50) do |b|
-  EQ.boot
-  b.report('memory-based sqlite') do
-    n.times { |i| EQ.queue.push! MyJob, i }
-    EQ.queue.waiting_count # block
-  end
-  EQ.shutdown
-
-  EQ.config do |config|
-    config[:sqlite] = Tempfile.new('').path
-  end
-  EQ.boot
-  b.report('file-based sqlite') do
-    n.times { |i| EQ.queue.push! MyJob, i }
-    EQ.queue.waiting_count # block
+class Executor < Struct.new(:n, :benchmark)
+  def report name, &configure
+    EQ.config &configure
+    EQ.boot :queue
+    benchmark.report name do
+      n.times { |i| EQ.push! MyJob, i }
+      EQ.count
+    end  
+    EQ.shutdown
+    sleep 0.05
   end
-  EQ.shutdown
 end
+
+QueueBackendBenchmark.new(Executor.new(1000)).run

data/benchmarks/working.rb

@@ -1,35 +1,24 @@
 #!/usr/bin/env ruby
 
-require File.join(File.dirname(__FILE__), '..', 'lib', 'eq', 'boot', 'all')
-
-require 'benchmark'
-require 'tempfile'
-
-EQ.logger.level = Logger::Severity::ERROR
+require File.join(File.dirname(__FILE__), 'queue_backend_benchmark')
 
 class MyJob
   def self.perform stuff
   end
 end
 
-n = 500
-Benchmark.bm(50) do |b|
-  EQ.boot_queueing
-  n.times { |i| EQ.queue.push! MyJob, i }
-  b.report('memory-based sqlite') do
-    EQ.boot_working
-    sleep 0.01 until EQ.queue.waiting_count == 0
-  end
-  EQ.shutdown
-
-  EQ.config do |config|
-    config[:sqlite] = Tempfile.new('').path
-  end
-  EQ.boot_queueing
-  n.times { |i| EQ.queue.push! MyJob, i }
-  b.report('file-based sqlite') do
-    EQ.boot_working
-    sleep 0.01 until EQ.queue.waiting_count == 0
+class Executor < Struct.new(:n, :benchmark)
+  def report name, &configure
+    EQ.config &configure
+    EQ.boot :queue
+    n.times { |i| EQ.push MyJob, i }
+    benchmark.report name do
+      EQ.boot :worker
+      sleep 0.01 until EQ.count(:waiting) == 0
+    end  
+    EQ.shutdown
+    sleep 0.05
   end
-  EQ.shutdown
 end
+
+QueueBackendBenchmark.new(Executor.new(1000)).run

data/eq.gemspec

@@ -15,9 +15,19 @@ Gem::Specification.new do |gem|
   gem.require_paths = ["lib"]
   gem.version       = EQ::VERSION
 
-  gem.add_dependency "sqlite3"
-  gem.add_dependency "sequel"
   gem.add_dependency "celluloid"
+
+  # just to test the web view
+  gem.add_development_dependency "sinatra"
+
+  # just to test the scheduling  
+  gem.add_development_dependency "clockwork"
+
+  # just to test the queueing backends
+  gem.add_development_dependency "sequel"       # sequel backend
+  gem.add_development_dependency "sqlite3"      # sequel with sqlite
+  gem.add_development_dependency "leveldb-ruby" # leveldb backend
+
   gem.add_development_dependency "guard"
   gem.add_development_dependency "guard-rspec"
   gem.add_development_dependency "rspec"

data/examples/queueing.rb

@@ -3,7 +3,7 @@ require File.join(File.dirname(__FILE__), '..', 'lib', 'eq', 'boot', 'queueing')
 
 EQ.logger.level = Logger::Severity::INFO
 EQ.config do |config|
-  config[:sqlite] = "foo.sqlite"
+  config.sequel = "sqlite://foo.sqlite"
 end
 
 def say words; EQ.logger.info(words); end
@@ -16,7 +16,7 @@ end
 
 require 'timeout'
 begin
-  Timeout.timeout(120) do
+  Timeout.timeout(10) do
     EQ.boot
     loop do
       say "pushed!"

data/examples/scheduling.rb

@@ -0,0 +1,19 @@
+#!/usr/bin/env ruby
+require File.join(File.dirname(__FILE__), '..', 'lib', 'eq', 'boot', 'all')
+
+class MyJob
+  def self.perform
+    puts 'perfoming...'
+  end
+end
+
+module Clockwork
+  every(1.seconds, MyJob)
+  every(1.day, MyJob, :at => '00:00')
+end
+
+EQ.boot
+
+sleep 3
+
+EQ.shutdown

data/examples/simple_usage.rb

@@ -1,6 +1,8 @@
 #!/usr/bin/env ruby
 require File.join(File.dirname(__FILE__), '..', 'lib', 'eq', 'boot', 'all')
 
+Celluloid.logger = Logger.new('/dev/null')
+
 # Define a Job class with a perform method.
 class MyJob
   RESULT_PATH = 'my_job_result.txt'
@@ -21,15 +23,25 @@ end
 # Cleanup results file.
 File.delete MyJob::RESULT_PATH if File.exists? MyJob::RESULT_PATH
 
-# Start the EQ system.
-EQ.boot
+EQ.config do |c|
+  c.sequel = 'sqlite://examples/foo'
+end
+
+require 'timeout'
+begin
+  Timeout.timeout(12) do
 
-# Enqueue some work.
-EQ.queue.push MyJob, Time.now, 1
-EQ.queue.push MyJob, Time.now, 2
+    # Start the EQ system.
+    EQ.boot
 
-# Wait some time to get the work done.
-sleep 3
+    # Enqueue some work.
+    EQ.queue.push! MyJob, Time.now, 0.1
+    EQ.queue.push! MyJob, Time.now, 0.5
+
+    # Wait some time to get the work done.
+    sleep 1
+  end
+end
 
 # Read the results file.
-puts File.read MyJob::RESULT_PATH
+puts File.read(MyJob::RESULT_PATH)

data/examples/working.rb

@@ -3,7 +3,7 @@ require File.join(File.dirname(__FILE__), '..', 'lib', 'eq', 'boot', 'all')
 
 EQ.logger.level = Logger::Severity::INFO
 EQ.config do |config|
-  config[:sqlite] = "foo.sqlite"
+  config.sequel = "sqlite://foo.sqlite"
 end
 
 def say words; EQ.logger.info(words); end
@@ -17,7 +17,7 @@ end
 
 require 'timeout'
 begin
-  Timeout.timeout(120) do
+  Timeout.timeout(10) do
     EQ.boot
     sleep 0.5 while EQ.working?
   end

data/lib/eq-queueing.rb

@@ -5,8 +5,10 @@ require File.join(File.dirname(__FILE__), 'eq-queueing', 'queue')
 module EQ::Queueing
   module_function
 
+  EQ_QUEUE = :_eq_queueing
+
   def boot
-    EQ::Queueing::Queue.supervise_as :_eq_queueing, initialize_queueing_backend
+    EQ::Queueing::Queue.supervise_as EQ_QUEUE, EQ::Queueing::Backends.init(EQ.config)
   end
 
   def shutdown
@@ -14,17 +16,6 @@ module EQ::Queueing
   end
 
   def queue
-    Celluloid::Actor[:_eq_queueing]
-  end
-
-  # @raise ConfigurationError when EQ.config.queue is not supported
-  def initialize_queueing_backend
-    queue_config = EQ.config.send(EQ.config.queue)
-    case EQ.config.queue
-    when 'sequel'
-      EQ::Queueing::Backends::Sequel.new queue_config
-    else
-      raise EQ::ConfigurationError, "EQ.config.queue = '#{EQ.config.queue}' is not supported!"
-    end
+    Celluloid::Actor[EQ_QUEUE]
   end
 end

data/lib/eq-queueing/backends.rb

@@ -1,6 +1,35 @@
 module EQ::Queueing
   module Backends
+    class BackendLoadError < LoadError; end
+
+  module_function
+
+    # @params [#queue, #"#{queue}"] config
+    # @raise ConfigurationError when config.queue is not supported
+    def init config
+      if %w[ sequel leveldb ].include? config.queue
+        initialize_queue config
+      else
+        raise EQ::ConfigurationError, "config.queue = '#{config.queue}' is not supported!"
+      end
+    end
+
+    # @raise LoadError when required gem is not available
+    def initialize_queue config
+      queue_config = config.send(config.queue)
+      case EQ.config.queue
+      when 'sequel'
+        require_queue 'sequel'
+        EQ::Queueing::Backends::Sequel.new queue_config
+      when 'leveldb'
+        require_queue 'leveldb'
+        EQ::Queueing::Backends::LevelDB.new queue_config
+      end
+    end
+    
+    def require_queue queue_name
+      require File.join(File.dirname(__FILE__), 'backends', queue_name)
+    end
   end
 end
 
-require File.join(File.dirname(__FILE__), 'backends', 'sequel')