eq 0.0.1 → 0.1.0

data/lib/eq-web/server.rb

@@ -0,0 +1,39 @@
+require 'sinatra'
+
+module EQ::Web
+  class Server < Sinatra::Base
+    dir = File.dirname(File.expand_path(__FILE__))
+    set :views,  "#{dir}/views"
+
+    get '/' do
+      erb :index
+    end
+
+    get '/delete' do
+      EQ.queue.clear
+    end
+
+    get '/delete/:id' do
+      EQ.queue.pop(params[:id])
+      redirect url_path
+    end
+
+    helpers do
+      include Rack::Utils
+      alias_method :h, :escape_html
+
+      def current_page
+        url_path request.path_info.sub('/','')
+      end
+
+      def url_path(*path_parts)
+        [ path_prefix, path_parts ].join("/").squeeze('/')
+      end
+      alias_method :u, :url_path
+
+      def path_prefix
+        request.env['SCRIPT_NAME']
+      end
+    end
+  end
+end

data/lib/eq-web/views/index.erb

@@ -0,0 +1,45 @@
+<% if EQ.worker && EQ.worker.alive? %>
+  <h2><%= EQ::Working.pool_size %> Worker</h2>
+<% else %>
+  <h2>Worker deaktiviert</h2>
+<% end %>
+<% if EQ.scheduler && EQ.scheduler.alive? %>
+  <h2>Scheduler</h2>
+  <table>
+    <% EQ::Scheduling.events.each do |job_class, period| %>
+      <tr>
+        <td><%= job_class %></td>
+        <td><%= period %></td>
+      </tr>
+    <% end %>
+  </table>
+<% else %>
+  <h2>Scheduler deaktiviert</h2>
+<% end %>
+<% if EQ.queue && EQ.queue.alive? %>
+  <h2><%= EQ.queue.count %> Queue Jobs (<a href='<%=u "/delete" %>'>Remove all</a>)</h2>
+  <table>
+    <thead>
+      <tr>
+        <th>Id</th>
+        <th>Queue</th>
+        <th>Payload</th>
+        <th>CreatedAt</th>
+        <th>StartedWorkingAt</th>
+      </tr>
+    </thead>
+    <tbody>
+      <% EQ.queue.iterator do |job| %>
+        <tr>
+          <td><%= job[:id] %> (<a href='<%=u "/delete/#{job[:id]}" %>'>Remove</a>)</td>
+          <td><%= job[:queue] %></td>
+          <td><%= job[:payload] %></td>
+          <td><%= job[:created_at] %></td>
+          <td><%= job[:started_working_at] %></td>
+        </tr>
+      <% end %>
+    </tbody>
+  </table>
+<% else %>
+  <h2>Queue deaktiviert</h2>
+<% end %>

data/lib/eq-working.rb

@@ -1,24 +1,32 @@
 require File.join(File.dirname(__FILE__), 'eq')
 require File.join(File.dirname(__FILE__), 'eq-working', 'worker')
-require File.join(File.dirname(__FILE__), 'eq-working', 'manager')
-require File.join(File.dirname(__FILE__), 'eq-working', 'system')
 
 module EQ::Working
   module_function
 
+  EQ_WORKER = :_eq_working
+
   def boot
-    Celluloid::Actor[:_eq_working] = EQ::Working::System.run!
+    pool_size =EQ.config.worker_pool_size
+    case pool_size
+    when 0
+      puts "pool empty"
+    when 1
+      EQ::Working::Worker.supervise_as EQ_WORKER
+    else
+      Celluloid::Actor[EQ_WORKER] = EQ::Working::Worker.pool size: pool_size
+    end
   end
 
   def shutdown
-    worker.finalize! if worker
+    worker.terminate! if worker
   end
 
   def worker
-    Celluloid::Actor[:_eq_working]
+    Celluloid::Actor[EQ_WORKER]
   end
 
-  def worker_pool
-    Celluloid::Actor[:_eq_working_pool]
+  def pool_size
+    EQ.config.worker_pool_size
   end
 end

data/lib/eq-working/worker.rb

@@ -3,8 +3,35 @@ module EQ::Working
     include Celluloid
     include EQ::Logging
 
-    def initialize
-      debug "initialized worker"
+    def initialize autostart=true
+      # start working async
+      process_jobs! if autostart
+    end
+
+    def process_jobs
+      sleep EQ.config.worker_delay
+
+      # TODO check if this is really what we want here, does it stop gracefully?
+      while Celluloid::Actor.current.alive?
+        if job = look_for_a_job
+          debug "got #{job.inspect}"
+          
+          # this should happen in sync mode, because we don't want to pick
+          # too much jobs
+          process job
+        else
+          # currently no job
+          sleep 0.05
+        end
+      end
+    rescue Celluloid::DeadActorError
+      log_error 'dead'
+      sleep 0.05
+      retry
+    end
+
+    def look_for_a_job
+      EQ.queue.reserve if EQ.queue && EQ.queue.alive?
     end
 
     # @param [EQ::Job] job instance
@@ -12,7 +39,7 @@ module EQ::Working
     def process job
       debug "processing #{job.inspect}"
       job.perform
-      EQ.queue.pop job.id
+      EQ.queue.pop! job.id
     end
   end
 end

data/lib/eq.rb

@@ -1,20 +1,29 @@
+# STDLIB
 require 'ostruct'
+require 'forwardable'
+
+# rubygems
 require 'celluloid'
 
 require File.join(File.dirname(__FILE__), 'eq', 'version')
+require File.join(File.dirname(__FILE__), 'eq', 'error')
 require File.join(File.dirname(__FILE__), 'eq', 'logging')
 require File.join(File.dirname(__FILE__), 'eq', 'job')
 
 module EQ
+  extend SingleForwardable
+
   class ConfigurationError < ArgumentError; end
 
   DEFAULT_CONFIG = {
     queue: 'sequel',
     sequel: 'sqlite:/',
-    job_timeout: 5 # in seconds
+    job_timeout: 5, # in seconds
+    worker_pool_size: Celluloid.cores, # in threads
+    worker_delay: 0
   }.freeze
 
-  module_function
+module_function
 
   def config
     @config ||= OpenStruct.new DEFAULT_CONFIG
@@ -25,41 +34,40 @@ module EQ
   # this boots queuing and working
   # optional: to use another queuing or working subsystem just do
   # require 'eq/working' or require 'eq/queueing' instead of require 'eq/all'
-  def boot
-    boot_queueing if defined? EQ::Queueing
-    boot_working if defined? EQ::Working
-  end
+  def boot just=nil; manage :boot, just; end
+  def shutdown just=nil; manage :shutdown, just; end
 
-  def shutdown
-    EQ::Working.shutdown if defined? EQ::Working
-    EQ::Queueing.shutdown if defined? EQ::Queueing
-  end
-
-  def boot_queueing
-    EQ::Queueing.boot
-  end
+  def queue; EQ::Queueing.queue if queueing_loaded?; end
+  def worker; EQ::Working.worker if working_loaded?; end
+  def scheduler; EQ::Scheduling.scheduler if scheduling_loaded?; end
 
-  def boot_working
-    EQ::Working.boot
+  # queue methods
+  %w[ jobs waiting working
+      push reserve pop
+      push! pop!
+      count ].each do |method_name|
+    def_delegator :queue, method_name
   end
 
-  def queue
-    EQ::Queueing.queue
+  def alive?
+    alive = false
+    alive &= queue.alive? if queue
+    alive &= worker.alive? if worker
+    alive
   end
 
-  def worker
-    EQ::Working.worker
-  end
-
-  def queueing?
-    queue.alive?
-  end
-
-  def working?
-    worker.alive?
-  end
+  def logger; Celluloid.logger; end
 
-  def logger
-    Celluloid.logger
+  def queueing_loaded?; defined? EQ::Queueing; end
+  def working_loaded?; defined? EQ::Working; end
+  def scheduling_loaded?; defined? EQ::Scheduling; end
+  
+  # @param [#to_s] action is the method name to execute on all parts
+  # @param [#to_s] specify just to execute the action on one part
+  def manage action, just=nil
+    what = just ? just.to_s : "queue work schedul"
+    EQ::Queueing.send(action) if what =~ /queue/ && queueing_loaded?
+    EQ::Working.send(action) if what =~ /work/ && working_loaded?
+    EQ::Scheduling.send(action) if what =~ /schedu/ && working_loaded?
   end
 end

data/lib/eq/boot/all.rb

@@ -1,2 +1,3 @@
 require File.join(File.dirname(__FILE__), 'queueing')
+require File.join(File.dirname(__FILE__), 'scheduling')
 require File.join(File.dirname(__FILE__), 'working')

data/lib/eq/boot/scheduling.rb

@@ -0,0 +1 @@
+require File.join(File.dirname(__FILE__), '..', '..', 'eq-scheduling')

data/lib/eq/error.rb

@@ -0,0 +1,4 @@
+module EQ
+  class Error < StandardError
+  end
+end

data/lib/eq/job.rb

@@ -1,25 +1,31 @@
 module EQ
-  class Job < Struct.new(:id, :serialized_payload)
-    class << self
-      def dump *unserialized_payload
-        Marshal.dump(unserialized_payload.flatten)
-      end
+  class Job
+    class UnknownJobClassError < EQ::Error; end
+    attr_reader :id, :queue, :payload
 
-      def load id, serialized_payload
-        Job.new id, serialized_payload
-      end
-    end
-
-    # unmarshals the serialized_payload
-    def unpack
-      #[const_name.split("::").inject(Kernel){|res,current| res.const_get(current)}, *payload]
-      Marshal.load(serialized_payload)
+    def initialize id, queue, payload=nil
+      @id = id
+      @queue = queue.to_s
+      @payload = payload
     end
 
     # calls MyJobClass.perform(*payload)
     def perform
-      const, *payload = unpack
-      const.perform *payload
+      job_class.perform *payload
+    end
+
+    def job_class
+      queue.split("::").inject(Kernel){|constant,part| constant.const_get(part)}
+    rescue NameError => e
+      raise UnknownJobClassError, e.to_s
+    end
+
+    def unique?
+      config(:unique) == true
+    end
+
+    def config name
+      job_class.instance_variable_get "@#{name}"
     end
   end
 end

data/lib/eq/version.rb

@@ -1,3 +1,3 @@
 module EQ
-  VERSION = '0.0.1'
+  VERSION = '0.1.0'
 end

data/log/.gitkeep

@@ -0,0 +1 @@
+.gitkeep

data/spec/lib/eq-queueing/backends/leveldb_spec.rb

@@ -0,0 +1,32 @@
+require 'spec_helper'
+
+EQ::Queueing::Backends.require_queue 'leveldb'
+
+describe EQ::Queueing::Backends::LevelDB do
+  subject do
+    FileUtils.rm_rf 'tmp/rspec/queue.leveldb'
+    EQ::Queueing::Backends::LevelDB.new 'tmp/rspec/queue.leveldb'
+  end
+  it_behaves_like 'queue backend'
+
+  it 'persists created_at correctly' do
+    job_id = nil
+    created_at = Time.new(1986, 01, 01, 00, 00)
+    Timecop.freeze(created_at) do  
+      job_id = subject.push EQ::Job.new(nil, AJob)
+    end
+    subject.jobs.find_created_at(job_id).should == created_at
+  end
+
+  it 'persists started_working_at correctly' do
+    job_id = nil
+    Timecop.freeze(Time.new(1986, 01, 01, 00, 00, 0)) do
+      job_id = subject.push EQ::Job.new(nil, AJob)
+    end
+    started_working_at = Time.new(1986, 01, 01, 00, 01)
+    Timecop.freeze(started_working_at) do
+      subject.reserve
+    end
+    subject.jobs.find_started_working_at(job_id).should == started_working_at
+  end
+end

data/spec/lib/eq-queueing/backends/sequel_spec.rb

@@ -1,8 +1,9 @@
 require 'spec_helper'
 
+EQ::Queueing::Backends.require_queue 'sequel'
+
 describe EQ::Queueing::Backends::Sequel do
   subject { EQ::Queueing::Backends::Sequel.new 'sqlite:/' }
-  it_behaves_like 'abstract queue'
   it_behaves_like 'queue backend'
 
   it 'handles ::Sequel::DatabaseError with retry' do
@@ -16,8 +17,8 @@ describe EQ::Queueing::Backends::Sequel do
         raise ::Sequel::DatabaseError, "failed"
       end
     end
-    subject.waiting_count.should == 0
-    subject.push "foo"
-    subject.waiting_count.should == 1
+    subject.count(:waiting).should == 0
+    subject.push EQ::Job.new(nil, AJob)
+    subject.count(:waiting).should == 1
   end
 end

data/spec/lib/eq-queueing/queue_spec.rb

@@ -1,67 +1,36 @@
 require 'spec_helper'
 
 describe EQ::Queueing::Queue do
-  let(:queue_backend) do
-    Class.new(Struct.new(:waiting, :working)) do
-      def push payload
-        raise ArgumentError, "queue_backend mock only supports one waiting job at a time" if waiting
-        self.waiting = [1, payload]
-        1
-      end
-
-      def reserve
-        raise ArgumentError, "queue_backend mock only supports one working job at a time" if working
-        if self.working = waiting
-          self.working << Time.now
-          self.waiting = nil
-          return working
-        end
-      end
-
-      def requeue_timed_out_jobs
-        raise ArgumentError, "queue_backend mock only supports on waiting job at a time" if waiting && working
-        # timeout after EQ.config.job_timeout seconds
-        if working && working.last <= (Time.now - EQ.config.job_timeout)
-          working.pop
-          self.waiting = working
-          self.working = nil
-          1
-        else
-          0
-        end
-      end
-
-      def pop id
-        result = false
-        
-        if waiting && id == waiting.first
-          self.waiting = nil
-          result = true
-        end
-
-        if working && id == working.first
-          self.working = nil
-          result = true
-        end
-        
-        result
-      end
-
-      def waiting_count; waiting ? 1 : 0; end
-      def working_count; working ? 1 : 0; end
-    end.new
+  subject do
+    FileUtils.rm_rf 'tmp/rspec/queue.leveldb'
+    EQ::Queueing::Queue.new EQ::Queueing::Backends::LevelDB.new('tmp/rspec/queue.leveldb')
   end
-  subject { EQ::Queueing::Queue.new(queue_backend) }
-  it_behaves_like 'abstract queue'
 
-  it 'serializes jobs' do
-    EQ::Job.should_receive(:dump).with(["foo"])
-    subject.push "foo"
+  it 'instantiates EQ::Job' do
+    id = nil
+    job_class = AJob
+    payload = ['bar', 'baz']
+    job = EQ::Job.new(id, job_class, payload)
+    EQ::Job.stub(:new).and_return(job)
+    EQ::Job.should_receive(:new).with(id, job_class, payload)
+    subject.push job_class, *payload
   end
 
-  it 'deserializes jobs' do
-    subject.push "foo"
-    EQ::Job.should_receive(:load).with(1, EQ::Job.dump(["foo"]))
-    subject.reserve
+  context 'unique jobs' do
+    it 'does not enqueue multiple times when args are the same' do
+      subject.count.should == 0
+      id = subject.push AUniqueJob, "foo"
+      subject.count.should == 1
+      id = subject.push AUniqueJob, "foo"
+      subject.count.should == 1
+    end
+
+    it 'does enqueue multiple times when args differ' do
+      subject.count.should == 0
+      id = subject.push AUniqueJob, "foo"
+      subject.count.should == 1
+      id = subject.push AUniqueJob, "bar"
+      subject.count.should == 2
+    end
   end
 end