eq 0.0.1 → 0.1.0

data/lib/eq-queueing/backends/leveldb.rb

@@ -0,0 +1,232 @@
+require 'leveldb'
+
+module EQ::Queueing::Backends
+
+  # @note This is a unoreded storage, so there is no guaranteed work order
+  # @note assume there is nothing else than jobs
+  class LevelDB
+    class JobsCollection < Struct.new(:db, :name)
+      include EQ::Logging
+
+      QUEUE              = 'queue'.freeze
+      PAYLOAD            = 'payload'.freeze
+      CREATED_AT         = 'created_at'.freeze
+      STARTED_WORKING_AT = 'started_working_at'.freeze
+      NOT_WORKING = ''.freeze
+
+      # @param [EQ::Job] job
+      def push job
+        job_id = find_free_job_id
+        db["#{QUEUE}:#{job_id}"] = job.queue
+        db["#{PAYLOAD}:#{job_id}"] = serialize(job.payload) unless job.payload.nil?
+        db["#{CREATED_AT}:#{job_id}"] = serialize(Time.now)
+        db["#{STARTED_WORKING_AT}:#{job_id}"] = NOT_WORKING
+        job_id
+      end
+
+      def first_waiting
+        db.each do |k,v|
+          if k.include?(STARTED_WORKING_AT) && v == NOT_WORKING
+            return job_id_from_key(k)
+          end
+        end
+        nil
+      end
+
+      def working_iterator
+        db.each do |k,v|
+          if k.include?(STARTED_WORKING_AT) && v != NOT_WORKING
+            yield job_id_from_key(k), deserialize(v)
+          end
+        end
+      end
+
+      # @param [EQ::Job] job without id
+      def exists? job
+        db.each do |k,v|
+          if k.include?(QUEUE) && v == job.queue
+            if find_payload(job_id_from_key(k)) == job.payload
+              return true
+            end
+          end
+        end
+        false
+      end
+
+      def delete job_id
+        did_exist = !db["#{QUEUE}:#{job_id}"].nil?
+        db.batch do |batch|
+          batch.delete "#{QUEUE}:#{job_id}"
+          batch.delete "#{PAYLOAD}:#{job_id}"
+          batch.delete "#{CREATED_AT}:#{job_id}"
+          batch.delete "#{STARTED_WORKING_AT}:#{job_id}"
+        end
+        does_not_exist = db["#{QUEUE}:#{job_id}"].nil?
+        did_exist && does_not_exist
+      end
+
+      def start_working job_id
+        db["#{STARTED_WORKING_AT}:#{job_id}"] = serialize(Time.now)
+      end
+
+      def stop_working job_id
+        db["#{STARTED_WORKING_AT}:#{job_id}"] = NOT_WORKING
+      end
+
+      def find_queue job_id
+        db["#{QUEUE}:#{job_id}"]
+      end
+
+      def find_payload job_id
+        if raw = db["#{PAYLOAD}:#{job_id}"]
+          deserialize db["#{PAYLOAD}:#{job_id}"]
+        else
+          nil
+        end
+      end
+
+      def find_created_at job_id
+        if serialized_time = db["#{CREATED_AT}:#{job_id}"]
+          deserialize(serialized_time)
+        end
+      end
+
+      def find_started_working_at job_id
+        if serialized_time = db["#{STARTED_WORKING_AT}:#{job_id}"]
+          deserialize(serialized_time)
+        end
+      end
+
+      def job_id_from_key key
+        prefix, job_id = *key.split(':')
+        job_id
+      end
+
+      # try as hard as you can to find a free slot
+      def find_free_job_id
+        loop do
+          job_id = generate_id
+          return job_id unless db.contains? "#{QUEUE}:#{job_id}"
+          debug "#{job_id} is not free"
+        end
+      end
+
+      # Time in milliseconds and 4 digit random
+      # @note Maybe this is a stupid idea, but for now it kinda works :)
+      def generate_id
+        '%d%04d' % [(Time.now.to_f * 1000.0).to_i, Kernel.rand(1000)]
+      end
+
+      def serialize data
+        Marshal.dump(data)
+      end
+
+      def deserialize data
+        Marshal.load(data)
+      end
+
+      def count
+        result = 0
+        db.each do |k,v|
+          result += 1 if k.include?(QUEUE)
+        end
+        result
+      end
+
+      def count_waiting
+        result = 0
+        db.each do |k,v|
+          if k.include?(STARTED_WORKING_AT) && v == NOT_WORKING
+            result += 1
+          end
+        end
+        result
+      end
+
+      def count_working
+        result = 0
+        db.each do |k,v|
+          if k.include?(STARTED_WORKING_AT) && v != NOT_WORKING
+            result += 1
+          end
+        end
+        result
+      end
+
+      def each
+        # TODO optimize this (and others) using range queries
+        db.each do |k,v|
+          if k.include?(QUEUE)
+            job_id = job_id_from_key(k)
+            yield(id: job_id,
+                  queue: v,
+                  payload: find_payload(job_id),
+                  created_at: find_created_at(job_id),
+                  started_working_at: find_started_working_at(job_id))
+          end
+        end
+      end
+    end
+
+    attr_reader :db
+    attr_reader :jobs
+
+    def initialize config
+      @db = ::LevelDB::DB.new config
+      @jobs = JobsCollection.new(db)
+    end
+
+    # @param [EQ::Job] job
+    def push job
+      if job.unique? && jobs.exists?(job)
+        false
+      else
+        jobs.push job
+      end
+    end
+
+    def reserve
+      if job_id = jobs.first_waiting
+        jobs.start_working job_id
+        EQ::Job.new job_id, jobs.find_queue(job_id), jobs.find_payload(job_id)
+      end
+    end
+
+    def pop job_id
+      jobs.delete job_id
+    end
+
+    def requeue_timed_out_jobs
+      requeued = 0
+      jobs.working_iterator do |job_id, started_working_at|
+        # older than x
+        if started_working_at <= (Time.now - EQ.config.job_timeout)
+          jobs.stop_working job_id
+          requeued += 1
+        end
+      end
+      requeued
+    end
+
+    def clear
+      db.each{|k,v| db.delete k}
+    end
+
+    def count name=nil
+      case name
+      when :waiting
+        jobs.count_waiting
+      when :working
+        jobs.count_working
+      else
+        jobs.count
+      end
+    end
+
+    def iterator
+      jobs.each do |job|
+        yield job
+      end
+    end
+  end
+end

data/lib/eq-queueing/backends/sequel.rb

@@ -19,10 +19,17 @@ module EQ::Queueing::Backends
       create_table_if_not_exists!
     end
 
-    # @param [#to_sequel_block] payload
+    # @param [EQ::Job] payload
     # @return [Fixnum] id of the job
-    def push payload
-      jobs.insert payload: payload.to_sequel_blob, created_at: Time.now
+    def push eq_job
+      job = {queue: eq_job.queue}
+      job[:payload] = Marshal.dump(eq_job.payload).to_sequel_blob unless eq_job.payload.nil?
+      if eq_job.unique? && jobs.where(job).count > 0
+        false
+      else
+        job[:created_at] = Time.now
+        jobs.insert job
+      end
     rescue ::Sequel::DatabaseError => e
       retry if on_error e
     end
@@ -34,10 +41,11 @@ module EQ::Queueing::Backends
     # @return [Array<Fixnum, String>] job data consisting of id and payload
     def reserve
       db.transaction do
-        if job = waiting.order(:id.asc).limit(1).first
+        if job = waiting.order(:id).last # asc
           job[:started_working_at] = Time.now
           update_job!(job)
-          [job[:id], job[:payload]]
+          payload = job[:payload].nil? ? nil : Marshal.load(job[:payload])
+          EQ::Job.new(job[:id], job[:queue], payload)
         end
       end
     rescue ::Sequel::DatabaseError => e
@@ -85,22 +93,30 @@ module EQ::Queueing::Backends
     # this re-enqueues jobs that timed out
     # @return [Fixnum] number of jobs that were re-enqueued
     def requeue_timed_out_jobs
-      # 10 seconds ago
+      # older than x
       jobs.where{started_working_at <= (Time.now - EQ.config.job_timeout)}\
           .update(started_working_at: nil)
     end
 
-    # statistics:
-    #   - #job_count
-    #   - #working_count
-    #   - #waiting_count
-    %w[ job working waiting ].each do |stats_name|
-      define_method "#{stats_name}_count" do
-        begin
-          send(stats_name).send(:count)
-        rescue ::Sequel::DatabaseError => e
-          retry if on_error e
-        end
+    def count name=nil
+      case name
+      when :waiting
+        waiting.count
+      when :working
+        working.count
+      else
+        jobs.count
+      end
+    end
+
+    def clear
+      jobs.delete
+    end
+
+    def iterator
+      jobs.each do |job|
+        job[:payload] = Marshal.load(job[:payload]) if job[:payload]
+        yield job
       end
     end
 
@@ -116,6 +132,7 @@ module EQ::Queueing::Backends
     def create_table_if_not_exists!
       db.create_table? TABLE_NAME do
         primary_key :id
+        String :queue
         Timestamp :created_at
         Timestamp :started_working_at
         Blob :payload

data/lib/eq-queueing/queue.rb

@@ -5,15 +5,10 @@ module EQ::Queueing
   # furthermore this class adds some functionality to serialize / deserialze
   # using the Job class
   class Queue
+    extend SingleForwardable
+
     include Celluloid
     include EQ::Logging
-
-    %w[ job_count waiting_count working_count waiting working ].each do |method_name|
-      define_method method_name do
-        queue.send(method_name)
-      end
-    end
-    alias :size :job_count
     
     # @param [Object] queue_backend
     def initialize queue_backend
@@ -22,33 +17,44 @@ module EQ::Queueing
 
     # @param [Array<Class, *payload>] unserialized_payload
     # @return [Fixnum] job_id 
-    def push *unserialized_payload
-      debug "enqueing #{unserialized_payload.inspect} ..."
-      queue.push EQ::Job.dump(unserialized_payload)
+    def push job_class, *job_payload
+      debug "enqueing #{job_payload.inspect} ..."
+      queue.push EQ::Job.new(nil, job_class, job_payload)
     end
 
     # @return [EQ::Job, nilClass] job instance
     def reserve
       requeue_timed_out_jobs
-      if serialized_job = queue.reserve
-        job_id, serialized_payload = *serialized_job
-        job = EQ::Job.load job_id, serialized_payload
+      if job = queue.reserve
         debug "dequeud #{job.inspect}"
         job
       end
     end
 
-    # @return [TrueClass, FalseClass]
-    def pop job_id
-      queue.pop job_id
+    # 
+    # TODO #pop method: shall we add a check, when the job is worked on, if we are the worker?
+    # 
+    def pop *args
+      queue.pop *args
+    end
+
+    def requeue_timed_out_jobs; queue.requeue_timed_out_jobs; end
+
+    def jobs; queue.jobs; end
+    def working; queue.working; end
+    def waiting; queue.waiting; end
+    def count name=nil; queue.count name; end
+
+    def iterator &block
+      queue.iterator &block
     end
 
-    # re-enqueues jobs that timed out
-    def requeue_timed_out_jobs
-      queue.requeue_timed_out_jobs
+    def clear
+      queue.clear
     end
 
-    attr_reader :queue
+  private
 
+    def queue; @queue; end
   end
 end

data/lib/eq-scheduling.rb

@@ -0,0 +1,33 @@
+require File.join(File.dirname(__FILE__), 'eq')
+
+require 'clockwork'
+Clockwork.handler do |job|
+  EQ.push job if EQ.queue
+end
+
+require File.join(File.dirname(__FILE__), 'eq-scheduling', 'scheduler')
+
+module EQ::Scheduling
+  module_function
+
+  EQ_SCHEDULER = :_eq_scheduler
+
+  def boot
+    EQ::Scheduling::Scheduler.supervise_as EQ_SCHEDULER, EQ.config
+  end
+
+  def shutdown
+    scheduler.terminate! if scheduler
+  end
+
+  def scheduler
+    Celluloid::Actor[EQ_SCHEDULER]
+  end
+
+  def events
+    Clockwork.class_variable_get('@@events').map do |event|
+      [ event.job,
+        event.instance_variable_get('@period') ]  
+    end
+  end
+end

data/lib/eq-scheduling/scheduler.rb

@@ -0,0 +1,19 @@
+module EQ::Scheduling
+  class Scheduler
+    include EQ::Logging
+    include Celluloid
+
+    def initialize config
+      clockwork!
+    end
+
+    def clockwork
+      debug 'scheduler running'
+      loop do
+        Clockwork.tick
+        sleep(Clockwork.config[:sleep_timeout])
+        return unless Actor.current.alive?
+      end
+    end
+  end
+end

data/lib/eq-web.rb

@@ -0,0 +1,5 @@
+module EQ
+  module Web
+    
+  end
+end