tennis-jobs 0.3.1 → 0.4.0

data/lib/tennis/{serializer/generic.rb → backend/serializer.rb}

@@ -1,12 +1,13 @@
 require "json"
 
 module Tennis
-  module Serializer
-    class Generic
+  module Backend
+    class Serializer
 
       RECOGNIZED_TYPES = {
-        active_record: "active_record".freeze,
-        class:         "class".freeze,
+        findable: "findable".freeze,
+        class:    "class".freeze,
+        job:      "job".freeze,
       }.freeze
 
       def load(message)
@@ -30,12 +31,18 @@ module Tennis
               _type: RECOGNIZED_TYPES[:class],
               _class: object.to_s
             }
-          when :active_record
+          when :findable
             {
-              _type: RECOGNIZED_TYPES[:active_record],
+              _type: RECOGNIZED_TYPES[:findable],
               _class: object.class.to_s,
               _id: object.id,
             }
+          when :job
+            {
+              _type: RECOGNIZED_TYPES[:job],
+              _class: object.class.to_s,
+              _dump: object.job_dump,
+            }
           else
             fail "Unexpected type: #{type} when visiting object"
           end
@@ -49,9 +56,12 @@ module Tennis
             object
           when :class
             Object.const_get(object["_class"])
-          when :active_record
+          when :findable
             klass = Object.const_get(object["_class"])
             klass.find(object["_id"])
+          when :job
+            klass = Object.const_get(object["_class"])
+            klass.job_load(object["_dump"])
           else
             fail "Unexpected type: #{type} when visiting object"
           end
@@ -65,8 +75,10 @@ module Tennis
       def visit_any(object, block)
         if object.kind_of?(Array)
           visit_array(object, block)
-        elsif is_active_record?(object)
-          block.call(:active_record, object)
+        elsif is_job?(object)
+          block.call(:job, object)
+        elsif is_findable?(object)
+          block.call(:findable, object)
         elsif is_class?(object)
           block.call(:class, object)
         elsif object.kind_of?(Hash)
@@ -86,14 +98,19 @@ module Tennis
         end
       end
 
-      def is_active_record?(object)
-        object.respond_to?(:attributes) ||
-        object.is_a?(Hash) && object["_type"] == RECOGNIZED_TYPES[:active_record]
+      def is_findable?(object)
+        (object.class.respond_to?(:find) && object.respond_to?(:id)) ||
+        (object.is_a?(Hash) && object["_type"] == RECOGNIZED_TYPES[:findable])
       end
 
       def is_class?(object)
         object.is_a?(Class) ||
-        object.is_a?(Hash) && object["_type"] == RECOGNIZED_TYPES[:class]
+        (object.is_a?(Hash) && object["_type"] == RECOGNIZED_TYPES[:class])
+      end
+
+      def is_job?(object)
+        object.is_a?(Tennis::Job) ||
+        (object.is_a?(Hash) && object["_type"] == RECOGNIZED_TYPES[:job])
       end
 
     end

data/lib/tennis/backend/task.rb

@@ -0,0 +1,36 @@
+module Tennis
+  module Backend
+    class Task
+
+      attr_reader :task_id, :job, :method, :args, :meta
+      attr_accessor :worker
+
+      def initialize(backend, task_id, job, method, args, meta = {})
+        @backend, @task_id, @acked = backend, task_id, false
+        @job, @method, @args = job, method, args
+        @meta = meta
+      end
+
+      def execute
+        @job.__send__(@method, *@args)
+      end
+
+      def ack
+        return if acked?
+        @backend.ack(self)
+        @acked = true
+      end
+
+      def requeue
+        return if acked?
+        @backend.requeue(self)
+        @acked = true
+      end
+
+      def acked?
+        @acked
+      end
+
+    end
+  end
+end

data/lib/tennis/cli.rb

@@ -1,32 +1,30 @@
-require "yaml"
 require "optparse"
-require "sneakers"
-require "sneakers/runner"
+
+require "tennis/launcher"
 
 module Tennis
   class CLI
 
     DEFAULT_OPTIONS = {
-      config: "./tennis.yml",
+      concurrency: 2,
+      job_class_names: [],
     }.freeze
 
     def self.start
       options = DEFAULT_OPTIONS.dup
       OptionParser.new do |opts|
-        opts.banner = "Usage: tennis [options] group"
-        opts.on("-c", "--config FILE", "Set the config file") do |file|
-          options[:config] = file
+        opts.banner = "Usage: tennis [options]"
+        opts.on("-j", "--jobs JOBS", "List of the job classes to handle") do |jobs|
+          options[:job_class_names] = jobs.split(",")
+        end
+        opts.on("-c", "--concurrency COUNT", "The number of concurrent jobs") do |concurrency|
+          options[:concurrency] = concurrency.to_i
         end
         opts.on("-r", "--require PATH", "Require files before starting") do |path|
           options[:require] ||= []
           options[:require] << path
         end
-        opts.on("-x", "--execute CODE", "Execute code before starting") do |code|
-          options[:execute] ||= []
-          options[:execute] << code
-        end
       end.parse!
-      options[:group] = ARGV.first
       new(options).start
     end
 
@@ -35,68 +33,30 @@ module Tennis
     end
 
     def start
-      do_require
-      execute_code
-      configure_tennis
-      start_group
+      require_paths
+      start_launcher
     end
 
     private
 
-    def do_require
+    def require_paths
       return unless requires = @options[:require]
       requires.each { |path| require path } if @options[:require]
     end
 
-    def execute_code
-      return unless codes = @options[:execute]
-      codes.each { |code| eval code }
-    end
-
-    def configure_tennis
-      Tennis.configure do |config|
-        config.async = true
-        config.exchange = group["exchange"]
-        config.workers = group["workers"].to_i
-        config.logger = Logger.new(STDOUT)
-        config.logger.level = Logger::WARN
-        config.sneakers_options = sneakers_options
-      end
+    def start_launcher
+      raise "You must specify at least one job class" if job_classes.empty?
+      Launcher.new({
+        job_classes: job_classes,
+        concurrency: @options[:concurrency]
+      }).start
     end
 
-    def sneakers_options
-      classes.map(&:options).each_with_object({}) do |options, all_options|
-        merge_options(all_options, options)
-      end
-    end
-
-    def merge_options(target, options)
-      options.each do |name, value|
-        if target[name].nil?
-          target[name] = value
-        elsif target[name] != value
-          fail "Workers shouldn't have different '#{name}' options"
-        end
-      end
-    end
-
-    def start_group
-      Sneakers::Runner.new(classes.map(&:worker)).run
-    end
-
-    def classes
-      @classes ||= group["classes"].map do |name|
+    def job_classes
+      @job_classes ||= @options[:job_class_names].map do |name|
         Object.const_get(name)
       end
     end
 
-    def group
-      @group ||= config[@options[:group]]
-    end
-
-    def config
-      YAML.load_file(@options[:config])
-    end
-
   end
 end

data/lib/tennis/configuration.rb

@@ -1,14 +1,14 @@
+require "logger"
+require "celluloid"
+
 module Tennis
   class Configuration
     DEFAULT = {
       async: true,
-      exchange: "tennis",
-      workers: 4,
-      logger: STDOUT,
-      sneakers_options: {},
+      logger: Logger.new(STDOUT),
     }.freeze
 
-    attr_accessor :async, :exchange, :workers, :logger, :sneakers_options
+    attr_accessor :async, :logger, :backend
 
     def initialize(opts = {})
       DEFAULT.merge(opts).each do |name, value|
@@ -17,11 +17,10 @@ module Tennis
     end
 
     def finalize!
-      Sneakers.configure({
-        exchange: exchange,
-        workers: workers,
-        log: logger,
-      }.merge(sneakers_options))
+      raise "You must specify a backend during the configuration" unless backend
+
+      # Set the celluloid logger.
+      Celluloid.logger = logger
     end
 
   end

data/lib/tennis/exceptions.rb

@@ -0,0 +1,3 @@
+module Tennis
+  class Shutdown < StandardError ; end
+end

data/lib/tennis/fetcher.rb

@@ -0,0 +1,36 @@
+require "tennis/actor"
+
+module Tennis
+  class Fetcher
+    include Actor
+
+    attr_reader :worker_pool
+
+    def initialize(worker_pool, options)
+      @job_classes = options[:job_classes]
+      @worker_pool = worker_pool
+      @backend = Tennis.config.backend
+      @done = false
+    end
+
+    def fetch
+      return if done?
+      if task = @backend.receive(job_classes: @job_classes)
+        worker_pool.async.work(task)
+      else
+        async.fetch
+      end
+    end
+
+    def done!
+      @done = true
+    end
+
+    private
+
+    def done?
+      @done
+    end
+
+  end
+end

data/lib/tennis/job.rb

@@ -0,0 +1,30 @@
+require "tennis/action"
+require "tennis/worker"
+
+module Tennis
+  module Job
+    def self.included(base)
+      base.extend(ClassMethods)
+    end
+
+    # Return a proxy object that will enqueue method calls into
+    # the Tennis's backend.
+    def async
+      Action.new(self)
+    end
+
+    # Dump a Job instance into a simple hash.
+    def job_dump
+      raise NotImplementedError
+    end
+
+    module ClassMethods
+
+      # Build a Job instance from a simple hash.
+      def job_load(hash)
+        raise NotImplementedError
+      end
+
+    end
+  end
+end

data/lib/tennis/launcher.rb

@@ -0,0 +1,38 @@
+require "celluloid/condition"
+
+require "tennis/actor"
+require "tennis/fetcher"
+require "tennis/worker_pool"
+
+module Tennis
+  class Launcher
+    include Actor
+
+    attr_reader :worker_pool, :fetcher
+
+    def initialize(options)
+      @stop_condition = Celluloid::Condition.new
+      @worker_pool = WorkerPool.new_link(@stop_condition, options)
+      @fetcher = Fetcher.new_link(worker_pool, options)
+      @worker_pool.fetcher = @fetcher
+    end
+
+    def start
+      worker_pool.async.start
+    end
+
+    def stop
+      # Stop fetching
+      fetcher.done!
+
+      # Gracefully stop the workers that are still working
+      worker_pool.async.stop
+      @stop_condition.wait
+
+      # Terminate the two actors
+      worker_pool.terminate if worker_pool.alive?
+      fetcher.terminate if fetcher.alive?
+    end
+
+  end
+end

data/lib/tennis/worker.rb

@@ -0,0 +1,46 @@
+require "tennis/actor"
+require "tennis/exceptions"
+
+module Tennis
+  class Worker
+    include Actor
+
+    attr_accessor :worker_id
+
+    def initialize(pool)
+      @pool = pool
+    end
+
+    def work(task)
+      # Send the current working thread to the pool.
+      register_working_thread
+
+      ack = true
+      begin
+        task.execute
+      rescue Shutdown
+        ack = false
+        raise
+      rescue Exception => exception
+        # TODO: add an error handler on the job's class
+        raise
+      ensure
+        task.ack if ack
+      end
+
+      # Tell the pool that we've successfully done the job.
+      notifies_work_done(task)
+    end
+
+    private
+
+    def register_working_thread
+      @pool.async.register_thread(worker_id, Thread.current)
+    end
+
+    def notifies_work_done(task)
+      @pool.async.work_done(task)
+    end
+
+  end
+end

data/lib/tennis/worker_pool.rb

@@ -0,0 +1,114 @@
+require "thread"
+
+require "tennis/actor"
+require "tennis/exceptions"
+require "tennis/worker"
+
+module Tennis
+  class WorkerPool
+    include Actor
+
+    trap_exit :worker_died
+
+    attr_accessor :fetcher
+
+    def initialize(stop_condition, options)
+      @stop_condition = stop_condition
+      @size = options[:concurrency]
+      @pending_tasks = []
+      @threads = {}
+      @workers = Queue.new
+    end
+
+    def start
+      @size.times { start_worker }
+    end
+
+    def stop(timeout: 30)
+      done!
+
+      if @pending_tasks.empty?
+        shutdown
+      elsif timeout
+        plan_hard_shutdown timeout
+      end
+    end
+
+    def work(task)
+      # Do not accept new tasks if done.
+      return task.requeue if done?
+
+      @pending_tasks << task
+      worker = @workers.pop(true)
+      task.worker = worker
+      worker.async.work(task)
+    end
+
+    def work_done(task)
+      @pending_tasks.delete(task)
+      @threads.delete(task.worker.object_id)
+      ready(task.worker) if task.worker.alive?
+
+      # If done and there is no more pending tasks, we can shutdown. It also
+      # means that every workers are in que @workers queue.
+      shutdown if done? && @pending_tasks.empty?
+    end
+
+    def register_thread(worker_id, thread)
+      @threads[worker_id] = thread
+    end
+
+    def worker_died(worker, reason)
+      @threads.delete(worker.object_id)
+      @pending_tasks.delete_if { |task| task.worker == worker }
+      start_worker unless reason.is_a?(Shutdown)
+    end
+
+    private
+
+    def done!
+      @done = true
+    end
+
+    def done?
+      @done
+    end
+
+    def plan_hard_shutdown(timeout)
+      after(timeout) do
+        @pending_tasks.each do |task|
+          worker = task.worker
+          thread = @threads.delete(worker.object_id)
+          thread.raise(Shutdown) if worker.alive?
+          task.requeue
+        end
+        @pending_tasks.clear
+
+        shutdown
+      end
+    end
+
+    def shutdown
+      # Terminate the worker actors.
+      @workers.size.times do
+        worker = @workers.pop
+        worker.terminate if worker.alive?
+      end
+
+      # Signal the launcher that we're done processing jobs
+      @stop_condition.signal
+    end
+
+    def start_worker
+      worker = Worker.new_link(current_actor)
+      worker.worker_id = worker.object_id
+      ready(worker)
+    end
+
+    def ready(worker)
+      @workers << worker
+      fetcher.async.fetch
+    end
+
+  end
+end