jobster 1.0.1 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b62587aac678545a75383e10bb98737ec8ddc0b2
-  data.tar.gz: 83f880bacae8180651f0cbeae92fee0f42937560
+  metadata.gz: 30f6aef2178e0d7768841444674349210d567b03
+  data.tar.gz: f0d28eb0e772a1ec33b28bb9fc108feaf6521b45
 SHA512:
-  metadata.gz: cd0fece9c0900cb682360535585e6731441194afa46c9605c98f4befedf2aef3b09efe6090261cee9841d356bfe82e32b76c297e2b10e2be0bbba468b23ef775
-  data.tar.gz: e1dc4e0823f5b1e1cc35d861ebdbcd00b8d912357c6ea01436f8667bbf79bef8b63192af80cebd1a038c7a406a93838836d670a4ec31159b7d0d508ad5a8d83f
+  metadata.gz: 2d58fc83c4df5ed65e71c1e6fe54d0013cee2d567cf26b48d4def603a4f749db523365ee03d2d596fd54e72207d765b0987526f47df7781d0d3736206bed1825
+  data.tar.gz: 335bae815fea0868611fa84740a1ef2b870aca0a5174c3893d2e6cd34bd1a6d0ef9166375c17112a1828f1f37201ee3153c0a702c024f610a8e53830174137ac

data/lib/jobster.rb

@@ -2,46 +2,45 @@ require 'bunny'
 require 'jobster/job'
 require 'jobster/worker'
 require 'jobster/version'
+require 'jobster/config'
 
 module Jobster
 
-  class << self
+  def self.config
+    @config ||= Config.new
+  end
 
-    def bunny
-      @bunny ||= begin
-        connection = Bunny.new
-        connection.start
-        connection
-      end
-    end
-    attr_writer :bunny
+  def self.configure(&block)
+    block.call(self.config)
+  end
 
-    def logger
-      @logger ||= Logger.new(STDOUT)
-    end
-    attr_writer :logger
+  def self.channel
+    @channel ||= config.bunny.create_channel(nil, config.worker_threads)
+  end
 
-    def ttl
-      @ttl ||= 60 * 60 * 10
-    end
-    attr_writer :ttl
+  def self.exchange
+    @exchange ||= channel.exchange(config.exchange_name, :type => :direct, :durable => true, :auto_delete => false)
+  end
 
-    def queue_prefix
-      @queue_prefix ||= "jobster"
-    end
-    attr_writer :queue_prefix
+  def self.delay_exchange
+    @delay_exchange ||= channel.exchange(config.delay_exchange_name, :type => :fanout, :durable => true, :auto_delete => false)
+  end
 
-    def channel
-      @channel ||= bunny.create_channel
+  def self.delay_queue
+    @delay_queue ||= begin
+      queue = channel.queue(config.delay_queue_name, :durable => true, :auto_delete => false, :arguments => {'x-dead-letter-exchange' => config.exchange_name})
+      queue.bind(delay_exchange)
+      queue
     end
+  end
 
-    def queue(name)
-      @queues ||= {}
-      @queues[name] ||= begin
-        channel.queue("#{queue_prefix}-#{name}", :durable => true, :arguments => {'x-message-ttl' => self.ttl})
-      end
+  def self.queue(name)
+    @queues ||= {}
+    @queues[name] ||= begin
+      queue = channel.queue("#{config.queue_name_prefix}-#{name}", :durable => true, :auto_delete => false)
+      queue.bind(exchange, :routing_key => name)
+      queue
     end
-
   end
 
 end

data/lib/jobster/config.rb

@@ -0,0 +1,61 @@
+module Jobster
+  class Config
+
+    def bunny
+      @bunny ||= begin
+        connection = Bunny.new
+        connection.start
+        connection
+      end
+    end
+    attr_writer :bunny
+
+    def logger
+      @logger ||= Logger.new(STDOUT)
+    end
+    attr_writer :logger
+
+    def exchange_name
+      @exchange_name || "jobster"
+    end
+    attr_writer :exchange_name
+
+    def delay_exchange_name
+      @delay_exchange_name || "#{self.exchange_name}-delay-exch"
+    end
+    attr_writer :delay_exchange_name
+
+    def delay_queue_name
+      @delay_queue_name || "#{self.exchange_name}-delay-queue"
+    end
+    attr_writer :delay_queue_name
+
+    def queue_name_prefix
+      @queue_name_prefix || "#{self.exchange_name}-queue"
+    end
+    attr_writer :queue_name_prefix
+
+    def worker_threads
+      @worker_threads || 1
+    end
+    attr_writer :worker_threads
+
+    def worker_callbacks
+      @worker_callbacks ||= {}
+    end
+
+    def worker_callback(name, &block)
+      worker_callbacks[name.to_sym] ||= []
+      worker_callbacks[name.to_sym] << block
+    end
+
+    def worker_error_handlers
+      @worker_error_handlers ||= []
+    end
+
+    def worker_error_handler(&block)
+      worker_error_handlers << block
+    end
+
+  end
+end

data/lib/jobster/job.rb

@@ -16,19 +16,34 @@ module Jobster
     end
 
     def log(text)
-      Jobster.logger.info "[#{@id}] #{text}"
+      Jobster.config.logger.info "[#{@id}] #{text}"
     end
 
-    def self.queue(queue = {}, params = {})
+    def self.queue(queue = {}, params = {}, &block)
+      queue, params = parse_args_for_queue(queue, params)
+      self.queue_job(Jobster.exchange, queue, self.name, params, &block)
+    end
+
+    def self.queue_with_delay(delay, queue = {}, params = {}, &block)
+      queue, params = parse_args_for_queue(queue, params)
+      self.queue_job(Jobster.delay_exchange, queue, self.name, params, :ttl => delay, &block)
+    end
 
-      if queue.is_a?(Hash)
-        params = queue
-        queue = :main
-      end
+    def self.parse_args_for_queue(queue, params)
+      queue.is_a?(Hash) ? [:main, queue] : [queue, params]
+    end
 
+    def self.queue_job(exchange, queue_name, class_name, params, options = {}, &block)
       job_id = SecureRandom.uuid[0,8]
-      job_payload = {'params' => params, 'class_name' => self.name, 'id' => job_id, 'queue' => queue}
-      Jobster.queue(queue).publish(job_payload.to_json, :persistent => false)
+      job_payload = {'params' => params, 'class_name' => class_name, 'id' => job_id, 'queue' => queue_name}
+      publish_opts = {}
+      publish_opts[:persistent] = true
+      publish_opts[:routing_key] = queue_name
+      publish_opts[:expiration] = options[:ttl] * 1000 if options[:ttl]
+      block.call(job_id, publish_opts) if block_given?
+      a = exchange.publish(job_payload.to_json, publish_opts)
+      when_string = (options[:ttl] ? "in #{options[:ttl]}s" : "immediately")
+      Jobster.config.logger.info "[#{job_id}] \e[34m#{class_name}\e[0m queued to run #{when_string} on #{queue_name} queue"
       job_id
     end
 

data/lib/jobster/version.rb

@@ -1,3 +1,3 @@
 module Jobster
-  VERSION = '1.0.1'
+  VERSION = '1.2.0'
 end

data/lib/jobster/worker.rb

@@ -4,30 +4,43 @@ module Jobster
     def initialize(queues = nil)
       @initial_queues = queues || self.class.queues || [:main]
       @active_queues = {}
+      @running_jobs = []
       @process_name = $0
+      set_process_name
+    end
+
+    def set_process_name
+      prefix = @process_name.to_s
+      prefix += " [exiting]" if @exit
+      if @running_jobs.empty?
+        $0 = "#{prefix} (idle)"
+      else
+        $0 = "#{prefix} (running #{@running_jobs.join(', ')})"
+      end
     end
 
     def work
-      logger.info "Jobster worker started"
-      self.class.run_callbacks(:after_start)
+      logger.info "Jobster worker started (#{Jobster.config.worker_threads} thread(s))"
+      run_callbacks :after_start
 
-      @running_job = false
-      Signal.trap("INT")  { @exit = true }
-      Signal.trap("TERM") { @exit = true }
+      Jobster.delay_queue # Declare it
 
-      Jobster.channel.prefetch(1)
+      Signal.trap("INT")  { @exit = true; set_process_name }
+      Signal.trap("TERM") { @exit = true; set_process_name }
+
+      Jobster.channel.prefetch(Jobster.config.worker_threads)
       @initial_queues.uniq.each { |queue | join_queue(queue) }
 
       exit_checks = 0
       loop do
-        if @exit && @running_job == false
-          logger.info "Exiting immediately because no job running"
-          self.class.run_callbacks(:before_quit, :immediate)
+        if @exit && @running_jobs.empty?
+          logger.info "Exiting immediately because no jobs running"
+          run_callbacks :before_quit, :immediate
           exit 0
         elsif @exit
           if exit_checks >= 300
             logger.info "Job did not finish in a timely manner. Exiting"
-            self.class.run_callbacks(:before_quit, :timeout)
+            run_callbacks :before_quit, :timeout
             exit 0
           end
           if exit_checks == 0
@@ -41,43 +54,49 @@ module Jobster
       end
     end
 
+    def perform_job(class_name, params = {}, id = nil)
+      id ||= SecureRandom.uuid[0,8]
+      start_time = Time.now
+      exception = nil
+      logger.info "[#{id}] Started processing \e[34m#{class_name}\e[0m job"
+      begin
+        klass = Object.const_get(class_name).new(id, params)
+        run_callbacks :before_job, klass
+        klass.perform
+      rescue Job::Abort => e
+        exception = e
+        logger.info "[#{id}] Job aborted (#{e.message})"
+      rescue => e
+        exception = e
+        logger.warn "[#{id}] \e[31m#{e.class}: #{e.message}\e[0m"
+        e.backtrace.each do |line|
+          logger.warn "[#{id}]    " + line
+        end
+        Jobster.config.worker_error_handlers.each { |handler| handler.call(e, klass) }
+      ensure
+        run_callbacks :after_job, klass, exception
+        logger.info "[#{id}] Finished processing \e[34m#{class_name}\e[0m job in #{Time.now - start_time}s"
+      end
+    end
+
     private
 
-    def receive_job(delivery_info, properties, body)
-      @running_job = true
+    def receive_job(properties, body)
       begin
         message = JSON.parse(body) rescue nil
         if message && message['class_name']
-          start_time = Time.now
-          $0 = "#{@process_name} (running #{message['class_name']})"
           Thread.current[:job_id] = message['id']
-          logger.info "[#{message['id']}] Started processing \e[34m#{message['class_name']}\e[0m job"
-          begin
-            klass = Object.const_get(message['class_name']).new(message['id'], message['params'])
-            self.class.run_callbacks(:before_job, klass)
-            klass.perform
-            self.class.run_callbacks(:after_job, klass)
-          rescue Job::Abort => e
-            logger.info "[#{message['id']}] Job aborted (#{e.message})"
-          rescue => e
-            logger.warn "[#{message['id']}] \e[31m#{e.class}: #{e.message}\e[0m"
-            e.backtrace.each do |line|
-              logger.warn "[#{message['id']}]    " + line
-            end
-            self.class.run_callbacks(:after_job, klass, e)
-            self.class.error_handlers.each { |handler| handler.call(e, klass) }
-          ensure
-            logger.info "[#{message['id']}] Finished processing \e[34m#{message['class_name']}\e[0m job in #{Time.now - start_time}s"
-          end
+          @running_jobs << message['id']
+          set_process_name
+          perform_job(message['class_name'], message['params'] || {}, message['id'])
         end
       ensure
         Thread.current[:job_id] = nil
-        $0 = @process_name
-        Jobster.channel.ack(delivery_info.delivery_tag)
-        @running_job = false
-        if @exit
-          logger.info "Exiting because a job has ended."
-          self.class.run_callbacks(:before_quit, :job_completed)
+        @running_jobs.delete(message['id']) if message['id']
+        set_process_name
+        if @exit && @running_jobs.empty?
+          logger.info "Exiting because all jobs have finished."
+          run_callbacks :before_quit, :job_completed
           exit 0
         end
       end
@@ -87,12 +106,16 @@ module Jobster
       if @active_queues[queue]
         logger.info "Attempted to join queue #{queue} but already joined."
       else
-        self.class.run_callbacks(:before_queue_join, queue)
+        run_callbacks :before_queue_join, queue
         consumer = Jobster.queue(queue).subscribe(:manual_ack => true) do |delivery_info, properties, body|
-          receive_job(delivery_info, properties, body)
+          begin
+            receive_job(properties, body)
+          ensure
+            Jobster.channel.ack(delivery_info.delivery_tag)
+          end
         end
         @active_queues[queue] = consumer
-        self.class.run_callbacks(:after_queue_join, queue, consumer)
+        run_callbacks :after_queue_join, queue, consumer
         logger.info "Joined \e[32m#{queue}\e[0m queue"
       end
     end
@@ -108,33 +131,16 @@ module Jobster
     end
 
     def logger
-      Jobster.logger
+      Jobster.config.logger
     end
 
     def self.queues
       @queues ||= [:main]
     end
 
-    def self.error_handlers
-      @error_handlers ||= []
-    end
-
-    def self.register_error_handler(&block)
-      error_handlers << block
-    end
-
-    def self.callbacks
-      @callbacks ||= {}
-    end
-
-    def self.add_callback(event, &block)
-      callbacks[event] ||= []
-      callbacks[event] << block
-    end
-
-    def self.run_callbacks(event, *args)
-      if callbacks[event]
-        callbacks[event].each do |callback|
+    def run_callbacks(event, *args)
+      if callbacks = Jobster.config.worker_callbacks[event]
+        callbacks.each do |callback|
           callback.call(*args)
         end
       end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: jobster
 version: !ruby/object:Gem::Version
-  version: 1.0.1
+  version: 1.2.0
 platform: ruby
 authors:
 - Adam Cooke
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-02-27 00:00:00.000000000 Z
+date: 2017-12-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bunny
@@ -40,6 +40,7 @@ extra_rdoc_files: []
 files:
 - bin/jobster
 - lib/jobster.rb
+- lib/jobster/config.rb
 - lib/jobster/job.rb
 - lib/jobster/version.rb
 - lib/jobster/worker.rb
@@ -63,9 +64,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 2.5.2
 signing_key: 
 specification_version: 4
 summary: A RabbitMQ-based job queueing system
 test_files: []
-has_rdoc: