jobster 0.0.0 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8b42cbde20bbcdd7c823f4d202590b81c6e5ed78
-  data.tar.gz: f3da2037dbb263d8f22f8293b7caa5362054732a
+  metadata.gz: 3f97863176fe30a7cef64b7fad42838ea0df3de1
+  data.tar.gz: 05d04d055613758d20d3be44f50379640b912263
 SHA512:
-  metadata.gz: 2c3622f0c889627374b070523cce325a9c2b6b34419b18b7ecebfe0f92205ec0d980c91d27d6301e15e7595ef5e050226a210fd314ce18bac4e0e8a80f2d7e11
-  data.tar.gz: f542c6eed046b828410597c2fb38537a2f33dcdf348e8e0a6717f9d3cc892a856548c9b6a22e39c32ea0e0ae8b5564e0a6df3f607dd91587ac44bbf3a81b96be
+  metadata.gz: 9ad5c5c3c8780ba623c8284b6ce6eeee482bbb24d655e6d327f771442ff2e8bd7bc540923bdc647ac0e9dc3ceb5d722adc823ceaf3992b86b166bfe3d7373532
+  data.tar.gz: 826bcfaadb5641153f9a1296400cfac9c91fa8d043e1b5aafa4d613b7064a0ba163da62037234c6116f22ad958bd660e652edfcc6c111bb1663d6dd5f93d63cc

data/bin/jobster

@@ -0,0 +1,36 @@
+#!/usr/bin/env ruby
+
+require 'optparse'
+require 'jobster'
+
+$stdout.sync = true
+$stderr.sync = true
+
+options = {}
+
+OptionParser.new do |opts|
+  opts.version = Jobster::VERSION
+  opts.banner = "Usage: jobster [options]"
+  opts.on("-c", "--config PATH", "The path to your jobster config file") do |config|
+    options[:config] = config
+  end
+  opts.on("-q", "--queues QUEUE1,QUEUE2", "Additional queues that you wish to subscribe to") do |queues|
+    queues.split(/,/).uniq.each do |queue|
+      Jobster::Worker.queues << queue
+    end
+  end
+
+end.parse!
+
+if options[:config]
+  if File.file?(options[:config])
+    file = File.expand_path(options[:config])
+    require file
+  else
+    puts "Jobster config file not found at #{options[:config]}"
+    exit 1
+  end
+end
+
+worker = Jobster::Worker.new
+worker.work

data/lib/jobster.rb

@@ -0,0 +1,47 @@
+require 'bunny'
+require 'jobster/job'
+require 'jobster/worker'
+require 'jobster/version'
+
+module Jobster
+
+  class << self
+
+    def bunny
+      @bunny ||= begin
+        connection = Bunny.new
+        connection.start
+        connection
+      end
+    end
+    attr_writer :bunny
+
+    def logger
+      @logger ||= Logger.new(STDOUT)
+    end
+    attr_writer :logger
+
+    def ttl
+      @ttl ||= 60 * 60 * 10
+    end
+    attr_writer :ttl
+
+    def queue_prefix
+      @queue_prefix ||= "jobster"
+    end
+    attr_writer :queue_prefix
+
+    def channel
+      @channel ||= bunny.create_channel
+    end
+
+    def queue(name)
+      @queues ||= {}
+      @queues[name] ||= begin
+        channel.queue("#{queue_prefix}-#{name}", :durable => true, :arguments => {'x-message-ttl' => self.ttl})
+      end
+    end
+
+  end
+
+end

data/lib/jobster/job.rb

@@ -0,0 +1,40 @@
+module Jobster
+  class Job
+
+    class Abort < StandardError; end
+
+    attr_reader :id
+    attr_reader :params
+
+    def initialize(id, params = {})
+      @id = id
+      @params = params.with_indifferent_access
+    end
+
+    def perform
+      # Override in child jobs
+    end
+
+    def log(text)
+      Jobster.logger.info "[#{@id}] #{text}"
+    end
+
+    def self.queue(queue = {}, params = {})
+
+      if queue.is_a?(Hash)
+        params = queue
+        queue = :main
+      end
+
+      job_id = SecureRandom.uuid[0,8]
+      job_payload = {'params' => params, 'class_name' => self.name, 'id' => job_id, 'queue' => queue}
+      Jobster.queue(queue).publish(job_payload.to_json, :persistent => false)
+      job_id
+    end
+
+    def self.perform(params = {})
+      new(nil, params).perform
+    end
+
+  end
+end

data/lib/jobster/version.rb

@@ -1,3 +1,3 @@
 module Jobster
-  VERSION = '0.0.0'
+  VERSION = '1.0.0'
 end

data/lib/jobster/worker.rb

@@ -0,0 +1,144 @@
+module Jobster
+  class Worker
+
+    def initialize(queues = nil)
+      @initial_queues = queues || self.class.queues || [:main]
+      @active_queues = {}
+      @process_name = $0
+    end
+
+    def work
+      logger.info "Jobster worker started"
+      self.class.run_callbacks(:after_start)
+
+      @running_job = false
+      Signal.trap("INT")  { @exit = true }
+      Signal.trap("TERM") { @exit = true }
+
+      Jobster.channel.prefetch(1)
+      @initial_queues.uniq.each { |queue | join_queue(queue) }
+
+      exit_checks = 0
+      loop do
+        if @exit && @running_job == false
+          logger.info "Exiting immediately because no job running"
+          self.class.run_callbacks(:before_quit, :immediate)
+          exit 0
+        elsif @exit
+          if exit_checks >= 300
+            logger.info "Job did not finish in a timely manner. Exiting"
+            self.class.run_callbacks(:before_quit, :timeout)
+            exit 0
+          end
+          if exit_checks == 0
+            logger.info "Exit requested but job is running. Waiting for job to finish."
+          end
+          sleep 5
+          exit_checks += 1
+        else
+          sleep 1
+        end
+      end
+    end
+
+    private
+
+    def receive_job(delivery_info, properties, body)
+      @running_job = true
+      begin
+        message = JSON.parse(body) rescue nil
+        if message && message['class_name']
+          start_time = Time.now
+          $0 = "#{@process_name} (running #{message['class_name']})"
+          Thread.current[:job_id] = message['id']
+          logger.info "[#{message['id']}] Started processing \e[34m#{message['class_name']}\e[0m job"
+          begin
+            klass = Object.const_get(message['class_name']).new(message['id'], message['params'])
+            self.class.run_callbacks(:before_job, klass)
+            klass.perform
+            self.class.run_callbacks(:after_job, klass)
+          rescue Job::Abort => e
+            logger.info "[#{message['id']}] Job aborted (#{e.message})"
+          rescue => e
+            logger.warn "[#{message['id']}] \e[31m#{e.class}: #{e.message}\e[0m"
+            e.backtrace.each do |line|
+              logger.warn "[#{message['id']}]    " + line
+            end
+            self.class.run_callbacks(:after_job, klass, e)
+            self.class.error_handlers.each { |handler| handler.call(e, klass) }
+          ensure
+            logger.info "[#{message['id']}] Finished processing \e[34m#{message['class_name']}\e[0m job in #{Time.now - start_time}s"
+          end
+        end
+      ensure
+        Thread.current[:job_id] = nil
+        $0 = @process_name
+        Jobster.channel.ack(delivery_info.delivery_tag)
+        @running_job = false
+        if @exit
+          logger.info "Exiting because a job has ended."
+          self.class.run_callbacks(:before_quit, :job_completed)
+          exit 0
+        end
+      end
+    end
+
+    def join_queue(queue)
+      if @active_queues[queue]
+        logger.info "Attempted to join queue #{queue} but already joined."
+      else
+        self.class.run_callbacks(:before_queue_join, queue)
+        consumer = Jobster.queue(queue).subscribe(:manual_ack => true) do |delivery_info, properties, body|
+          receive_job(delivery_info, properties, body)
+        end
+        @active_queues[queue] = consumer
+        self.class.run_callbacks(:after_queue_join, queue, consumer)
+        logger.info "Joined \e[32m#{queue}\e[0m queue"
+      end
+    end
+
+    def leave_queue(queue)
+      if consumer = @active_queues[queue]
+        consumer.cancel
+        @active_queues.delete(queue)
+        logger.info "Left \e[32m#{queue}\e[0m queue"
+      else
+        logger.info "Not joined #{queue} so cannot leave"
+      end
+    end
+
+    def logger
+      Jobster.logger
+    end
+
+    def self.queues
+      @queues ||= [:main]
+    end
+
+    def self.error_handlers
+      @error_handlers ||= []
+    end
+
+    def self.register_error_handler(&block)
+      error_handlers << block
+    end
+
+    def self.callbacks
+      @callbacks ||= {}
+    end
+
+    def self.add_callback(event, &block)
+      callbacks[event] ||= []
+      callbacks[event] << block
+    end
+
+    def self.run_callbacks(event, *args)
+      if callbacks[event]
+        callbacks[event].each do |callback|
+          callback.call(*args)
+        end
+      end
+    end
+
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: jobster
 version: !ruby/object:Gem::Version
-  version: 0.0.0
+  version: 1.0.0
 platform: ruby
 authors:
 - Adam Cooke
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-02-22 00:00:00.000000000 Z
+date: 2017-02-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bunny
@@ -33,11 +33,16 @@ dependencies:
 description: A RabbitMQ-based job queueing system
 email:
 - me@adamcooke.io
-executables: []
+executables:
+- jobster
 extensions: []
 extra_rdoc_files: []
 files:
+- bin/jobster
+- lib/jobster.rb
+- lib/jobster/job.rb
 - lib/jobster/version.rb
+- lib/jobster/worker.rb
 homepage: https://github.com/adamcooke/jobster
 licenses:
 - MIT
@@ -58,7 +63,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.2
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: A RabbitMQ-based job queueing system