worker-army 0.2.1 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3333bafc54ba37dcac68ad1e8ea338ee18e26512
-  data.tar.gz: 273ea0c00e78379fa78d6b7808c34d08898d20c5
+  metadata.gz: 8091229fb2c866bd2995c9a60ec791e43b4668fb
+  data.tar.gz: 9ff89f626a9d0f0ec49f1ff026037c0b474dc23d
 SHA512:
-  metadata.gz: 62bd6433faf7aa220080946c80092e2e1f3abac80f9e716830ca43aeaada2cfca07611881b98e3ea3f577a751e4c277cd8142991faf8d2e2d84c50c184bff151
-  data.tar.gz: 46a3af7c5a75245bb6ebbd9b281317655bb012e589b15d9fae33fd99e1c229bd33b7471e06cf8f27a536b3bd10e1c9dd2119984a08e135117951dc945187c66d
+  metadata.gz: d6f235480c8cb018de1759ee8f4da0b0d01a274ada3146892f4a27193e17105dadc40b7d15b33beaa175514820aebe35d92ee775477ef0b810f098db4c3b9dd8
+  data.tar.gz: 3dd6d94b5f2080d6187e649c8c27aa37201584ea7ea584db38338ab37af5daf31e921ce4ddd616d17f5e7a191feeb2cadd65ef159f32eb9a89419cc1e3284a32

data/Gemfile

@@ -11,8 +11,6 @@ gem 'unicorn'
 gem "foreman"
 gem "jeweler", "~> 2.0.1"
 
-# Add dependencies to develop your gem here.
-# Include everything needed to run rake, tests, features, etc.
 group :development do
   gem "bundler", "~> 1.6.2"
 end

data/README.rdoc

@@ -1,6 +1,6 @@
 = worker-army
 
-Description goes here.
+Simple redis based worker queue with a HTTP/Rest interface.
 
 == Contributing to worker-army
  
@@ -14,6 +14,5 @@ Description goes here.
 
 == Copyright
 
-Copyright (c) 2013 Oliver Kiessler. See LICENSE.txt for
+Copyright (c) 2013-2014 Oliver Kiessler. See LICENSE.txt for
 further details.
-

data/Rakefile

@@ -13,7 +13,6 @@ require 'rake'
 
 require 'jeweler'
 Jeweler::Tasks.new do |gem|
-  # gem is a Gem::Specification... see http://docs.rubygems.org/read/chapter/20 for more options
   gem.name = "worker-army"
   gem.homepage = "http://github.com/okiess/worker-army"
   gem.license = "MIT"
@@ -49,6 +48,7 @@ task 'start_example_worker' do
   worker.process_queue
 end
 
+desc "Start a worker-army worker to execute a job class"
 task :start_worker, :job_class do |t, args|
   if args[:job_class]
     worker = WorkerArmy::Worker.new

data/VERSION

@@ -1 +1 @@
-0.2.1
+0.3.0

data/lib/worker-army.rb

@@ -1,4 +1,18 @@
 require "redis"
+require "logger"
+
+module WorkerArmy
+  class Log
+    attr_accessor :log
+
+    def initialize
+      self.log = Logger.new('/tmp/worker-army.log')
+      self.log.level = Logger::DEBUG
+    end
+  end
+end
+
+$WORKER_ARMY_LOG = WorkerArmy::Log.new.log
 
 require File.dirname(__FILE__) + '/worker_army/queue'
 require File.dirname(__FILE__) + '/worker_army/worker'

data/lib/worker_army/client.rb

@@ -4,32 +4,59 @@ require "multi_json"
 
 module WorkerArmy
   class Client
-    def self.push_job(job_class, data = {}, callback_url = nil, queue_name = 'queue')
-      raise "No data" unless data
-      raise "No job class provided" unless job_class
-      
-      if ENV['worker_army_endpoint']
-        puts "Using environment variables for config..."
-        @config = { endpoint: ENV['worker_army_endpoint'] }
-      else
+    class << self
+      def push_job(job_class, data = {}, callback_url = nil, queue_prefix = 'queue', retry_count = 0)
+        raise "No data" unless data
+        raise "No job class provided" unless job_class
+        
+        if ENV['worker_army_endpoint']
+          # puts "Using environment variables for config..."
+          @config = { endpoint: ENV['worker_army_endpoint'] }
+        else
+          begin
+            # puts "Using config in your home directory"
+            @config = YAML.load(File.read("#{ENV['HOME']}/.worker_army.yml"))
+          rescue Errno::ENOENT
+            raise "worker_army.yml expected in ~/.worker_army.yml"
+          end
+        end
+
+        worker_army_base_url = @config['endpoint']
+        callback_url = "#{worker_army_base_url}/generic_callback" unless callback_url
+        response = nil
         begin
-          puts "Using config in your home directory"
-          @config = YAML.load(File.read("#{ENV['HOME']}/.worker_army.yml"))
-        rescue Errno::ENOENT
-          raise "worker_army.yml expected in ~/.worker_army.yml"
+          response = RestClient.post "#{worker_army_base_url}/jobs",
+            data.merge(
+              job_class: job_class,
+              callback_url: "#{worker_army_base_url}/callback?callback_url=#{callback_url}",
+              queue_prefix: queue_prefix
+            ).to_json,
+            :content_type => :json, :accept => :json
+        rescue => e
+          puts "Failed! Retrying (#{retry_count})..."
+          retry_count += 1
+          if retry_count < client_retry_count(@config)
+            sleep (retry_count * 2)
+            push_job(job_class, data, callback_url, queue_prefix, retry_count)
+          end
+        end
+        if response and response.body and response.code == 200
+          hash = JSON.parse(response.body)
+          hash.merge(success: true)
+        else
+          { success: false }
         end
       end
 
-      worker_army_base_url = @config['endpoint']
-      callback_url = "#{worker_army_base_url}/generic_callback" unless callback_url
-      response = RestClient.post "#{worker_army_base_url}/jobs",
-        data.merge(
-          job_class: job_class,
-          callback_url: "#{worker_army_base_url}/callback?callback_url=#{callback_url}",
-          queue_name: queue_name
-        ).to_json,
-        :content_type => :json, :accept => :json
-      response.code == 200
+      def client_retry_count(config)
+        if ENV['worker_army_client_retry_count']
+          return ENV['worker_army_client_retry_count'].to_i
+        elsif config and config['client_retry_count']
+          return config['client_retry_count'].to_i
+        else
+          return 10
+        end
+      end
     end
   end
 end

data/lib/worker_army/example_job.rb

@@ -1,7 +1,9 @@
 class ExampleJob
+  attr_accessor :log
+
   def perform(data = {})
     response_data = {foo: 'bar'}
-    puts "in example worker with data: #{data}"
+    log.debug("in example worker with data: #{data}")
     sleep 2
     response_data
   end

data/lib/worker_army/queue.rb

@@ -3,6 +3,7 @@ require "rest-client"
 require "json"
 require "multi_json"
 require "yaml"
+require 'securerandom'
 
 module WorkerArmy
   class Queue
@@ -12,83 +13,127 @@ module WorkerArmy
       @config = Queue.config
       # puts "Config: #{@config}"
       Queue.redis_instance
+      @log = $WORKER_ARMY_LOG
     end
-    
-    def self.config
-      if ENV['worker_army_redis_host'] and ENV['worker_army_redis_port']
-        config = { 'redis_host' => ENV['worker_army_redis_host'], 'redis_port' => ENV['worker_army_redis_port'] }
-        if ENV['worker_army_redis_auth']
-          config['redis_auth'] = ENV['worker_army_redis_auth']
-        end
-      else
-        begin
-          # puts "Using config in your home directory"
-          config = YAML.load(File.read("#{ENV['HOME']}/.worker_army.yml"))
-        rescue Errno::ENOENT
-          raise "worker_army.yml expected in ~/.worker_army.yml"
+
+    class << self
+      def config
+        if ENV['worker_army_redis_host'] and ENV['worker_army_redis_port']
+          config = { 'redis_host' => ENV['worker_army_redis_host'], 'redis_port' => ENV['worker_army_redis_port'] }
+          if ENV['worker_army_redis_auth']
+            config['redis_auth'] = ENV['worker_army_redis_auth']
+          end
+        else
+          begin
+            # puts "Using config in your home directory"
+            config = YAML.load(File.read("#{ENV['HOME']}/.worker_army.yml"))
+          rescue Errno::ENOENT
+            raise "worker_army.yml expected in ~/.worker_army.yml"
+          end
         end
+        config
       end
-      config
-    end
 
-    def self.redis_instance
-      $config = Queue.config unless $config
-      unless $redis
-        $redis = Redis.new(host: $config['redis_host'], port: $config['redis_port'])
+      def redis_instance
+        $config = Queue.config unless $config
+        unless $redis
+          $redis = Redis.new(host: $config['redis_host'], port: $config['redis_port'])
+        end
+        $redis.auth($config['redis_auth']) if $config['redis_auth']
+        $redis
       end
-      $redis.auth($config['redis_auth']) if $config['redis_auth']
-      $redis
-    end
 
-    def self.close_redis_connection
-      $redis.quit if $redis
-      $redis = nil
+      def close_redis_connection
+        $redis.quit if $redis
+        $redis = nil
+      end
     end
 
-    def push(data, queue_name = "queue")
+    def push(data, queue_prefix = "queue")
       if Queue.redis_instance and data
-        job_count = Queue.redis_instance.incr("#{queue_name}_counter")
-        queue_name = data['queue_name'] if data['queue_name']
-        queue_name = "#{queue_name}_#{data['job_class']}"
-        Queue.redis_instance.rpush queue_name, data.merge(job_count: job_count).to_json
+        job_count = Queue.redis_instance.incr("#{queue_prefix}_counter")
+        queue_prefix = queue_prefix if queue_prefix
+        queue_prefix = data['queue_prefix'] if data['queue_prefix']
+        queue_name = "#{queue_prefix}_#{data['job_class']}"
+        Queue.redis_instance.sadd 'known_queues', queue_name
+        queue_count = Queue.redis_instance.incr("#{queue_name}_counter")
+        job_id = SecureRandom.uuid
+        Queue.redis_instance.rpush queue_name, data.merge(job_count: job_count,
+          queue_count: queue_count, job_id: job_id, queue_name: queue_name).to_json
       end
       raise "No data" unless data
       raise "No redis connection!" unless Queue.redis_instance
+      { job_count: job_count, job_id: job_id, queue_count: queue_count,
+        queue_name: queue_name }
     end
 
-    def pop(job_class_name, queue_name = "queue")
+    def pop(job_class_name, queue_prefix = "queue")
       raise "No redis connection!" unless Queue.redis_instance
-      return Queue.redis_instance.blpop("#{queue_name}_#{job_class_name}")
+      return Queue.redis_instance.blpop("#{queue_prefix}_#{job_class_name}")
     end
 
     def save_result(data)
       if data
-        job_count = data['job_count']
+        job_id = data['job_id']
         callback_url = data['callback_url']
-        Queue.redis_instance["job_#{job_count}"] = data
+        Queue.redis_instance["job_#{job_id}"] = data
+        Queue.redis_instance.lpush 'jobs', job_id
         if callback_url
           data.delete("callback_url")
           begin
             response = RestClient.post callback_url.split("?callback_url=").last,
               data.to_json, :content_type => :json, :accept => :json
           rescue => e
-            puts e
+            @log.error(e)
           end
         end
       end
     end
 
+    def add_failed_job(job_id)
+      Queue.redis_instance.lpush 'failed_jobs', job_id
+    end
+    
+    def failed_jobs
+      Queue.redis_instance.llen 'failed_jobs'
+    end
+
     def ping(data)
       Queue.redis_instance.lpush 'workers', data.to_json
+      Queue.redis_instance.set 'last_ping', data[:timestamp].to_i
+    end
+
+    def last_ping
+      Queue.redis_instance.get 'last_ping'
     end
 
     def get_known_workers(recent_worker_pings = 1000)
       worker_pings = Queue.redis_instance.lrange 'workers', 0, recent_worker_pings
-      worker_pings ? worker_pings.collect {|json| JSON.parse(json)} : []
+      return [] unless worker_pings
+      worker_pings = worker_pings.collect {|json| JSON.parse(json)}.sort_by {|h| h['timestamp'].to_i}.reverse
+      uniq_workers = worker_pings.collect {|h| [h['host_name'], h['worker_pid']]}.uniq
+      workers = []
+      uniq_workers.each do |worker_pair|
+        worker_pings.each do |hash|
+          if hash['host_name'] == worker_pair[0] and hash['worker_pid'] == worker_pair[1]
+            workers << hash
+            break
+          end
+        end
+      end
+      workers
+    end
+    
+    def get_known_queues
+      Queue.redis_instance.smembers 'known_queues'
+    end
+
+    def finished_jobs
+      Queue.redis_instance.llen 'jobs'
     end
 
-    def get_job_count(queue_name = "queue")
-      Queue.redis_instance["#{queue_name}_counter"]
+    def get_job_count(queue_prefix = "queue")
+      Queue.redis_instance["#{queue_prefix}_counter"].to_i
     end
   end
 end

data/lib/worker_army/web.rb

@@ -9,14 +9,21 @@ queue = WorkerArmy::Queue.new
 get '/' do
   job_count = queue.get_job_count || 0
   workers = queue.get_known_workers
-  data = { job_count: job_count, workers: workers }
+  last_ping = queue.last_ping || 0
+  queues = queue.get_known_queues
+  finished_jobs = queue.finished_jobs
+  failed_jobs = queue.failed_jobs
+  data = { job_count: job_count, finished_jobs: finished_jobs,
+    failed_jobs: failed_jobs, workers: workers,
+    last_worker_ping: last_ping.to_i, queues: queues
+  }
   json data
 end
 
 post '/jobs' do
   data = JSON.parse(request.body.read)
-  queue.push data if data
-  json data
+  queue_job = queue.push data if data
+  json queue_job
 end
 
 post '/callback' do

data/lib/worker_army/worker.rb

@@ -5,47 +5,83 @@ require 'socket'
 
 module WorkerArmy
   class Worker
-    attr_accessor :queue, :job, :worker_name
+    attr_accessor :queue, :job, :worker_name, :processed, :failed, :config
     def initialize(worker_name = nil)
       @queue = WorkerArmy::Queue.new
       @worker_name = worker_name
       @host_name = Socket.gethostname
+      @processed = 0
+      @failed = 0
+      begin
+        # puts "Using config in your home directory"
+        @config = YAML.load(File.read("#{ENV['HOME']}/.worker_army.yml"))
+      rescue Errno::ENOENT
+        # ignore
+      end
+      @log = $WORKER_ARMY_LOG
     end
 
     def process_queue
       raise "No job class set!" unless @job
+      @job.log = @log if @job.respond_to?(:log)
       @queue.ping(worker_pid: Process.pid, job_name: @job.class.name, host_name: @host_name,
         timestamp: Time.now.utc.to_i)
-      puts "Worker ready!"
+      @log.info("Worker ready! Waiting for jobs: #{@job.class.name}")
+      @log.info("Processed: #{@processed} - Failed: #{@failed}")
       list, element = @queue.pop(@job.class.name)
       if list and element
-        puts "List: #{list} => #{element}"
-        response_data = {}
-        job_count = 0
-        begin
-          data = JSON.parse(element)
-          job_count = data['job_count']
-          callback_url = data['callback_url']
-          if @job and @job.class.name == data['job_class']
-            response_data = @job.perform(data)
-            response_data.merge!(job_count: job_count, callback_url: callback_url,
-              finished_at: Time.now.utc.to_i, host_name: @host_name)
-            if @worker_name
-              response_data.merge!(worker_name: @worker_name)
-            end
+        execute_job(list, element, 0)
+      end
+    end
+
+    private
+    def execute_job(list, element, retry_count = 0)
+      @log.debug("Queue: #{list} => #{element}") if retry_count == 0
+      response_data = {}
+      job_count = 0
+      begin
+        data = JSON.parse(element)
+        job_id = data['job_id']
+        callback_url = data['callback_url']
+        if @job and @job.class.name == data['job_class']
+          response_data = @job.perform(data)
+          response_data = {} unless response_data
+          response_data.merge!(job_id: job_id, callback_url: callback_url,
+            finished_at: Time.now.utc.to_i, host_name: @host_name)
+          @processed += 1
+          if @worker_name
+            response_data.merge!(worker_name: @worker_name)
           end
-        rescue => e
-          puts e
         end
-        if response_data
-          begin
-            response = RestClient.post data['callback_url'],
-              response_data.to_json, :content_type => :json, :accept => :json
-          rescue => e
-            puts e
-          end
+        response_data
+      rescue => e
+        @log.error(e)
+        retry_count += 1
+        if retry_count < worker_retry_count(@config)
+          @log.debug("Failed! Retrying (#{retry_count})...")
+          sleep (retry_count * 2)
+          execute_job(list, element, retry_count)
+        else
+          @failed += 1
+          @queue.add_failed_job(job_id)
         end
-        self.process_queue
+      end
+      begin
+        response = RestClient.post data['callback_url'],
+          response_data.to_json, :content_type => :json, :accept => :json
+      rescue => e
+        @logger.error(e)
+      end
+      self.process_queue
+    end
+
+    def worker_retry_count(config = nil)
+      if ENV['worker_army_worker_retry_count']
+        return ENV['worker_army_worker_retry_count'].to_i
+      elsif config and config['worker_retry_count']
+        return config['worker_retry_count'].to_i
+      else
+        return 10
       end
     end
   end

data/worker-army.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: worker-army 0.2.1 ruby lib
+# stub: worker-army 0.3.0 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "worker-army"
-  s.version = "0.2.1"
+  s.version = "0.3.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib"]
   s.authors = ["Oliver Kiessler"]
-  s.date = "2014-05-23"
+  s.date = "2014-05-27"
   s.description = "Simple redis based worker queue with a HTTP/Rest interface"
   s.email = "kiessler@inceedo.com"
   s.executables = ["worker_army"]

data/worker-army.yml.sample

@@ -1,4 +1,6 @@
 endpoint: "http://localhost:9292"
 redis_host: "localhost"
 redis_port: 6379
-redis_auth: "YOUR_PASS"
+redis_auth: "YOUR_PASS"
+client_retry_count: 10
+worker_retry_count: 10

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: worker-army
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.3.0
 platform: ruby
 authors:
 - Oliver Kiessler
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-05-23 00:00:00.000000000 Z
+date: 2014-05-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis