worker-army 0.3.2 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: dc2f98f3447708cde0fa772761604dd850335719
-  data.tar.gz: 30a2b6653e1bd686d6c02f97dae56cd2853e0518
+  metadata.gz: 356fda8449bf808280461964fd361d5960d4098f
+  data.tar.gz: 2bc6419214d9b6498bc132ff8cb5db842b0ebb43
 SHA512:
-  metadata.gz: edd9e384ff237692df629d3e91fe6a97fa7c0f879f8ffa94c19f279d08106f21da28abce50964ea3fd9718fcc2708c263f1179c8fd6b121fa3949c0213cfa170
-  data.tar.gz: 36a454d8a2981a1c315b12c2f3d92d99b5f4fef5569583286a8658236d161eb975b3d2aeeaffe24665f38c656beca6ad13b7dbef5173963b5d832d2f4cf9e644
+  metadata.gz: 3b62516a6387cf9e5ab8350bf823f63812fa9b90197cb746bc26e98a2bab4d3ba1a38b4e282d83c3f55480f1b01db7d15ded35cbbbe94869102fd5b4c889bdb4
+  data.tar.gz: 0104f1aa27d28a0f3b2d18455dff4dde4f23c4bc23295ab989a06d540cbb89b98489e2791a463b8c769fcb96c64a4a0f951fd5c930d3628e6a116018060f2405

data/Rakefile

@@ -43,17 +43,13 @@ end
 
 require File.dirname(__FILE__) + '/lib/worker-army'
 task 'start_example_worker' do
-  worker = WorkerArmy::Worker.new
-  worker.job = ExampleJob.new
-  worker.process_queue
+  WorkerArmy::Worker.new(ExampleJob.new).process_queue
 end
 
 desc "Start a worker-army worker to execute a job class"
 task :start_worker, :job_class do |t, args|
   if args[:job_class]
-    worker = WorkerArmy::Worker.new
     clazz = Object.const_get(args[:job_class].to_s)
-    worker.job = clazz.new
-    worker.process_queue
+    WorkerArmy::Worker.new(clazz.new).process_queue
   end
 end

data/VERSION

@@ -1 +1 @@
-0.3.2
+0.4.0

data/lib/worker-army.rb

@@ -2,6 +2,7 @@ require "redis"
 require "logger"
 
 require File.dirname(__FILE__) + '/worker_army/log'
+require File.dirname(__FILE__) + '/worker_army/base'
 require File.dirname(__FILE__) + '/worker_army/queue'
 require File.dirname(__FILE__) + '/worker_army/worker'
 require File.dirname(__FILE__) + '/worker_army/client'

data/lib/worker_army/base.rb

@@ -0,0 +1,33 @@
+module WorkerArmy
+  class Base
+    attr_accessor :config
+
+    class << self
+      def config
+        if ENV['worker_army_redis_host'] and ENV['worker_army_redis_port']
+          config = { 'redis_host' => ENV['worker_army_redis_host'], 'redis_port' => ENV['worker_army_redis_port'] }
+          if ENV['worker_army_redis_auth']
+            config['redis_auth'] = ENV['worker_army_redis_auth']
+          end
+          if ENV['worker_army_worker_retry_count']
+            config['worker_retry_count'] = ENV['worker_army_worker_retry_count'].to_i
+          end
+          if ENV['worker_army_client_retry_count']
+            config['client_retry_count'] = ENV['worker_army_client_retry_count'].to_i
+          end
+          if ENV['worker_army_endpoint']
+            config['endpoint'] = ENV['worker_army_endpoint']
+          end
+        else
+          begin
+            # puts "Using config in your home directory"
+            config = YAML.load(File.read("#{ENV['HOME']}/.worker_army.yml"))
+          rescue Errno::ENOENT
+            raise "worker-army configuration expected in ~/.worker_army.yml or provide env vars..."
+          end
+        end
+        config
+      end
+    end  
+  end
+end

data/lib/worker_army/client.rb

@@ -1,26 +1,16 @@
 require "rest-client"
 require "json"
 require "multi_json"
+require File.dirname(__FILE__) + '/base'
 
 module WorkerArmy
-  class Client
+  class Client < Base
     class << self
       def push_job(job_class, data = {}, callback_url = nil, queue_prefix = 'queue', retry_count = 0)
         raise "No data" unless data
         raise "No job class provided" unless job_class
-        
-        if ENV['worker_army_endpoint']
-          # puts "Using environment variables for config..."
-          @config = { endpoint: ENV['worker_army_endpoint'] }
-        else
-          begin
-            # puts "Using config in your home directory"
-            @config = YAML.load(File.read("#{ENV['HOME']}/.worker_army.yml"))
-          rescue Errno::ENOENT
-            raise "worker_army.yml expected in ~/.worker_army.yml"
-          end
-        end
 
+        @config = self.config
         worker_army_base_url = @config['endpoint']
         callback_url = "#{worker_army_base_url}/generic_callback" unless callback_url
         response = nil
@@ -48,11 +38,11 @@ module WorkerArmy
         end
       end
 
-      def client_retry_count(config)
+      def client_retry_count(conf = nil)
         if ENV['worker_army_client_retry_count']
           return ENV['worker_army_client_retry_count'].to_i
-        elsif config and config['client_retry_count']
-          return config['client_retry_count'].to_i
+        elsif conf and conf['client_retry_count']
+          return conf['client_retry_count'].to_i
         else
           return 10
         end

data/lib/worker_army/queue.rb

@@ -4,38 +4,19 @@ require "json"
 require "multi_json"
 require "yaml"
 require 'securerandom'
+require File.dirname(__FILE__) + '/base'
 
 module WorkerArmy
-  class Queue
-    attr_accessor :config
-
+  class Queue < Base
     def initialize
-      @config = Queue.config
-      # puts "Config: #{@config}"
+      @config = self.config
       Queue.redis_instance
       @log = WorkerArmy::Log.new.log
     end
 
     class << self
-      def config
-        if ENV['worker_army_redis_host'] and ENV['worker_army_redis_port']
-          config = { 'redis_host' => ENV['worker_army_redis_host'], 'redis_port' => ENV['worker_army_redis_port'] }
-          if ENV['worker_army_redis_auth']
-            config['redis_auth'] = ENV['worker_army_redis_auth']
-          end
-        else
-          begin
-            # puts "Using config in your home directory"
-            config = YAML.load(File.read("#{ENV['HOME']}/.worker_army.yml"))
-          rescue Errno::ENOENT
-            raise "worker_army.yml expected in ~/.worker_army.yml"
-          end
-        end
-        config
-      end
-
       def redis_instance
-        $config = Queue.config unless $config
+        $config = self.config unless $config
         unless $redis
           $redis = Redis.new(host: $config['redis_host'], port: $config['redis_port'])
         end
@@ -76,28 +57,73 @@ module WorkerArmy
       if data
         job_id = data['job_id']
         callback_url = data['callback_url']
-        Queue.redis_instance["job_#{job_id}"] = data
+        Queue.redis_instance["job_#{job_id}"] = data.to_json
         Queue.redis_instance.lpush 'jobs', job_id
         if callback_url
           data.delete("callback_url")
           begin
             response = RestClient.post callback_url.split("?callback_url=").last,
               data.to_json, :content_type => :json, :accept => :json
+            if response.code == 404 or response.code == 500
+              @log.error("Response from callback url: #{response.code}")
+              add_failed_callback_job(job_id)
+            end
           rescue => e
             @log.error(e)
+            add_failed_callback_job(job_id)
           end
         end
       end
     end
 
+    def add_current_job(job_id)
+      Queue.redis_instance.sadd 'current_jobs', job_id
+    end
+
+    def remove_current_job(job_id)
+      Queue.redis_instance.srem 'current_jobs', job_id
+    end
+    
+    def current_jobs_count
+      Queue.redis_instance.scard 'current_jobs'
+    end
+
+    def current_jobs
+      Queue.redis_instance.smembers 'current_jobs'
+    end
+    
+    def clear_current_jobs
+      Queue.redis_instance.del 'current_jobs'
+    end
+
+    def job_data(job_id)
+      Queue.redis_instance["job_#{job_id}"]
+    end
+
     def add_failed_job(job_id)
       Queue.redis_instance.lpush 'failed_jobs', job_id
     end
-    
-    def failed_jobs
+
+    def add_failed_callback_job(job_id)
+      Queue.redis_instance.lpush 'failed_callback_jobs', job_id
+    end
+
+    def failed_jobs_count
       Queue.redis_instance.llen 'failed_jobs'
     end
 
+    def failed_jobs
+      Queue.redis_instance.lrange 'failed_jobs', 0, failed_jobs_count
+    end
+
+    def failed_callback_jobs_count
+      Queue.redis_instance.llen 'failed_callback_jobs'
+    end
+
+    def failed_callback_jobs
+      Queue.redis_instance.lrange 'failed_callback_jobs', 0, failed_callback_jobs_count
+    end
+
     def ping(data)
       Queue.redis_instance.lpush 'workers', data.to_json
       Queue.redis_instance.set 'last_ping', data[:timestamp].to_i
@@ -121,14 +147,15 @@ module WorkerArmy
           end
         end
       end
-      workers
+      now = Time.now.utc.to_i
+      workers.select {|h| h if now - h['timestamp'].to_i < 3600}
     end
-    
+
     def get_known_queues
       Queue.redis_instance.smembers 'known_queues'
     end
 
-    def finished_jobs
+    def finished_jobs_count
       Queue.redis_instance.llen 'jobs'
     end
 

data/lib/worker_army/web.rb

@@ -7,16 +7,23 @@ require File.dirname(__FILE__) + '/queue'
 
 queue = WorkerArmy::Queue.new
 
+before do
+  content_type 'application/json', :charset => 'utf-8'
+end
+
 get '/' do
   job_count = queue.get_job_count || 0
   workers = queue.get_known_workers
   last_ping = queue.last_ping || 0
   queues = queue.get_known_queues
-  finished_jobs = queue.finished_jobs
-  failed_jobs = queue.failed_jobs
+  finished_jobs = queue.finished_jobs_count
+  failed_callback_jobs = queue.failed_callback_jobs_count
+  failed_jobs = queue.failed_jobs_count
+  current_jobs = queue.current_jobs
   data = { job_count: job_count, finished_jobs: finished_jobs,
-    failed_jobs: failed_jobs, workers: workers,
-    last_worker_ping: last_ping.to_i, queues: queues
+    failed_jobs: failed_jobs, failed_callback_jobs: failed_callback_jobs,
+    workers: workers, last_worker_ping: last_ping.to_i, queues: queues,
+    current_jobs: current_jobs
   }
   json data
 end
@@ -27,6 +34,11 @@ post '/jobs' do
   json queue_job
 end
 
+get '/jobs/:job_id' do
+  data = queue.job_data(params[:job_id])
+  json JSON.parse(data)
+end
+
 post '/callback' do
   data = JSON.parse(request.body.read)
   queue.save_result(data) if data
@@ -38,3 +50,13 @@ post '/generic_callback' do
   status = { :status => 'ok' }
   json status
 end
+
+get '/failed_jobs' do
+  failed_jobs = queue.failed_jobs
+  json failed_jobs
+end
+
+get '/failed_callback_jobs' do
+  failed_callback_jobs = queue.failed_callback_jobs
+  json failed_callback_jobs
+end

data/lib/worker_army/worker.rb

@@ -2,22 +2,19 @@ require "rest-client"
 require "json"
 require "multi_json"
 require 'socket'
+require File.dirname(__FILE__) + '/base'
 
 module WorkerArmy
-  class Worker
-    attr_accessor :queue, :job, :worker_name, :processed, :failed, :config
-    def initialize(worker_name = nil)
+  class Worker < Base
+    attr_accessor :queue, :job, :worker_name, :processed, :failed
+    def initialize(job, worker_name = nil)
       @queue = WorkerArmy::Queue.new
+      @job = job
       @worker_name = worker_name
       @host_name = Socket.gethostname
       @processed = 0
       @failed = 0
-      begin
-        # puts "Using config in your home directory"
-        @config = YAML.load(File.read("#{ENV['HOME']}/.worker_army.yml"))
-      rescue Errno::ENOENT
-        # ignore
-      end
+      @config = self.config
       @log = WorkerArmy::Log.new.log
     end
 
@@ -26,8 +23,8 @@ module WorkerArmy
       @job.log = @log if @job.respond_to?(:log)
       @queue.ping(worker_pid: Process.pid, job_name: @job.class.name, host_name: @host_name,
         timestamp: Time.now.utc.to_i)
-      @log.info("Worker ready! Waiting for jobs: #{@job.class.name}")
-      @log.info("Processed: #{@processed} - Failed: #{@failed}")
+      @log.info("Worker #{@host_name}-#{Process.pid} => Queue: queue_#{@job.class.name}")
+      @log.info("Worker #{@host_name}-#{Process.pid} => Processed: #{@processed} - Failed: #{@failed}")
       list, element = @queue.pop(@job.class.name)
       if list and element
         execute_job(list, element, 0)
@@ -44,6 +41,7 @@ module WorkerArmy
         job_id = data['job_id']
         callback_url = data['callback_url']
         if @job and @job.class.name == data['job_class']
+          @queue.add_current_job(job_id)
           response_data = @job.perform(data)
           response_data = {} unless response_data
           response_data.merge!(job_id: job_id, callback_url: callback_url,
@@ -53,8 +51,10 @@ module WorkerArmy
             response_data.merge!(worker_name: @worker_name)
           end
         end
+        @queue.remove_current_job(job_id)
         response_data
       rescue => e
+        @queue.remove_current_job(job_id)
         @log.error(e)
         retry_count += 1
         if retry_count < worker_retry_count(@config)
@@ -75,11 +75,11 @@ module WorkerArmy
       self.process_queue
     end
 
-    def worker_retry_count(config = nil)
+    def worker_retry_count(conf = nil)
       if ENV['worker_army_worker_retry_count']
         return ENV['worker_army_worker_retry_count'].to_i
-      elsif config and config['worker_retry_count']
-        return config['worker_retry_count'].to_i
+      elsif conf and conf['worker_retry_count']
+        return conf['worker_retry_count'].to_i
       else
         return 10
       end

data/worker-army.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: worker-army 0.3.2 ruby lib
+# stub: worker-army 0.4.0 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "worker-army"
-  s.version = "0.3.2"
+  s.version = "0.4.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib"]
   s.authors = ["Oliver Kiessler"]
-  s.date = "2014-05-28"
+  s.date = "2014-05-29"
   s.description = "Simple redis based worker queue with a HTTP/Rest interface"
   s.email = "kiessler@inceedo.com"
   s.executables = ["worker_army"]
@@ -32,6 +32,7 @@ Gem::Specification.new do |s|
     "bin/worker_army",
     "config.ru",
     "lib/worker-army.rb",
+    "lib/worker_army/base.rb",
     "lib/worker_army/client.rb",
     "lib/worker_army/example_job.rb",
     "lib/worker_army/log.rb",

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: worker-army
 version: !ruby/object:Gem::Version
-  version: 0.3.2
+  version: 0.4.0
 platform: ruby
 authors:
 - Oliver Kiessler
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-05-28 00:00:00.000000000 Z
+date: 2014-05-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
@@ -171,6 +171,7 @@ files:
 - bin/worker_army
 - config.ru
 - lib/worker-army.rb
+- lib/worker_army/base.rb
 - lib/worker_army/client.rb
 - lib/worker_army/example_job.rb
 - lib/worker_army/log.rb