worker-army 0.4.0 → 0.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 356fda8449bf808280461964fd361d5960d4098f
-  data.tar.gz: 2bc6419214d9b6498bc132ff8cb5db842b0ebb43
+  metadata.gz: 927bd98e49089fff441dad39d22b1415e72846e3
+  data.tar.gz: 543b233b40f905bac71fc9378767412de284cfe8
 SHA512:
-  metadata.gz: 3b62516a6387cf9e5ab8350bf823f63812fa9b90197cb746bc26e98a2bab4d3ba1a38b4e282d83c3f55480f1b01db7d15ded35cbbbe94869102fd5b4c889bdb4
-  data.tar.gz: 0104f1aa27d28a0f3b2d18455dff4dde4f23c4bc23295ab989a06d540cbb89b98489e2791a463b8c769fcb96c64a4a0f951fd5c930d3628e6a116018060f2405
+  metadata.gz: 3c8150b9f82e39b7dc5de67659166c975ef39ec8978065fa1f30b7b1842c2fc5e8f524a86093c6df9c7af281692775bb45489e089ae4d2955ce8f59eb936e97e
+  data.tar.gz: 81516c5776426b4b19030eac5ff4c722c22f0713b6c6756e9e1ff504982b77743737faf20090a6826b6630b3ec824670a800c8ad12539839c95be67b1ea32d13

data/README.md

@@ -0,0 +1,115 @@
+# worker-army
+
+worker-army is a simple redis based worker queue written in ruby for running background jobs with a HTTP/REST interface. It doesn't have any ties to Ruby on Rails and aims to be lightweight and memory efficient.
+
+worker-army is work in progress and not yet production-ready!
+
+## Installation
+
+To install worker-army, add the gem to your Gemfile:
+
+    gem "worker-army"
+
+Then run `bundle`. If you're not using Bundler, just `gem install worker-army`.
+
+## Requirements
+
+* Ruby 2.1
+* Redis 2.x
+
+## Configuration
+
+Checkout `worker_army.yml.sample` on the available configuration options. Then create your own `~/.worker_army.yml` file.
+
+If you can't provide or don't want to provide the config file at that location, you can also set environment variables (when you're deploying on heroku for example):
+
+     worker_army_endpoint
+     worker_army_redis_host
+     worker_army_redis_port
+     worker_army_redis_auth
+     worker_army_worker_retry_count
+     worker_army_client_retry_count
+     worker_army_callback_retry_count
+
+## Server / Queue
+
+Start the worker-army server by executing:
+
+    $ worker_army
+    
+or use Foreman (checkout `Procfile`)
+
+    $ foreman start
+
+You easily run the worker-army server on heroku. It should work out of the box. You're just going to have provide the configuration environment variables.
+
+You can open the server status overview by calling the server root url:
+
+http://your-worker-army-server.com/
+
+The status overview returns JSON, so you can easily embed it into your own applications.
+
+To view the result of a single job execution:
+
+http://your-worker-army-server.com/jobs/YOUR_JOB_ID
+
+## Client
+
+You can push jobs onto the queue with this ruby client (or any other HTTP REST capable mechanism). The ruby client will communicate with the server from anywhere:
+
+    WorkerArmy::Client.push_job("ExampleJob", {"foo" => "bar"}, "http://your-callback-server.com/data-callback?some_id=1234")
+
+Provide an (optional) URL as the last argument and worker-army will return the job result to the URL as a HTTP POST callback.
+
+## Workers
+
+You can start up a worker with a job class assigned with the following:
+
+    $ rake start_worker[ExampleJob]
+
+## Jobs
+
+Jobs are just regular ruby classes that look like this:
+
+    class ExampleJob
+      attr_accessor :log
+      
+      def perform(data = {})
+        # Your Code
+      end
+    end
+    
+This is how the data looks like for example that the worker passes into the `perform` method:
+
+    {
+      "foo"=>"bar",
+      "job_class"=>"ExampleJob",
+      "callback_url"=>"http://your-worker-army-server.com/callback?callback_url=http://your-callback-server.com/data-callback?some_id=1234",
+      "queue_prefix"=>"queue",
+      "job_count"=>1081,
+      "queue_count"=>1,
+      "job_id"=>"9192a85e-320d-4e15-bdc5-2fa41e862370",
+      "queue_name"=>"queue_ExampleJob"
+    }
+
+The client who created the job will get this in return:
+
+    {
+      "job_count"=>1081,
+      "job_id"=>"9192a85e-320d-4e15-bdc5-2fa41e862370",
+      "queue_count"=>1,
+      "queue_name"=>"queue_ExampleJob",
+      "success"=>true
+    }
+    
+The `success` field indicates if the job creation (not the execution!) was successfull.
+
+The result of the job execution will be provided via the callback or by manually calling the job data URL (http://your-worker-army-server.com/jobs/YOUR_JOB_ID).
+
+## Logging
+
+Per default, worker-army will create a log file in /tmp/worker-army.log.
+
+## Copyright
+
+Copyright (c) 2013-2014 Oliver Kiessler. See LICENSE.txt for further details.

data/VERSION

@@ -1 +1 @@
-0.4.0
+0.4.1

data/lib/worker_army/base.rb

@@ -15,6 +15,9 @@ module WorkerArmy
           if ENV['worker_army_client_retry_count']
             config['client_retry_count'] = ENV['worker_army_client_retry_count'].to_i
           end
+          if ENV['worker_army_callback_retry_count']
+            config['callback_retry_count'] = ENV['worker_army_callback_retry_count'].to_i
+          end
           if ENV['worker_army_endpoint']
             config['endpoint'] = ENV['worker_army_endpoint']
           end
@@ -28,6 +31,36 @@ module WorkerArmy
         end
         config
       end
+      
+      def client_retry_count(conf = nil)
+        if ENV['worker_army_client_retry_count']
+          return ENV['worker_army_client_retry_count'].to_i
+        elsif conf and conf['client_retry_count']
+          return conf['client_retry_count'].to_i
+        else
+          return 10
+        end
+      end
+    end
+
+    def worker_retry_count(conf = nil)
+      if ENV['worker_army_worker_retry_count']
+        return ENV['worker_army_worker_retry_count'].to_i
+      elsif conf and conf['worker_retry_count']
+        return conf['worker_retry_count'].to_i
+      else
+        return 10
+      end
+    end
+
+    def callback_retry_count(conf = nil)
+      if ENV['worker_army_callback_retry_count']
+        return ENV['worker_army_callback_retry_count'].to_i
+      elsif conf and conf['callback_retry_count']
+        return conf['callback_retry_count'].to_i
+      else
+        return 3
+      end
     end  
   end
 end

data/lib/worker_army/client.rb

@@ -32,19 +32,9 @@ module WorkerArmy
         end
         if response and response.body and response.code == 200
           hash = JSON.parse(response.body)
-          hash.merge(success: true)
+          hash.merge('success' => true)
         else
-          { success: false }
-        end
-      end
-
-      def client_retry_count(conf = nil)
-        if ENV['worker_army_client_retry_count']
-          return ENV['worker_army_client_retry_count'].to_i
-        elsif conf and conf['client_retry_count']
-          return conf['client_retry_count'].to_i
-        else
-          return 10
+          { 'success' => false }
         end
       end
     end

data/lib/worker_army/queue.rb

@@ -61,17 +61,28 @@ module WorkerArmy
         Queue.redis_instance.lpush 'jobs', job_id
         if callback_url
           data.delete("callback_url")
-          begin
-            response = RestClient.post callback_url.split("?callback_url=").last,
-              data.to_json, :content_type => :json, :accept => :json
-            if response.code == 404 or response.code == 500
-              @log.error("Response from callback url: #{response.code}")
-              add_failed_callback_job(job_id)
-            end
-          rescue => e
-            @log.error(e)
-            add_failed_callback_job(job_id)
-          end
+          deliver_callback(job_id, callback_url, data)
+        end
+      end
+    end
+
+    def deliver_callback(job_id, callback_url, data, retry_count = 0)
+      begin
+        response = RestClient.post callback_url.split("?callback_url=").last,
+          data.to_json, :content_type => :json, :accept => :json
+        if response.code == 404 or response.code == 500
+          @log.error("Response from callback url: #{response.code}")
+          add_failed_callback_job(job_id)
+        end
+      rescue => e
+        @log.error(e)
+        retry_count += 1
+        if retry_count < callback_retry_count(@config)
+          @log.debug("Delivering external callback failed! Retrying (#{retry_count})...")
+          sleep (retry_count * 2)
+          deliver_callback(job_id, callback_url, data, retry_count)
+        else
+          add_failed_callback_job(job_id)
         end
       end
     end

data/lib/worker_army/web.rb

@@ -36,7 +36,7 @@ end
 
 get '/jobs/:job_id' do
   data = queue.job_data(params[:job_id])
-  json JSON.parse(data)
+  json data ? JSON.parse(data): {}
 end
 
 post '/callback' do

data/lib/worker_army/worker.rb

@@ -42,23 +42,27 @@ module WorkerArmy
         callback_url = data['callback_url']
         if @job and @job.class.name == data['job_class']
           @queue.add_current_job(job_id)
+          started_at = Time.now.utc.to_i
           response_data = @job.perform(data)
           response_data = {} unless response_data
-          response_data.merge!(job_id: job_id, callback_url: callback_url,
-            finished_at: Time.now.utc.to_i, host_name: @host_name)
-          @processed += 1
+          if callback_url and not callback_url.empty?
+            response_data.merge!(callback_url: callback_url)
+          end
+          response_data.merge!(job_id: job_id, started_at: started_at,
+            finished_at: Time.now.utc.to_i, host_name: @host_name,
+            worker_pid: Process.pid)
           if @worker_name
             response_data.merge!(worker_name: @worker_name)
           end
+          @processed += 1
         end
         @queue.remove_current_job(job_id)
-        response_data
       rescue => e
         @queue.remove_current_job(job_id)
         @log.error(e)
         retry_count += 1
         if retry_count < worker_retry_count(@config)
-          @log.debug("Failed! Retrying (#{retry_count})...")
+          @log.debug("Job execution failed! Retrying (#{retry_count})...")
           sleep (retry_count * 2)
           execute_job(list, element, retry_count)
         else
@@ -66,22 +70,24 @@ module WorkerArmy
           @queue.add_failed_job(job_id)
         end
       end
+      if callback_url and not callback_url.empty?
+        deliver_callback(data, response_data)
+      end
+      self.process_queue
+    end
+
+    def deliver_callback(data, response_data, retry_count = 0)
       begin
         response = RestClient.post data['callback_url'],
           response_data.to_json, :content_type => :json, :accept => :json
       rescue => e
         @log.error(e)
-      end
-      self.process_queue
-    end
-
-    def worker_retry_count(conf = nil)
-      if ENV['worker_army_worker_retry_count']
-        return ENV['worker_army_worker_retry_count'].to_i
-      elsif conf and conf['worker_retry_count']
-        return conf['worker_retry_count'].to_i
-      else
-        return 10
+        retry_count += 1
+        if retry_count < callback_retry_count(@config)
+          @log.debug("Delivering worker-army callback failed! Retrying (#{retry_count})...")
+          sleep (retry_count * 2)
+          deliver_callback(data, response_data, retry_count)
+        end
       end
     end
   end

data/worker-army.gemspec

@@ -2,22 +2,22 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: worker-army 0.4.0 ruby lib
+# stub: worker-army 0.4.1 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "worker-army"
-  s.version = "0.4.0"
+  s.version = "0.4.1"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib"]
   s.authors = ["Oliver Kiessler"]
-  s.date = "2014-05-29"
+  s.date = "2014-05-30"
   s.description = "Simple redis based worker queue with a HTTP/Rest interface"
   s.email = "kiessler@inceedo.com"
   s.executables = ["worker_army"]
   s.extra_rdoc_files = [
     "LICENSE.txt",
-    "README.rdoc"
+    "README.md"
   ]
   s.files = [
     ".document",
@@ -26,7 +26,7 @@ Gem::Specification.new do |s|
     "Gemfile.lock",
     "LICENSE.txt",
     "Procfile",
-    "README.rdoc",
+    "README.md",
     "Rakefile",
     "VERSION",
     "bin/worker_army",
@@ -44,7 +44,7 @@ Gem::Specification.new do |s|
     "test/test_worker-army.rb",
     "unicorn.rb",
     "worker-army.gemspec",
-    "worker-army.yml.sample"
+    "worker_army.yml.sample"
   ]
   s.homepage = "http://github.com/okiess/worker-army"
   s.licenses = ["MIT"]

data/{worker-army.yml.sample → worker_army.yml.sample}

@@ -4,3 +4,4 @@ redis_port: 6379
 redis_auth: "YOUR_PASS"
 client_retry_count: 10
 worker_retry_count: 10
+callback_retry_count: 5

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: worker-army
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.4.1
 platform: ruby
 authors:
 - Oliver Kiessler
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-05-29 00:00:00.000000000 Z
+date: 2014-05-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
@@ -157,7 +157,7 @@ executables:
 extensions: []
 extra_rdoc_files:
 - LICENSE.txt
-- README.rdoc
+- README.md
 files:
 - ".document"
 - ".rvmrc"
@@ -165,7 +165,7 @@ files:
 - Gemfile.lock
 - LICENSE.txt
 - Procfile
-- README.rdoc
+- README.md
 - Rakefile
 - VERSION
 - bin/worker_army
@@ -183,7 +183,7 @@ files:
 - test/test_worker-army.rb
 - unicorn.rb
 - worker-army.gemspec
-- worker-army.yml.sample
+- worker_army.yml.sample
 homepage: http://github.com/okiess/worker-army
 licenses:
 - MIT

data/README.rdoc

@@ -1,18 +0,0 @@
-= worker-army
-
-Simple redis based worker queue with a HTTP/Rest interface.
-
-== Contributing to worker-army
- 
-* Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet.
-* Check out the issue tracker to make sure someone already hasn't requested it and/or contributed it.
-* Fork the project.
-* Start a feature/bugfix branch.
-* Commit and push until you are happy with your contribution.
-* Make sure to add tests for it. This is important so I don't break it in a future version unintentionally.
-* Please try not to mess with the Rakefile, version, or history. If you want to have your own version, or is otherwise necessary, that is fine, but please isolate to its own commit so I can cherry-pick around it.
-
-== Copyright
-
-Copyright (c) 2013-2014 Oliver Kiessler. See LICENSE.txt for
-further details.