backburner 1.3.1 → 1.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ffcc49440304bb0d25a3710a3d94442fd1881f51
-  data.tar.gz: 2e9318efc9a04979135fecc8edfce4e8c8912386
+  metadata.gz: c37d62cede36418881281ed1e7da9293db41cc6d
+  data.tar.gz: 2348422b939c94a4fa7d2ed62bae148eac97d78f
 SHA512:
-  metadata.gz: caa6aa25343e41b2e8097be38d6b5679f304b9f417f2b0559e085caa2165660fc3ab132f5a920a49360faf16629b84b2d1b740ddf8332b0106df0437458a6f40
-  data.tar.gz: 457cb6f9ce73c900e0c54face79f78c8f0f6a89b1462580208f5afa9450f17c717fd7b2dd5701299addb334e260e55baa5c5360583867268c9f4ea921c94e38b
+  metadata.gz: ef295e9609ba6e52dea4c468dbcdc8b152ece6ef77d7202349aa49b9f496beee8cf71712982f3d248d80e8f17491b21810351a4c9912fd35e9a204878a6827ba
+  data.tar.gz: 63751243abb07484e4638cfb51832063b5a3f2c0cd8eb24e8b55f241b0fa68aff8c80690bbd759790c66797356205a6b0c04a72503bf40ebf0e9323e959982f4

data/.travis.yml

@@ -9,11 +9,14 @@ before_install:
   - make
   - ./beanstalkd &
   - cd $TRAVIS_BUILD_DIR
+  - gem update --system
+  - gem update bundler
+install:
+  - bundle install
 matrix:
   allow_failures:
     - rvm: rbx-2
 script:
-  - bundle install
   - bundle exec rake test
 gemfile: Gemfile
 notifications:

data/CHANGELOG.md

@@ -1,5 +1,12 @@
 # CHANGELOG
 
+## Version 1.4.0 (May 13 2017)
+
+* Fix unit tests to be more consistent (@eltone)
+* Ensure job supports body hash with symbol keys (@eltone)
+* Add support for custom serialization formats (@eltone)
+* Log the params when a job timeout occurs (@nathantsoi)
+
 ## Version 1.3.1 (April 21 2016)
 
 * Addition of thread-pool-based concurrency (@contentfree)

data/README.md

@@ -88,7 +88,7 @@ Backburner is extremely simple to setup. Just configure basic settings for backb
 
 ```ruby
 Backburner.configure do |config|
-  config.beanstalk_url       = ["beanstalk://127.0.0.1", "..."]
+  config.beanstalk_url       = "beanstalk://127.0.0.1"
   config.tube_namespace      = "some.app.production"
   config.namespace_separator = "."
   config.on_error            = lambda { |e| puts e }
@@ -102,6 +102,9 @@ Backburner.configure do |config|
   config.primary_queue       = "backburner-jobs"
   config.priority_labels     = { :custom => 50, :useless => 1000 }
   config.reserve_timeout     = nil
+  config.job_serializer_proc = lambda { |body| JSON.dump(body) }
+  config.job_parser_proc     = lambda { |body| JSON.parse(body) }
+
 end
 ```
 
@@ -109,7 +112,7 @@ The key options available are:
 
 | Option                | Description                                                          |
 | -----------------     | -------------------------------                                      |
-| `beanstalk_url`       | Address such as 'beanstalk://127.0.0.1' or an array of addresses.    |
+| `beanstalk_url`       | Address for beanstalkd connection i.e 'beanstalk://127.0.0.1'        |
 | `tube_namespace`      | Prefix used for all tubes related to this backburner queue.          |
 | `namespace_separator` | Separator used for namespace and queue name                          |
 | `on_error`            | Lambda invoked with the error whenever any job in the system fails.  |
@@ -123,6 +126,8 @@ The key options available are:
 | `primary_queue`       | Primary queue used for a job when an alternate queue is not given.   |
 | `priority_labels`     | Hash of named priority definitions for your app.                     |
 | `reserve_timeout`     | Duration to wait for work from a single server, or nil for forever.  |
+| `job_serializer_proc` | Lambda serializes a job body to a string to write to the task        |
+| `job_parser_proc`     | Lambda parses a task body string to a hash                           |
 
 ## Breaking Changes
 

data/backburner.gemspec

@@ -23,5 +23,4 @@ Gem::Specification.new do |s|
   s.add_development_dependency 'rake'
   s.add_development_dependency 'minitest', '3.2.0'
   s.add_development_dependency 'mocha'
-  s.add_development_dependency 'byebug'
 end

data/lib/backburner/configuration.rb

@@ -17,6 +17,8 @@ module Backburner
     attr_accessor :primary_queue       # the general queue
     attr_accessor :priority_labels     # priority labels
     attr_accessor :reserve_timeout     # duration to wait to reserve on a single server
+    attr_accessor :job_serializer_proc # proc to write the job body to a string
+    attr_accessor :job_parser_proc     # proc to parse a job body from a string
 
     def initialize
       @beanstalk_url       = "beanstalk://127.0.0.1"
@@ -34,6 +36,8 @@ module Backburner
       @primary_queue       = "backburner-jobs"
       @priority_labels     = PRIORITY_LABELS
       @reserve_timeout     = nil
+      @job_serializer_proc = lambda { |body| body.to_json }
+      @job_parser_proc     = lambda { |body| JSON.parse(body) }
     end
 
     def namespace_separator=(val)

data/lib/backburner/job.rb

@@ -21,8 +21,9 @@ module Backburner
     def initialize(task)
       @hooks = Backburner::Hooks
       @task = task
-      @body = task.body.is_a?(Hash) ? task.body : JSON.parse(task.body)
-      @name, @args = body["class"], body["args"]
+      @body = task.body.is_a?(Hash) ? task.body : Backburner.configuration.job_parser_proc.call(task.body)
+      @name = body["class"] || body[:class]
+      @args = body["args"] || body[:args]
     rescue => ex # Job was not valid format
       self.bury
       raise JobFormatInvalid, "Job body could not be parsed: #{ex.inspect}"
@@ -93,7 +94,7 @@ module Backburner
       begin
         Timeout::timeout(secs) { yield }
       rescue Timeout::Error => e
-        raise JobTimeout, "#{name} hit #{secs}s timeout.\nbacktrace: #{e.backtrace}"
+        raise JobTimeout, "#{name}(#{(@args||[]).join(', ')}) hit #{secs}s timeout.\nbacktrace: #{e.backtrace}"
       end
     end
 

data/lib/backburner/version.rb

@@ -1,3 +1,3 @@
 module Backburner
-  VERSION = "1.3.1"
+  VERSION = "1.4.0"
 end

data/lib/backburner/worker.rb

@@ -39,7 +39,8 @@ module Backburner
         connection = Backburner::Connection.new(Backburner.configuration.beanstalk_url)
         connection.retryable do
           tube = connection.tubes[expand_tube_name(queue || job_class)]
-          response = tube.put(data.to_json, :pri => pri, :delay => delay, :ttr => ttr)
+          serialized_data = Backburner.configuration.job_serializer_proc.call(data)
+          response = tube.put(serialized_data, :pri => pri, :delay => delay, :ttr => ttr)
         end
         return nil unless Backburner::Hooks.invoke_hook_events(job_class, :after_enqueue, *args)
       ensure

data/test/job_test.rb

@@ -15,16 +15,29 @@ describe "Backburner::Job module" do
   describe "for initialize" do
     describe "with hash" do
       before do
-        @task_body =  { "class" => "NewsletterSender", "args" => ["foo@bar.com", "bar@foo.com"] }
-        @task = stub(:body => @task_body, :ttr => 120, :delete => true, :bury => true)
+        @task = stub(:body => task_body, :ttr => 120, :delete => true, :bury => true)
       end
 
-      it "should create job with correct task data" do
-        @job = Backburner::Job.new(@task)
-        assert_equal @task, @job.task
-        assert_equal ["class", "args"], @job.body.keys
-        assert_equal @task_body["class"], @job.name
-        assert_equal @task_body["args"], @job.args
+      describe "with string keys" do
+        let(:task_body) { { "class" => "NewsletterSender", "args" => ["foo@bar.com", "bar@foo.com"] } }
+        it "should create job with correct task data" do
+          @job = Backburner::Job.new(@task)
+          assert_equal @task, @job.task
+          assert_equal ["class", "args"], @job.body.keys
+          assert_equal task_body["class"], @job.name
+          assert_equal task_body["args"], @job.args
+        end
+      end
+
+      describe "with symbol keys" do
+        let(:task_body) { { :class => "NewsletterSender", :args => ["foo@bar.com", "bar@foo.com"] } }
+        it "should create job with correct task data" do
+          @job = Backburner::Job.new(@task)
+          assert_equal @task, @job.task
+          assert_equal [:class, :args], @job.body.keys
+          assert_equal task_body[:class], @job.name
+          assert_equal task_body[:args], @job.args
+        end
       end
     end # with hash
 

data/test/test_helper.rb

@@ -92,12 +92,12 @@ class MiniTest::Spec
   end
 
   # pop_one_job(tube_name)
-  def pop_one_job(tube_name=Backburner.configuration.primary_queue, &block)
+  def pop_one_job(tube_name=Backburner.configuration.primary_queue)
     tube_name  = [Backburner.configuration.tube_namespace, tube_name].join(".")
     connection = beanstalk_connection
     connection.tubes.watch!(tube_name)
     silenced(3) { @res = connection.tubes.reserve }
-    yield @res, JSON.parse(@res.body)
+    yield @res, Backburner.configuration.job_parser_proc.call(@res.body)
   ensure
     connection.close if connection
   end

data/test/worker_test.rb

@@ -125,4 +125,33 @@ describe "Backburner::Worker module" do
       assert_equal ['baz', 'bam'], worker.tube_names
     end
   end # tube_names
+
+  describe "for custom serialization" do
+    before do
+      Backburner.configure do |config|
+        @old_parser = config.job_parser_proc
+        @old_serializer = config.job_serializer_proc
+        config.job_parser_proc = lambda { |body| Marshal.load(body) }
+        config.job_serializer_proc = lambda { |body| Marshal.dump(body) }
+      end
+    end
+
+    after do
+      clear_jobs!('test-plain')
+      Backburner.configure do |config|
+        config.job_parser_proc = @old_parser
+        config.job_serializer_proc = @old_serializer
+      end
+    end
+
+    it "should support enqueuing a job" do
+      Backburner::Worker.enqueue TestPlainJob, [7, 9], :ttr => 100, :pri => 2000
+      pop_one_job("test-plain") do |job, body|
+        assert_equal "TestPlainJob", body[:class]
+        assert_equal [7, 9], body[:args]
+        assert_equal 100, job.ttr
+        assert_equal 2000, job.pri
+      end
+    end
+  end # custom serialization
 end # Backburner::Worker

data/test/workers/forking_worker_test.rb

@@ -96,6 +96,7 @@ describe "Backburner::Workers::Forking module" do
 
     after do
       @templogger.close
+      Backburner.configuration.logger = nil
       clear_jobs!('response')
       clear_jobs!('bar.foo.1', 'bar.foo.2', 'bar.foo.3', 'bar.foo.4', 'bar.foo.5')
     end

data/test/workers/threading_worker_test.rb

@@ -54,6 +54,8 @@ describe "Backburner::Workers::Threading worker" do
     before do
       @worker = @worker_class.new(["foo:3"])
       capture_stdout { @worker.prepare }
+      $worker_test_count = 0
+      $worker_success = false
     end
 
     it 'runs work_on_job per thread' do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: backburner
 version: !ruby/object:Gem::Version
-  version: 1.3.1
+  version: 1.4.0
 platform: ruby
 authors:
 - Nathan Esquenazi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-10-02 00:00:00.000000000 Z
+date: 2017-05-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: beaneater
@@ -94,20 +94,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: byebug
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 description: Beanstalk background job processing made easy
 email:
 - nesquena@gmail.com
@@ -195,7 +181,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.8
+rubygems_version: 2.6.7
 signing_key: 
 specification_version: 4
 summary: Reliable beanstalk background job processing made easy for Ruby and Sinatra