chore-core 1.10.0 → 4.0.0

data/spec/chore/strategies/worker/single_worker_strategy_spec.rb

@@ -3,7 +3,7 @@ require 'spec_helper'
 describe Chore::Strategy::SingleWorkerStrategy do
   let(:manager) { double('Manager') }
   let(:job_timeout) { 60 }
-  let(:job) { Chore::UnitOfWork.new(SecureRandom.uuid, 'test', job_timeout, Chore::Encoder::JsonEncoder.encode(TestJob.job_hash([1,2,"3"])), 0) }
+  let(:job) { Chore::UnitOfWork.new(SecureRandom.uuid, nil, 'test', job_timeout, Chore::Encoder::JsonEncoder.encode(TestJob.job_hash([1,2,"3"])), 0) }
   subject       { described_class.new(manager) }
 
   describe '#stop!' do

data/spec/chore/worker_spec.rb

@@ -2,6 +2,10 @@ require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
 
 describe Chore::Worker do
 
+  before(:each) do
+    allow(consumer).to receive(:duplicate_message?).and_return(false)
+  end
+
   class SimpleJob
     include Chore::Job
     queue_options :name => 'test',
@@ -13,6 +17,30 @@ describe Chore::Worker do
     end
   end
 
+  class SimpleDedupeJob
+    include Chore::Job
+    queue_options :name => 'dedupe_test',
+      :publisher => FakePublisher,
+      :max_attempts => 100,
+      :dedupe_lambda => lambda { |first, second, third| first }
+
+    def perform(first, second, third)
+      return second
+    end
+  end
+
+  class InvalidDedupeJob
+    include Chore::Job
+    queue_options :name => 'invalid_dedupe_test',
+      :publisher => FakePublisher,
+      :max_attempts => 100,
+      :dedupe_lambda => lambda { |first, second, third| first }
+
+    def perform(first, second)
+      return second
+    end
+  end
+
   let(:consumer) { double('consumer', :complete => nil, :reject => nil) }
   let(:job_args) { [1,2,'3'] }
   let(:job) { SimpleJob.job_hash(job_args) }
@@ -24,9 +52,9 @@ describe Chore::Worker do
 
   shared_examples_for "a worker" do
     it 'processing a single job' do
-      work = Chore::UnitOfWork.new('1', 'test', 60, encoded_job, 0, consumer)
+      work = Chore::UnitOfWork.new('1', nil, 'test', 60, encoded_job, 0, consumer)
       SimpleJob.should_receive(:perform).with(*payload)
-      consumer.should_receive(:complete).with('1')
+      consumer.should_receive(:complete).with('1', nil)
       w = Chore::Worker.new(work, {:payload_handler => payload_handler})
       w.start
     end
@@ -34,12 +62,39 @@ describe Chore::Worker do
     it 'processing multiple jobs' do
       work = []
       10.times do |i|
-        work << Chore::UnitOfWork.new(i, 'test', 60, encoded_job, 0, consumer)
+        work << Chore::UnitOfWork.new(i, nil, 'test', 60, encoded_job, 0, consumer)
       end
       SimpleJob.should_receive(:perform).exactly(10).times
       consumer.should_receive(:complete).exactly(10).times
       Chore::Worker.start(work, {:payload_handler => payload_handler})
     end
+
+    context 'when the job has a dedupe_lambda defined' do
+      context 'when the value being deduped on is unique' do
+        let(:job_args) { [rand,2,'3'] }
+        let(:encoded_job) { Chore::Encoder::JsonEncoder.encode(job) }
+        let(:job) { SimpleDedupeJob.job_hash(job_args) }
+        it 'should call complete for each unique value' do
+          allow(consumer).to receive(:duplicate_message?).and_return(false)
+          work = []
+          work << Chore::UnitOfWork.new(1, nil, 'dedupe_test', 60, Chore::Encoder::JsonEncoder.encode(SimpleDedupeJob.job_hash([rand,2,'3'])), 0, consumer)
+          SimpleDedupeJob.should_receive(:perform).exactly(1).times
+          consumer.should_receive(:complete).exactly(1).times
+          Chore::Worker.start(work, {:payload_handler => payload_handler})
+        end
+      end
+
+      context 'when the dedupe lambda does not take the same number of arguments as perform' do
+        it 'should raise an error and not complete the job' do
+          work = []
+          work << Chore::UnitOfWork.new(1, nil, 'invalid_dedupe_test', 60, Chore::Encoder::JsonEncoder.encode(InvalidDedupeJob.job_hash([rand,2,'3'])), 0, consumer)
+          work << Chore::UnitOfWork.new(2, nil, 'invalid_dedupe_test', 60, Chore::Encoder::JsonEncoder.encode(InvalidDedupeJob.job_hash([rand,2,'3'])), 0, consumer)
+          work << Chore::UnitOfWork.new(1, nil, 'invalid_dedupe_test', 60, Chore::Encoder::JsonEncoder.encode(InvalidDedupeJob.job_hash([rand,2,'3'])), 0, consumer)
+          consumer.should_not_receive(:complete)
+          Chore::Worker.start(work, {:payload_handler => payload_handler})
+        end
+      end
+    end
   end
 
   describe "with default payload handler" do
@@ -54,7 +109,7 @@ describe Chore::Worker do
     let(:queue_timeouts) { [10, 20, 30] }
     let(:work) do
       queue_timeouts.map do |queue_timeout|
-        Chore::UnitOfWork.new('1', 'test', queue_timeout, Chore::Encoder::JsonEncoder.encode(job), 0, consumer)
+        Chore::UnitOfWork.new('1', nil, 'test', queue_timeout, Chore::Encoder::JsonEncoder.encode(job), 0, consumer)
       end
     end
     let(:worker) do
@@ -85,14 +140,14 @@ describe Chore::Worker do
       let(:job) { "Not-A-Valid-Json-String" }
 
       it 'should fail cleanly' do
-        work = Chore::UnitOfWork.new(2,'test',60,job,0,consumer)
+        work = Chore::UnitOfWork.new(2,nil,'test',60,job,0,consumer)
         consumer.should_not_receive(:complete)
         Chore.should_receive(:run_hooks_for).with(:on_failure, job, anything())
         Chore::Worker.start(work)
       end
 
       it 'should reject job' do
-        work = Chore::UnitOfWork.new(2,'test',60,job,0,consumer)
+        work = Chore::UnitOfWork.new(2,nil,'test',60,job,0,consumer)
         consumer.should_receive(:reject).with(2)
         Chore::Worker.start(work)
       end
@@ -103,14 +158,14 @@ describe Chore::Worker do
         end
 
         it 'should permanently fail' do
-          work = Chore::UnitOfWork.new(2,'test',60,job,9,consumer)
+          work = Chore::UnitOfWork.new(2,nil,'test',60,job,9,consumer)
           Chore.should_receive(:run_hooks_for).with(:on_permanent_failure, 'test', job, anything())
           Chore::Worker.start(work)
         end
 
         it 'should mark the item as completed' do
-          work = Chore::UnitOfWork.new(2,'test',60,job,9,consumer)
-          consumer.should_receive(:complete).with(2)
+          work = Chore::UnitOfWork.new(2,nil,'test',60,job,9,consumer)
+          consumer.should_receive(:complete).with(2, nil)
           Chore::Worker.start(work)
         end
       end
@@ -126,7 +181,7 @@ describe Chore::Worker do
       end
 
       it 'should fail cleanly' do
-        work = Chore::UnitOfWork.new(2,'test',60,encoded_job,0,consumer)
+        work = Chore::UnitOfWork.new(2,nil,'test',60,encoded_job,0,consumer)
         consumer.should_not_receive(:complete)
         SimpleJob.should_receive(:run_hooks_for).with(:on_failure, parsed_job, anything())
 
@@ -134,7 +189,7 @@ describe Chore::Worker do
       end
 
       it 'should reject job' do
-        work = Chore::UnitOfWork.new(2,'test',60,encoded_job,0,consumer)
+        work = Chore::UnitOfWork.new(2,nil,'test',60,encoded_job,0,consumer)
         consumer.should_receive(:reject).with(2)
 
         Chore::Worker.start(work)
@@ -142,14 +197,14 @@ describe Chore::Worker do
 
       context 'more than the maximum allowed times' do
         it 'should permanently fail' do
-          work = Chore::UnitOfWork.new(2,'test',60,encoded_job,999,consumer)
+          work = Chore::UnitOfWork.new(2,nil,'test',60,encoded_job,999,consumer)
           SimpleJob.should_receive(:run_hooks_for).with(:on_permanent_failure, 'test', parsed_job, anything())
           Chore::Worker.start(work)
         end
 
         it 'should mark the item as completed' do
-          work = Chore::UnitOfWork.new(2,'test',60,encoded_job,999,consumer)
-          consumer.should_receive(:complete).with(2)
+          work = Chore::UnitOfWork.new(2,nil,'test',60,encoded_job,999,consumer)
+          consumer.should_receive(:complete).with(2, nil)
           Chore::Worker.start(work)
         end
       end
@@ -159,7 +214,7 @@ describe Chore::Worker do
   describe 'delaying retries' do
     let(:encoded_job) { Chore::Encoder::JsonEncoder.encode(job) }
     let(:parsed_job) { JSON.parse(encoded_job) }
-    let(:work) { Chore::UnitOfWork.new(2, 'test', 60, encoded_job, 0, consumer) }
+    let(:work) { Chore::UnitOfWork.new(2, nil, 'test', 60, encoded_job, 0, consumer) }
 
     before(:each) do
       SimpleJob.options[:backoff] = lambda { |work| work.current_attempt }

data/spec/spec_helper.rb

@@ -26,7 +26,7 @@ class FakePublisher < Chore::Publisher
   end
 end
 
-TestMessage = Struct.new(:handle, :queue, :body, :receive_count) do
+TestMessage = Struct.new(:handle, :receipt_handle, :queue, :body, :receive_count) do
   def empty?
     false
   end

data/spec/support/queues/sqs/fake_objects.rb

@@ -0,0 +1,18 @@
+describe Chore::Queues::SQS do
+  RSpec.shared_context 'fake objects' do
+    let(:queue_name) { 'test_queue' }
+    let(:queue_url) { "http://amazon.sqs.url/queues/#{queue_name}" }
+
+    let(:queue) do
+      double(Aws::SQS::Queue,
+        attributes: {'VisibilityTimeout' => rand(10)}
+      )
+    end
+
+    let(:sqs) do
+      double(Aws::SQS::Client,
+        get_queue_url: double(Aws::SQS::Types::GetQueueUrlResult, :queue_url => queue_url),
+      )
+    end
+  end
+end

metadata

@@ -1,103 +1,111 @@
 --- !ruby/object:Gem::Specification
 name: chore-core
 version: !ruby/object:Gem::Version
-  version: 1.10.0
+  version: 4.0.0
 platform: ruby
 authors:
 - Tapjoy
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-03-24 00:00:00.000000000 Z
+date: 2020-09-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: json
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: aws-sdk-v1
+  name: aws-sdk-sqs
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.56'
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: 1.56.0
+        version: '1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: '1.56'
-    - - ! '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.56.0
+        version: '1'
 - !ruby/object:Gem::Dependency
   name: thread
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.1.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.1.3
+- !ruby/object:Gem::Dependency
+  name: get_process_mem
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.0
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.3.0
+        version: '3.3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.3.0
+        version: '3.3'
 - !ruby/object:Gem::Dependency
   name: rdoc
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '3.12'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '3.12'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 description: Job processing with pluggable backends and strategies
@@ -137,7 +145,15 @@ files:
 - lib/chore/strategies/consumer/batcher.rb
 - lib/chore/strategies/consumer/single_consumer_strategy.rb
 - lib/chore/strategies/consumer/threaded_consumer_strategy.rb
+- lib/chore/strategies/consumer/throttled_consumer_strategy.rb
 - lib/chore/strategies/worker/forked_worker_strategy.rb
+- lib/chore/strategies/worker/helpers/ipc.rb
+- lib/chore/strategies/worker/helpers/preforked_worker.rb
+- lib/chore/strategies/worker/helpers/work_distributor.rb
+- lib/chore/strategies/worker/helpers/worker_info.rb
+- lib/chore/strategies/worker/helpers/worker_killer.rb
+- lib/chore/strategies/worker/helpers/worker_manager.rb
+- lib/chore/strategies/worker/preforked_worker_strategy.rb
 - lib/chore/strategies/worker/single_worker_strategy.rb
 - lib/chore/tasks/queues.task
 - lib/chore/unit_of_work.rb
@@ -160,11 +176,20 @@ files:
 - spec/chore/strategies/consumer/batcher_spec.rb
 - spec/chore/strategies/consumer/single_consumer_strategy_spec.rb
 - spec/chore/strategies/consumer/threaded_consumer_strategy_spec.rb
+- spec/chore/strategies/consumer/throttled_consumer_strategy_spec.rb
 - spec/chore/strategies/worker/forked_worker_strategy_spec.rb
+- spec/chore/strategies/worker/helpers/ipc_spec.rb
+- spec/chore/strategies/worker/helpers/preforked_worker_spec.rb
+- spec/chore/strategies/worker/helpers/work_distributor_spec.rb
+- spec/chore/strategies/worker/helpers/worker_info_spec.rb
+- spec/chore/strategies/worker/helpers/worker_killer_spec.rb
+- spec/chore/strategies/worker/helpers/worker_manager_spec.rb
+- spec/chore/strategies/worker/preforked_worker_strategy_spec.rb
 - spec/chore/strategies/worker/single_worker_strategy_spec.rb
 - spec/chore/worker_spec.rb
 - spec/chore_spec.rb
 - spec/spec_helper.rb
+- spec/support/queues/sqs/fake_objects.rb
 - spec/test_job.rb
 homepage: http://github.com/Tapjoy/chore
 licenses:
@@ -176,19 +201,18 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.8
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: Job processing... for the future!
 test_files: []
-has_rdoc: