chore-core 1.9.0 → 1.10.0

checksums.yaml

@@ -1,7 +1,15 @@
 ---
-SHA1:
-  metadata.gz: 0800ed952e9c16ccde3058ff9db8a4b9a72adf44
-  data.tar.gz: b24b312e93158e7942ae11896d9680f9f8a7714d
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    OWVhNTgxZTYxMzNkNDVjODU1NTI5YmJjOGYxOWU4NmUzNDkzMDljYw==
+  data.tar.gz: !binary |-
+    ZTliZWFkNzRhZDZlZDBiZWZlNjg4MDcxNDEzMTlhNDkwNmJkOWU4NA==
 SHA512:
-  metadata.gz: 86f131b78103e360f0219b7b55a53da68338e676791dedcea638130e38e132665a4d33cf2bd0f90e78957184dd8b9d19ea88b7801866097b276d113debb654bd
-  data.tar.gz: 5f32aa7d1db7aab0579b1bfb43c31497502f76964e71e4d9faf0d6c1230be157901694602cb8a76f82e6c7627d2a74337e82f377a71652f03cbd18f1dfd4b23f
+  metadata.gz: !binary |-
+    MDJkYmVlYWY4YTljMTA4ZjkxNGFlNGYxMDRiYThhNTJiYTJjNjhkYzVkNWFl
+    YTUzNjBlMDIxYjM1MTcxNjE4NzU1YjdkNGVmOWY1Y2IzODg1MTQ5YjIxZGUx
+    ZWI5N2U4ZGQwZGRhYTVkZTRiZmMxYzY0NGM1MjQ1YzM5NmJiZTM=
+  data.tar.gz: !binary |-
+    Mjc2ODk3ZmFjZjc2ZDdiODlkMTA4ODQzM2I1N2JiZDI5M2QwZjU2ODNkOWRl
+    NGY4ZDg3NDRiMDAzMTUxNzM0MGQxNTNiNDM4ZmMwMmVhOTQzZTgwOTc4ZTc2
+    MTg1YjI4Y2M1ZjE2YTM5ZDJiNTMwY2NkMjNjNTlmMTAyZmM1YTQ=

data/README.md

@@ -33,6 +33,7 @@ Other options include:
     --threads-per-queue 4 # number of threads per queue for consuming from a given queue.
     --dedupe-servers # if using SQS or similiar queue with at-least once delivery and your memcache is running on something other than localhost
     --batch-size 50 # how many messages are batched together before handing them to a worker
+    --batch-timeout 20 # maximum number of seconds to wait until handing a message over to a worker
     --queue_prefix prefixy # A prefix to prepend to queue names, mainly for development and qa testing purposes
     --max-attempts 100 # The maximum number of times a job can be attempted
     --dupe-on-cache-failure # Determines the deduping behavior when a cache connection error occurs. When set to `false`, the message is assumed not to be a duplicate. Defaults to `false`.
@@ -92,6 +93,7 @@ Chore.configure do |c|
   c.max_attempts = 100
   ...
   c.batch_size = 50
+  c.batch_timeout = 20
 end
 ```
 

data/lib/chore.rb

@@ -34,6 +34,7 @@ module Chore #:nodoc:
     :fetcher               => Fetcher,
     :consumer_strategy     => Strategy::ThreadedConsumerStrategy,
     :batch_size            => 50,
+    :batch_timeout         => 20,
     :log_level             => Logger::WARN,
     :log_path              => STDOUT,
     :default_queue_timeout => (12 * 60 * 60), # 12 hours
@@ -186,6 +187,7 @@ module Chore #:nodoc:
   #   Chore.configure do |c|
   #     c.consumer = Chore::Queues::SQS::Consumer
   #     c.batch_size = 50
+  #     c.batch_timeout = 20
   #   end
   def self.configure(opts={})
     @config = (@config ? @config.merge_hash(opts) : Chore::Configuration.new(DEFAULT_OPTIONS.merge(opts)))

data/lib/chore/strategies/consumer/batcher.rb

@@ -11,27 +11,27 @@ module Chore
         @size = size
         @batch = []
         @mutex = Mutex.new
-        @last_message = nil
         @callback = nil
         @running = true
       end
 
       # The main entry point of the Batcher, <tt>schedule</tt> begins a thread with the provided +batch_timeout+ 
       # as the only argument. While the Batcher is running, it will attempt to check if either the batch is full, 
-      # or if the +batch_timeout+ has elapsed since the last batch was executed. If the batch is full, it will be executed.
-      # If the +batch_timeout+ has elapsed, as soon as the next message enters the batch, it will be executed.
+      # or if the +batch_timeout+ has elapsed since the oldest message was added. If either case is true, the
+      # items in the batch will be executed.
       # 
       # Calling <tt>stop</tt> will cause the thread to finish it's current check, and exit
-      def schedule(batch_timeout=20)
+      def schedule(batch_timeout)
         @thread = Thread.new(batch_timeout) do |timeout|
           Chore.logger.info "Batching timeout thread starting"
           while @running do
             begin 
-              Chore.logger.debug "Last message added to batch: #{@last_message}: #{@batch.size}"
-              if @last_message && Time.now > (@last_message + timeout)
-                Chore.logger.debug "Batching timeout reached (#{@last_message + timeout}), current size: #{@batch.size}"
+              oldest_item = @batch.first
+              timestamp = oldest_item && oldest_item.created_at
+              Chore.logger.debug "Oldest message in batch: #{timestamp}, size: #{@batch.size}"
+              if timestamp && Time.now > (timestamp + timeout)
+                Chore.logger.debug "Batching timeout reached (#{timestamp + timeout}), current size: #{@batch.size}"
                 self.execute(true)
-                @last_message = nil
               end
               sleep(1) 
             rescue => e
@@ -44,7 +44,6 @@ module Chore
       # Adds the +item+ to the current batch
       def add(item)
         @batch << item
-        @last_message = Time.now
         execute if ready?
       end
 

data/lib/chore/strategies/consumer/threaded_consumer_strategy.rb

@@ -5,13 +5,14 @@ module Chore
       attr_accessor :batcher
 
       Chore::CLI.register_option 'batch_size', '--batch-size SIZE', Integer, 'Number of items to collect for a single worker to process'
+      Chore::CLI.register_option 'batch_timeout', '--batch-timeout SIZE', Integer, 'Maximum number of seconds to wait until processing a message'
       Chore::CLI.register_option 'threads_per_queue', '--threads-per-queue NUM_THREADS', Integer, 'Number of threads to create for each named queue'
 
       def initialize(fetcher)
         @fetcher = fetcher
         @batcher = Batcher.new(Chore.config.batch_size)
         @batcher.callback = lambda { |batch| @fetcher.manager.assign(batch) }
-        @batcher.schedule
+        @batcher.schedule(Chore.config.batch_timeout)
         @running = true
       end
 

data/lib/chore/unit_of_work.rb

@@ -9,6 +9,14 @@ module Chore
   # * +:consumer+ The consumer instance used to fetch this message. Most queue implementations won't need access to this, but some (RabbitMQ) will. So we
   # make sure to pass it along with each message. This instance will be used by the Worker for things like <tt>complete</tt> and </tt>reject</tt>.
   class UnitOfWork < Struct.new(:id,:queue_name,:queue_timeout,:message,:previous_attempts,:consumer,:decoded_message, :klass)
+    # The time at which this unit of work was created
+    attr_accessor :created_at
+
+    def initialize(*) #:nodoc:
+      super
+      @created_at = Time.now
+    end
+
     # The current attempt number for the worker processing this message.
     def current_attempt
       previous_attempts + 1

data/lib/chore/version.rb

@@ -1,7 +1,7 @@
 module Chore
   module Version #:nodoc:
     MAJOR = 1
-    MINOR = 9
+    MINOR = 10
     PATCH = 0
 
     STRING = [ MAJOR, MINOR, PATCH ].join('.')

data/spec/chore/strategies/consumer/batcher_spec.rb

@@ -90,4 +90,54 @@ describe Chore::Strategy::Batcher do
       subject.batch.should == ['test']
     end
   end
+
+  describe 'schedule' do
+    let(:timeout) { 5 }
+    let(:batch) { [] }
+
+    before(:each) do
+      Thread.stub(:new) do |&block|
+        # Stop the batcher on the next iteration
+        subject.stub(:sleep) { subject.stop }
+
+        # Run the scheduling thread
+        block.call(timeout)
+      end
+
+      subject.batch = batch.dup
+    end
+
+    context 'with no items' do
+      it 'should not invoke the callback' do
+        callback.should_not_receive(:call)
+        subject.schedule(timeout)
+      end
+    end
+
+    context 'with new items' do
+      let(:batch) do
+        [
+          Chore::UnitOfWork.new.tap {|work| work.created_at = Time.now - 2}
+        ]
+      end
+
+      it 'should not invoke the callback' do
+        callback.should_not_receive(:call).with(batch)
+        subject.schedule(timeout)
+      end
+    end
+
+    context 'with old items' do
+      let(:batch) do
+        [
+          Chore::UnitOfWork.new.tap {|work| work.created_at = Time.now - 6}
+        ]
+      end
+
+      it 'should invoke the callback' do
+        callback.should_receive(:call).with(batch)
+        subject.schedule(timeout)
+      end
+    end
+  end
 end

data/spec/chore/strategies/consumer/threaded_consumer_strategy_spec.rb

@@ -51,6 +51,7 @@ describe Chore::Strategy::ThreadedConsumerStrategy do
       work.message.should == "test"
       work.previous_attempts.should == 0
       work.current_attempt.should == 1
+      work.created_at.should_not be_nil
     end
   end
 

metadata

@@ -1,103 +1,103 @@
 --- !ruby/object:Gem::Specification
 name: chore-core
 version: !ruby/object:Gem::Version
-  version: 1.9.0
+  version: 1.10.0
 platform: ruby
 authors:
 - Tapjoy
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-03-09 00:00:00.000000000 Z
+date: 2016-03-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: json
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: aws-sdk-v1
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '1.56'
-    - - ">="
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: 1.56.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '1.56'
-    - - ">="
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: 1.56.0
 - !ruby/object:Gem::Dependency
   name: thread
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 0.1.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 0.1.3
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 3.3.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 3.3.0
 - !ruby/object:Gem::Dependency
   name: rdoc
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '3.12'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '3.12'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
 description: Job processing with pluggable backends and strategies
@@ -176,12 +176,12 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
@@ -191,3 +191,4 @@ signing_key:
 specification_version: 4
 summary: Job processing... for the future!
 test_files: []
+has_rdoc: