em-bucketer 0.1.1 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: eaaf6c54984c095f03204b6682e10fb6a357ac82
-  data.tar.gz: 011accd36e8332183ddb1b4028416027c81e10cb
+  metadata.gz: aa43207fc878f4d9f0f15851590ec66803f45694
+  data.tar.gz: 9cd15644641ffccdd96552f45bbc5b420de5a262
 SHA512:
-  metadata.gz: a9ee2a1101c26baf9bce9383079a7192251231fccb02cceca8d46c2d90d6aa1e1cdeb9ce22228812fbfa56b707ae603a2b92d0241dc923e66828921a12847258
-  data.tar.gz: 985c4c1b96dee1b3ecba7e94aa7779c13733397aea0e3ef3fec52f63d9d9614e2f4894a95fd833453ca4892e0213bb6ce0f5f6c119b9181f6e443081b784a8a4
+  metadata.gz: 7b4ee20c2e7ad8cb933b5a0de2dfea6cd9b05ee7375cd156ea8659feda615890e8a6d52490e63dafb6d71e44b992eb33b066889b207b21efe693862fcff19fd7
+  data.tar.gz: 96447882b8d6cdf6a20dc62a25f69af698de3708eb523735fcec40ae503edab3237b919ba72aa69ae9861d2350e7ec9b3a0110383eb5e857db439ed075e44a0d

data/README.md

@@ -21,7 +21,43 @@ Or install it yourself as:
 
     $ gem install em-bucketer
 
-## Usage
+## Ordered Bucketer
+The ordered bucketer adds items to buckets and gives them back in the same
+order in which they were put in.
+
+### Usage
+
+```ruby
+require 'em-bucketer'
+EM.run do
+  bucketer = EM::Bucketer::Ordered::InMemory.new(:bucket_threshold_size => 5)
+
+  bucketer.on_bucket_full do |bucket_id|
+    p "yay bucket #{bucket_id} filled up!"
+
+    bucketer.pop_all(bucket_id) do |items|
+      EM.stop
+      items.each do |item|
+        p "got back #{item}"
+      end
+    end
+  end
+
+  bucketer.add_item("1", {:foo => :bar})
+  bucketer.add_item("1", {:foo => :bar})
+  bucketer.add_item("1", {:foo => :bar})
+  bucketer.add_item("1", {:bar => :foo})
+  bucketer.add_item("1", {:bar => :foo})
+end
+```
+
+## Unordered Bucketer
+The unordered bucketer requires you to pass in an 'id' which is used to ensure
+that no duplicates occur. This bucketer type should be used if you want to not
+have two of the same item in a bucket at one time. This type of bucketer does
+not guarantee that you get back items in the same order that they went in.
+
+### Usage
 
 ```ruby
 require 'em-bucketer'

data/lib/em-bucketer/base.rb

@@ -153,11 +153,9 @@ module EventMachine::Bucketer
     private
 
     def get_and_remove_iterator(bucket_id, count, values, completion)
-      added = 0
       proc do |tuple, iter|
         key, val = tuple[0], tuple[1]
-        if added < count
-          added += 1
+        if values.count < count
           values << val
           iter.next
         else

data/lib/em-bucketer/ordered.rb

@@ -0,0 +1,8 @@
+module EventMachine::Bucketer
+  module Ordered
+  end
+end
+
+require 'eventmachine'
+require 'em-bucketer/ordered/in_memory'
+#require 'em-bucketer/ordered/redis'

data/lib/em-bucketer/ordered/base.rb

@@ -0,0 +1,191 @@
+module EventMachine::Bucketer
+  module Ordered::Base
+    def setup(bucket_threshold_size, bucket_max_age)
+      @bucket_threshold_size = bucket_threshold_size
+      @bucket_max_age = bucket_max_age
+      @buckets = {}
+      @on_bucket_full_callbacks = []
+      @on_bucket_timeout_callbacks = []
+      @timers = {}
+    end
+
+    # Adds a item to the specified bucket and
+    # calls the block when it is done
+    #
+    # @param bucket_id [String] the bucket id of
+    # the bucket to put the item in
+    # @param item [Object] the item to be
+    # placed in the bucket
+    def add_item(bucket_id, item, &blk)
+      add_timer_if_first(bucket_id)
+      EM::Completion.new.tap do |c|
+        c.callback(&blk) if block_given?
+        add_item_to_db(bucket_id, item).callback do
+          c.succeed
+          check_bucket_full(bucket_id)
+        end.errback do |e|
+          c.fail e
+        end
+      end
+    end
+
+    # Get at most `count` items back from a
+    # specific bucket and remove them from
+    # the bucket. These should be the first
+    # `count` items you added to the bucket
+    # that have not yet been removed from the
+    # bucket.
+    #
+    # @param bucket_id [String] the bucket id
+    # you want the items from
+    # @param count [Integer] the number of
+    # items you want from the bucket
+    # @yield [Array] the first `count`
+    # items in the bucket
+    def pop_count(bucket_id, count, reset_timer: true, &blk)
+      EM::Completion.new.tap do |c|
+        c.callback(&blk) if block_given?
+        pop_count_from_db(bucket_id, count, &blk).callback do |items|
+          reset_timer(bucket_id) if reset_timer
+          c.succeed items
+        end.errback do |e|
+          c.fail e
+        end
+      end
+    end
+
+    # Get all items back from a specific bucket
+    # and remove them from the bucket.
+    #
+    # @param bucket_id [String] the bucket id
+    # you want the items from
+    # @yield [Array] all items in the bucket
+    def pop_all(bucket_id, &blk)
+      clear_timer(bucket_id)
+      EM::Completion.new.tap do |c|
+        c.callback(&blk) if block_given?
+        pop_all_from_db(bucket_id, &blk).callback do |items|
+          c.succeed items
+        end.errback do |e|
+          c.fail e
+        end
+      end
+    end
+
+    # Used to set a callback hook for when a bucket
+    # reaches the threshold size. It is IMPORTANT
+    # to note that the bucket will not automatically
+    # be emptied you must call empty_bucket if you
+    # want the bucket to be emptied. Also the callback
+    # will be called every time a item is added
+    # until the bucket is emptied.
+    #
+    # @yield [String] The bucket id of the full bucket
+    def on_bucket_full(&blk)
+      @on_bucket_full_callbacks << blk
+    end
+
+    # Used to set a callback hook for when a bucket
+    # reaches the time limit. It is IMPORTANT
+    # to note that the bucket will not automatically
+    # be emptied you must call empty_bucket if you
+    # want the bucket to be emptied.
+    #
+    # This timer is started once the bucket gets its
+    # first item and is cleared only when the
+    # bucket is emptied. The callback will only be
+    # called once at this time and then not again
+    # unless you empty the bucket and add something
+    # again.
+    #
+    # @yield [String] The bucket id of the full bucket
+    def on_bucket_timeout(&blk)
+      @on_bucket_timeout_callbacks << blk
+    end
+
+    # Get the contents of a bucket.
+    #
+    # @param bucket_id [String] the bucket id
+    # of the bucket you want to get
+    # @yield [Array] the items you put
+    # into the bucket
+    def get_bucket(bucket_id, &blk)
+      get_bucket_from_db(bucket_id, &blk)
+    end
+
+    # Empty a bucket
+    #
+    # @param bucket_id [String] the bucket id
+    # of the bucket you want to empty
+    def empty_bucket(bucket_id, &blk)
+      EM::Completion.new.tap do |c|
+        c.callback(&blk) if block_given?
+        empty_bucket_in_db(bucket_id).callback do
+          clear_timer(bucket_id)
+          c.succeed
+        end.errback do |e|
+          c.fail e
+        end
+      end
+    end
+
+    private
+
+    def get_and_remove_iterator(bucket_id, count, values, completion)
+      proc do |tuple, iter|
+        key, val = tuple[0], tuple[1]
+        if values.count < count
+          values << val
+          iter.next
+        else
+          add_item(bucket_id, key, val).callback do
+            iter.next
+          end.errback do |e|
+            completion.fail e
+          end
+        end
+      end
+    end
+
+    def bucket_full?(bucket_id, &blk)
+      bucket_size_from_db(bucket_id).callback do |size|
+        blk.call size >= @bucket_threshold_size
+      end
+    end
+
+    def check_bucket_full(bucket_id)
+      bucket_full?(bucket_id) do |is_full|
+        if is_full
+          @on_bucket_full_callbacks.each do |callback|
+            callback.call bucket_id
+          end
+        end
+      end
+    end
+
+    def add_timer_if_first(bucket_id)
+      return unless @bucket_max_age
+      @timers[bucket_id] ||= EM::Timer.new(@bucket_max_age, timeout_callback(bucket_id))
+    end
+
+    def timeout_callback(bucket_id)
+      proc do |bar|
+        @on_bucket_timeout_callbacks.each do |callback|
+          callback.call bucket_id
+        end
+      end
+    end
+
+    def clear_timer(bucket_id)
+      return unless @bucket_max_age
+      timer = @timers.delete(bucket_id)
+      timer.cancel if timer
+    end
+
+    def reset_timer(bucket_id)
+      return unless @bucket_max_age
+      clear_timer(bucket_id)
+      @timers[bucket_id] = EM::Timer.new(@bucket_max_age, timeout_callback(bucket_id))
+    end
+  end
+end

data/lib/em-bucketer/ordered/database.rb

@@ -0,0 +1,6 @@
+module EventMachine::Bucketer
+  module Ordered
+    module Database
+    end
+  end
+end

data/lib/em-bucketer/ordered/database/hash.rb

@@ -0,0 +1,64 @@
+require 'em-bucketer/ordered/database'
+
+module EventMachine::Bucketer
+  module Ordered
+    module Database
+      module Hash
+        private
+
+        def pop_all_from_db(bucket_id, &blk)
+          EM::Completion.new.tap do |c|
+            c.callback(&blk) if block_given?
+            all = bucket_by_id(bucket_id)
+            @buckets[bucket_id] = []
+            c.succeed all
+          end
+        end
+
+        def pop_count_from_db(bucket_id, count, &blk)
+          EM::Completion.new.tap do |c|
+            c.callback(&blk) if block_given?
+            all = bucket_by_id(bucket_id)
+            result = all.first(count)
+            @buckets[bucket_id] = all[count..-1]
+            c.succeed result
+          end
+        end
+
+        def bucket_size_from_db(bucket_id, &blk)
+          EM::Completion.new.tap do |c|
+            c.callback(&blk) if block_given?
+            c.succeed bucket_by_id(bucket_id).count
+          end
+        end
+
+        def add_item_to_db(bucket_id, item, &blk)
+          EM::Completion.new.tap do |c|
+            c.callback(&blk) if block_given?
+            bucket_by_id(bucket_id) << item
+            c.succeed
+          end
+        end
+
+        def get_bucket_from_db(bucket_id, &blk)
+          EM::Completion.new.tap do |c|
+            c.callback(&blk) if block_given?
+            c.succeed bucket_by_id(bucket_id)
+          end
+        end
+
+        def empty_bucket_in_db(bucket_id, &blk)
+          EM::Completion.new.tap do |c|
+            c.callback(&blk) if block_given?
+            @buckets[bucket_id] = []
+            c.succeed
+          end
+        end
+
+        def bucket_by_id(bucket_id)
+          @buckets[bucket_id] ||= []
+        end
+      end
+    end
+  end
+end

data/lib/em-bucketer/ordered/in_memory.rb

@@ -0,0 +1,24 @@
+require 'eventmachine'
+require 'em-bucketer/ordered/database/hash'
+require 'em-bucketer/ordered/base'
+
+module EventMachine::Bucketer
+  class Ordered::InMemory
+    include Ordered::Database::Hash
+    include Ordered::Base
+
+    BUCKET_THRESHOLD_SIZE_DEFAULT = 1000
+    BUCKET_MAX_AGE_DEFAULT = 3600
+
+    # Creates a new in memory Bucketer with the requested
+    # configurations
+    #
+    # @param bucket_threshold_size [Integer] the max size of the bucket
+    # after which the on_bucket_full callback is called
+    # @param bucket_max_age [Integer] max number of seconds a bucket
+    # can remain before the on_bucket_timed_out is called
+    def initialize(bucket_threshold_size: BUCKET_THRESHOLD_SIZE_DEFAULT, bucket_max_age: BUCKET_MAX_AGE_DEFAULT)
+      setup(bucket_threshold_size, bucket_max_age)
+    end
+  end
+end

data/lib/em-bucketer/version.rb

@@ -1,5 +1,5 @@
 module EventMachine
   module Bucketer
-    VERSION = "0.1.1"
+    VERSION = "0.2.0"
   end
 end

data/spec/em_bucketer_ordered_examples.rb

@@ -0,0 +1,133 @@
+require 'spec_helper'
+require 'em-bucketer/ordered'
+
+shared_examples "an ordered bucketer" do
+  describe '#add_item' do
+    it 'adds a item to the bucket' do
+      EM.run do
+        EM.add_timer(0.1) { fail "didn't reach EM.stop" }
+        bucketer.add_item("1", {:foo => :bar}) do
+          bucketer.get_bucket("1") do |bucket|
+            expect(bucket).to eq([{:foo => :bar}])
+            EM.stop
+          end
+        end
+      end
+    end
+
+    it 'handles multiple buckets' do
+      EM.run do
+        EM.add_timer(0.1) { fail "didn't reach EM.stop" }
+        bucketer.on_bucket_full do |bucket_id|
+          fail "shouldn't have called full"
+        end
+
+        add_n_items_ordered(bucketer, "1", 3) do
+          add_n_items_ordered(bucketer, "2", 3) do
+
+            bucketer.get_bucket("1") do |bucket|
+              expect(bucket).to eq([{:id => 0}, {:id => 1}, {:id => 2}])
+              EM.stop
+            end
+          end
+        end
+      end
+    end
+
+    it 'calls on_bucket_full when a bucket fills up' do
+      EM.run do
+        EM.add_timer(0.1) { fail "didn't reach EM.stop" }
+        bucketer.on_bucket_full do |bucket_id|
+          expect(bucket_id).to eq("1")
+          EM.stop
+        end
+
+        add_n_items_ordered(bucketer, "1", 5)
+      end
+    end
+  end
+
+  describe '#empty_bucket' do
+    it 'emptys a bucket' do
+      EM.run do
+        EM.add_timer(0.1) { fail "didn't reach EM.stop" }
+        add_n_items_ordered(bucketer, "1", 3) do
+          bucketer.empty_bucket("1") do
+            bucketer.get_bucket("1") do |bucket|
+              expect(bucket).to eq([])
+              EM.stop
+            end
+          end
+        end
+      end
+    end
+  end
+
+  describe '#on_bucket_timeout' do
+    it 'calls the block when the timer times out' do
+      ran = false
+      EM.run do
+        EM.add_timer(0.05) { EM.stop }
+        # Stub out the timer that the bucketer uses
+        allow(EM).to receive(:add_timer) { |age, callback| callback.call }
+        bucketer.on_bucket_timeout do |bucket_id|
+          ran = true
+          expect(bucket_id).to eq("1")
+        end
+        bucketer.add_item("1", :foo => :bar)
+      end
+      fail "didn't call timeout" unless ran
+    end
+
+    it 'doesnt call the block when the timer doesnt time out' do
+      EM.run do
+        EM.add_timer(0.1) { EM.stop }
+        allow(EM).to receive(:add_timer)
+        bucketer.on_bucket_timeout do |bucket_id|
+          fail "shouldn't have called timeout"
+        end
+        bucketer.add_item("1", :foo => :bar)
+      end
+    end
+  end
+
+  describe '#pop_all' do
+    it 'gets and emptys the bucket' do
+      EM.run do
+        EM.add_timer(0.1) { fail "didn't reach EM.stop" }
+        add_n_items_ordered(bucketer, "1", 3) do
+
+          bucketer.pop_all("1") do |bucket|
+            expect(bucket).to eq([{:id => 0}, {:id => 1}, {:id => 2}])
+
+            bucketer.get_bucket("1") do |empty_bucket|
+              expect(empty_bucket).to eq([])
+              EM.stop
+            end
+          end
+        end
+      end
+    end
+  end
+
+  describe '#pop_count' do
+    it 'gets 2 items and removes them' do
+      EM.run do
+        EM.add_timer(0.1) { fail "didn't reach EM.stop" }
+        add_n_items_ordered(bucketer, "1", 3) do
+          bucketer.get_bucket("1") do |total_bucket|
+            expect(total_bucket.count).to eq(3)
+            bucketer.pop_count("1", 2) do |bucket|
+              expect(bucket.count).to eq(2)
+
+              bucketer.get_bucket("1") do |remaining_bucket|
+                expect(remaining_bucket.count).to eq(1)
+                EM.stop
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/spec/in_memory_ordered_spec.rb

@@ -0,0 +1,8 @@
+require 'spec_helper'
+require 'em-bucketer/ordered'
+
+describe EventMachine::Bucketer::Ordered::InMemory do
+  it_behaves_like "an ordered bucketer" do
+    let(:bucketer) { EM::Bucketer::Ordered::InMemory.new(:bucket_threshold_size => 5) }
+  end
+end

data/spec/spec_helper.rb

@@ -4,6 +4,7 @@ require 'rspec'
 require 'pry'
 require 'spec_methods'
 require 'em_bucketer_examples'
+require 'em_bucketer_ordered_examples'
 
 RSpec.configure do |config|
   config.order = :rand

data/spec/spec_methods.rb

@@ -7,4 +7,13 @@ module SpecMethods
     end
     EM::Iterator.new(0...n).each(worker, blk)
   end
+
+  def add_n_items_ordered(bucketer, bucket, n, &blk)
+    worker = proc do |i, iter|
+      bucketer.add_item(bucket, {:id => i}) do
+        iter.next
+      end
+    end
+    EM::Iterator.new(0...n).each(worker, blk)
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: em-bucketer
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.0
 platform: ruby
 authors:
 - Richard Heycock
@@ -143,9 +143,16 @@ files:
 - lib/em-bucketer/database/hash.rb
 - lib/em-bucketer/database/redis.rb
 - lib/em-bucketer/in_memory.rb
+- lib/em-bucketer/ordered.rb
+- lib/em-bucketer/ordered/base.rb
+- lib/em-bucketer/ordered/database.rb
+- lib/em-bucketer/ordered/database/hash.rb
+- lib/em-bucketer/ordered/in_memory.rb
 - lib/em-bucketer/redis.rb
 - lib/em-bucketer/version.rb
 - spec/em_bucketer_examples.rb
+- spec/em_bucketer_ordered_examples.rb
+- spec/in_memory_ordered_spec.rb
 - spec/in_memory_spec.rb
 - spec/redis_spec.rb
 - spec/spec_helper.rb
@@ -177,6 +184,8 @@ summary: A generic eventmachine library for storing arbitrary objects in buckets
   callbacks on threshold reached
 test_files:
 - spec/em_bucketer_examples.rb
+- spec/em_bucketer_ordered_examples.rb
+- spec/in_memory_ordered_spec.rb
 - spec/in_memory_spec.rb
 - spec/redis_spec.rb
 - spec/spec_helper.rb