em-bucketer 0.1.0

data/README.md

@@ -0,0 +1,63 @@
+# EventMachine::Bucketer
+
+This is a generic EventMachine library for putting arbitrary objects into
+buckets and setting callbacks to be called when any bucket exceeds a specific
+threshold size. Although the `Bucketer::InMemory` is synchronous (it's just
+using a ruby hash) the interface is still what would be expected for an
+asynchronous API for consistency with other Bucketers that are actually
+asynchronous.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'em-bucketer'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install em-bucketer
+
+## Usage
+
+```ruby
+require 'em-bucketer'
+EM.run do
+  bucketer = EM::Bucketer::InMemory.new(:bucket_threshold_size => 5)
+
+  bucketer.on_bucket_full do |bucket_id|
+    p "yay bucket #{bucket_id} filled up!"
+
+    bucketer.get_and_empty_bucket(bucket_id) do |items|
+      EM.stop
+      items.each do |item|
+        p "got back #{item}"
+      end
+    end
+  end
+
+  bucketer.add_item("1", "1", {:foo => :bar})
+  bucketer.add_item("1", "2", {:foo => :bar})
+  bucketer.add_item("1", "3", {:foo => :bar})
+  bucketer.add_item("1", "4", {:bar => :foo})
+  bucketer.add_item("1", "5", {:bar => :foo})
+end
+```
+
+## Redis Bucketer
+
+This gem also supports a redis backed bucketer which uses the `em-hiredis` gem.
+This bucketer uses `Marshal.dump` to store objects in redis and thus there are
+limitations on what can be placed in a bucket. Specifically you cannot store
+procs in buckets using the redis bucketer.
+
+## Contributing
+
+1. Fork it ( https://github.com/dgvz/em-bucketer/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+

data/em-bucketer.gemspec

@@ -0,0 +1,29 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'em-bucketer/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "em-bucketer"
+  spec.version       = EventMachine::Bucketer::VERSION
+  spec.authors       = ["Richard Heycock", "Dylan Griffith"]
+  spec.email         = ["dyl.griffith@gmail.com"]
+  spec.summary       = %q{A generic eventmachine library for storing arbitrary objects in buckets with callbacks on threshold reached}
+  spec.homepage      = "https://github.com/dgvz/em-bucketer"
+  spec.license       = "GPL"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_runtime_dependency     "eventmachine"
+  spec.add_runtime_dependency     "em-hiredis"
+
+  spec.add_development_dependency "bundler", "~> 1.6"
+  spec.add_development_dependency "redis"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rspec"
+  spec.add_development_dependency "pry"
+  spec.add_development_dependency "yard"
+end

data/lib/em-bucketer.rb

@@ -0,0 +1,8 @@
+require "em-bucketer/version"
+
+module EventMachine::Bucketer
+end
+
+require 'eventmachine'
+require 'em-bucketer/in_memory'
+require 'em-bucketer/redis'

data/lib/em-bucketer/base.rb

@@ -0,0 +1,153 @@
+module EventMachine::Bucketer
+  module Base
+    def setup(bucket_threshold_size, bucket_max_age)
+      @bucket_threshold_size = bucket_threshold_size
+      @bucket_max_age = bucket_max_age
+      @buckets = {}
+      @on_bucket_full_callbacks = []
+      @on_bucket_timeout_callbacks = []
+      @buckets_with_timers = Set.new
+    end
+
+    # Adds a item to the specified bucket and
+    # calls the block when it is done
+    #
+    # @param bucket_id [String] the bucket id of
+    # the bucket to put the item in
+    # @param item_id [String] the item_id
+    # of the item (used to ensure uniqueness
+    # within a bucket)
+    # @param item [Object] the item to be
+    # placed in the bucket
+    def add_item(bucket_id, item_id, item, &blk)
+      add_timer_if_first(bucket_id)
+      EM::Completion.new.tap do |c|
+        c.callback(&blk) if block_given?
+        add_bucket_to_db(bucket_id, item_id, item).callback do
+          c.succeed
+        end.errback do |e|
+          c.fail e
+        end
+        check_bucket_full(bucket_id)
+      end
+    end
+
+    # Used to set a callback hook for when a bucket
+    # reaches the threshold size. It is IMPORTANT
+    # to note that the bucket will not automatically
+    # be emptied you must call empty_bucket if you
+    # want the bucket to be emptied. Also the callback
+    # will be called every time a item is added
+    # until the bucket is emptied.
+    #
+    # @yield [String] The bucket id of the full bucket
+    def on_bucket_full(&blk)
+      @on_bucket_full_callbacks << blk
+    end
+
+    # Used to set a callback hook for when a bucket
+    # reaches the time limit. It is IMPORTANT
+    # to note that the bucket will not automatically
+    # be emptied you must call empty_bucket if you
+    # want the bucket to be emptied.
+    #
+    # This timer is started once the bucket gets its
+    # first item and is cleared only when the
+    # bucket is emptied. The callback will only be
+    # called once at this time and then not again
+    # unless you empty the bucket and add something
+    # again.
+    #
+    # @yield [String] The bucket id of the full bucket
+    def on_bucket_timeout(&blk)
+      @on_bucket_timeout_callbacks << blk
+    end
+
+    # Get the contents of a bucket.
+    #
+    # @param bucket_id [String] the bucket id
+    # of the bucket you want to get
+    # @yield [Array] the items you put
+    # into the bucket
+    def get_bucket(bucket_id, &blk)
+      EM::Completion.new.tap do |c|
+        c.callback(&blk) if block_given?
+        get_bucket_from_db(bucket_id).callback do |bucket|
+          c.succeed bucket.values
+        end.errback do |e|
+          c.fail e
+        end
+      end
+    end
+
+    # Get the contents of a bucket then empty it
+    #
+    # @param bucket_id [String] the bucket id
+    # of the bucket you want to get
+    # @yield [Array] the items you put
+    # into the bucket
+    def get_and_empty_bucket(bucket_id, &blk)
+      EM::Completion.new.tap do |c|
+        c.callback(&blk) if block_given?
+        get_bucket(bucket_id).callback do |contents|
+          empty_bucket(bucket_id).callback do
+            c.succeed contents
+          end.errback do |e|
+            c.fail e
+          end
+        end.errback do |e|
+          c.fail e
+        end
+      end
+    end
+
+    # Empty a bucket
+    #
+    # @param bucket_id [String] the bucket id
+    # of the bucket you want to empty
+    def empty_bucket(bucket_id, &blk)
+      EM::Completion.new.tap do |c|
+        c.callback(&blk) if block_given?
+        empty_bucket_in_db(bucket_id).callback do
+          clear_timer(bucket_id)
+          c.succeed
+        end.errback do |e|
+          c.fail e
+        end
+      end
+    end
+
+    private
+
+    def bucket_full?(bucket_id, &blk)
+      bucket_size_from_db(bucket_id).callback do |size|
+        blk.call size >= @bucket_threshold_size
+      end
+    end
+
+    def check_bucket_full(bucket_id)
+      bucket_full?(bucket_id) do |is_full|
+        if is_full
+          @on_bucket_full_callbacks.each do |callback|
+            callback.call bucket_id
+          end
+        end
+      end
+    end
+
+    def add_timer_if_first(bucket_id)
+      return unless @bucket_max_age
+      if @buckets_with_timers.add?(bucket_id)
+        EM.add_timer(@bucket_max_age) do
+          @on_bucket_timeout_callbacks.each do |callback|
+            callback.call bucket_id
+          end
+        end
+      end
+    end
+
+    def clear_timer(bucket_id)
+      @buckets_with_timers.delete(bucket_id)
+    end
+  end
+end

data/lib/em-bucketer/database.rb

@@ -0,0 +1,2 @@
+module Database
+end

data/lib/em-bucketer/database/hash.rb

@@ -0,0 +1,42 @@
+require 'em-bucketer/database'
+
+module EventMachine::Bucketer
+  module Database
+    module Hash
+      private
+
+      def bucket_size_from_db(bucket_id, &blk)
+        EM::Completion.new.tap do |c|
+          c.callback(&blk) if block_given?
+          @buckets[bucket_id] ||= {}
+          c.succeed @buckets[bucket_id].size
+        end
+      end
+
+      def add_bucket_to_db(bucket_id, item_id, item, &blk)
+        EM::Completion.new.tap do |c|
+          c.callback(&blk) if block_given?
+          @buckets[bucket_id] ||= {}
+          @buckets[bucket_id][item_id] = item
+          c.succeed
+        end
+      end
+
+      def get_bucket_from_db(bucket_id, &blk)
+        EM::Completion.new.tap do |c|
+          c.callback(&blk) if block_given?
+          @buckets[bucket_id] ||= {}
+          c.succeed @buckets[bucket_id]
+        end
+      end
+
+      def empty_bucket_in_db(bucket_id, &blk)
+        EM::Completion.new.tap do |c|
+          c.callback(&blk) if block_given?
+          @buckets[bucket_id] = {}
+          c.succeed
+        end
+      end
+    end
+  end
+end

data/lib/em-bucketer/database/redis.rb

@@ -0,0 +1,88 @@
+require 'em-bucketer/database'
+require 'em-hiredis'
+
+module EventMachine::Bucketer
+  module Database
+    module Redis
+      private
+
+      def bucket_size_from_db(bucket_id, &blk)
+        EM::Completion.new.tap do |c|
+          c.callback(&blk) if block_given?
+          redis.hlen(redis_key(bucket_id)).callback do |len|
+            c.succeed len.to_i
+          end.errback do |e|
+            c.fail e
+          end
+        end
+      end
+
+      def add_bucket_to_db(bucket_id, item_id, item, &blk)
+        EM::Completion.new.tap do |c|
+          c.callback(&blk) if block_given?
+          redis.hset(redis_key(bucket_id), item_id, Marshal.dump(item)).callback do
+            add_to_known_buckets(bucket_id).callback do
+              c.succeed
+            end.errback do |e|
+              c.fail e
+            end
+          end.errback do |e|
+            c.fail e
+          end
+        end
+      end
+
+      def get_bucket_from_db(bucket_id, &blk)
+        EM::Completion.new.tap do |c|
+          c.callback(&blk) if block_given?
+          redis.hgetall(redis_key(bucket_id)) do |data|
+            bucket = {}
+            index = 0
+            while(index < data.size)
+              bucket[data[index]] = Marshal.load(data[index + 1])
+              index += 2
+            end
+            c.succeed bucket
+          end.errback do |e|
+            c.errback
+          end
+        end
+      end
+
+      def empty_bucket_in_db(bucket_id, &blk)
+        EM::Completion.new.tap do |c|
+          c.callback(&blk) if block_given?
+          redis.del(redis_key(bucket_id)).callback do
+            remove_from_known_buckets(bucket_id).callback do
+              c.succeed
+            end.errback do |e|
+              c.fail e
+            end
+          end.errback do |e|
+            c.fail e
+          end
+        end
+      end
+
+      def known_buckets(&blk)
+        redis.smembers(redis_known_buckets_key, &blk)
+      end
+
+      def add_to_known_buckets(bucket_id, &blk)
+        redis.sadd(redis_known_buckets_key, bucket_id, &blk)
+      end
+
+      def remove_from_known_buckets(bucket_id, &blk)
+        redis.srem(redis_known_buckets_key, bucket_id, &blk)
+      end
+
+      def redis_key(bucket_id)
+        "em_bucketer:#{redis_prefix}:#{bucket_id}"
+      end
+
+      def redis_known_buckets_key
+        "em_bucketer_known_buckets:#{redis_prefix}"
+      end
+    end
+  end
+end

data/lib/em-bucketer/in_memory.rb

@@ -0,0 +1,24 @@
+require 'eventmachine'
+require 'em-bucketer/database/hash'
+require 'em-bucketer/base'
+
+module EventMachine::Bucketer
+  class InMemory
+    include Database::Hash
+    include Base
+
+    BUCKET_THRESHOLD_SIZE_DEFAULT = 1000
+    BUCKET_MAX_AGE_DEFAULT = 3600
+
+    # Creates a new in memory Bucketer with the requested
+    # configurations
+    #
+    # @param bucket_threshold_size [Integer] the max size of the bucket
+    # after which the on_bucket_full callback is called
+    # @param bucket_max_age [Integer] max number of seconds a bucket
+    # can remain before the on_bucket_timed_out is called
+    def initialize(bucket_threshold_size: BUCKET_THRESHOLD_SIZE_DEFAULT, bucket_max_age: BUCKET_MAX_AGE_DEFAULT)
+      setup(bucket_threshold_size, bucket_max_age)
+    end
+  end
+end

data/lib/em-bucketer/redis.rb

@@ -0,0 +1,68 @@
+require 'eventmachine'
+require 'em-bucketer/database/redis'
+require 'em-bucketer/base'
+
+module EventMachine::Bucketer
+  class Redis
+    include Database::Redis
+    include Base
+
+    BUCKET_THRESHOLD_SIZE_DEFAULT = 1000
+    BUCKET_MAX_AGE_DEFAULT = 3600
+
+    # Creates a new redis Bucketer with the requested
+    # configurations.
+    # *NOTE* The redis bucketer uses Marshal to store
+    # the objects in redis. This puts limitations on
+    # the data that cannot be stored in these buckets.
+    # For example you cannot store an object that
+    # references a proc as an instance variable.
+    #
+    # The redis bucketer also sets all timers on
+    # startup for buckets already in the redis
+    # database. This ensures that even if your
+    # app is restarted the previous timers will still
+    # get set and you won't ever lose a bucket.
+    #
+    # @param redis_prefix [String] The prefix for the
+    # bucket in redis. This is necessary because you
+    # may want to have multiple bucketers using one
+    # redis instance and you don't want them conflicting.
+    # Also this can't just be random because the whole
+    # point of the redis bucketer is that you can restart
+    # your app and get back the same bucketer without any
+    # data loss.
+    # @param bucket_threshold_size [Integer] the max size of the bucket
+    # after which the on_bucket_full callback is called
+    # @param bucket_max_age [Integer] max number of seconds a bucket
+    # can remain before the on_bucket_timed_out is called
+    def initialize(redis_prefix, bucket_threshold_size: BUCKET_THRESHOLD_SIZE_DEFAULT, bucket_max_age: BUCKET_MAX_AGE_DEFAULT)
+      @redis = EM::Hiredis.connect
+      @redis_prefix = redis_prefix
+      setup(bucket_threshold_size, bucket_max_age)
+      set_timers
+    end
+
+    def set_timers
+      known_buckets.callback do |bucket_ids|
+        bucket_ids.each do |bucket_id|
+          add_timer_if_first(bucket_id)
+        end
+      end.errback do |e|
+        # I think this is okay since it will only happen when
+        # you are initializing the bucketer so it will hopefully
+        # bring the issue to your attention on startup. I
+        # couldn't actually pass this error back through to
+        # anyone so I needed to raise it. It is also a bad
+        # exception since it means you are not properly reloading
+        # your buckets from redis
+        raise e
+      end
+    end
+
+    private
+
+    # Used by Database::Redis
+    attr_reader :redis_prefix, :redis
+  end
+end