em-bucketer 0.2.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: aa43207fc878f4d9f0f15851590ec66803f45694
-  data.tar.gz: 9cd15644641ffccdd96552f45bbc5b420de5a262
+  metadata.gz: c433f1cc453123f151e4a89363b6a6ebcb02e991
+  data.tar.gz: a541740a352cdc8c3a4e6474d6a7f0d69b4979a2
 SHA512:
-  metadata.gz: 7b4ee20c2e7ad8cb933b5a0de2dfea6cd9b05ee7375cd156ea8659feda615890e8a6d52490e63dafb6d71e44b992eb33b066889b207b21efe693862fcff19fd7
-  data.tar.gz: 96447882b8d6cdf6a20dc62a25f69af698de3708eb523735fcec40ae503edab3237b919ba72aa69ae9861d2350e7ec9b3a0110383eb5e857db439ed075e44a0d
+  metadata.gz: fa4fcfa73e8d0311c2534cdd759fd8150a2fc96868faa271b96b26f12a125084822cd131816dd886df66e95ebecc89a0b474ae1297843b3bc5f5c5db5a3de50a
+  data.tar.gz: 39cd7430e0855f9048e6294cfa38cf7137d83172a6e5af997c4b7b14b4919bdf411a0d25838336325fe99695b02c7593f232373ad7cc5bee6f08c9f9eed8eefc

data/lib/em-bucketer/base.rb

@@ -3,7 +3,6 @@ module EventMachine::Bucketer
     def setup(bucket_threshold_size, bucket_max_age)
       @bucket_threshold_size = bucket_threshold_size
       @bucket_max_age = bucket_max_age
-      @buckets = {}
       @on_bucket_full_callbacks = []
       @on_bucket_timeout_callbacks = []
       @buckets_with_timers = Set.new

data/lib/em-bucketer/database/hash.rb

@@ -1,10 +1,14 @@
-require 'em-bucketer/database'
+require 'em-bucketer/ordered/database'
 
 module EventMachine::Bucketer
   module Database
     module Hash
       private
 
+      def setup_db
+        @buckets = {}
+      end
+
       def bucket_size_from_db(bucket_id, &blk)
         EM::Completion.new.tap do |c|
           c.callback(&blk) if block_given?

data/lib/em-bucketer/database/redis.rb

@@ -6,6 +6,9 @@ module EventMachine::Bucketer
     module Redis
       private
 
+      def setup_db
+      end
+
       def bucket_size_from_db(bucket_id, &blk)
         EM::Completion.new.tap do |c|
           c.callback(&blk) if block_given?

data/lib/em-bucketer/in_memory.rb

@@ -19,6 +19,7 @@ module EventMachine::Bucketer
     # can remain before the on_bucket_timed_out is called
     def initialize(bucket_threshold_size: BUCKET_THRESHOLD_SIZE_DEFAULT, bucket_max_age: BUCKET_MAX_AGE_DEFAULT)
       setup(bucket_threshold_size, bucket_max_age)
+      setup_db
     end
   end
 end

data/lib/em-bucketer/ordered/base.rb

@@ -3,7 +3,6 @@ module EventMachine::Bucketer
     def setup(bucket_threshold_size, bucket_max_age)
       @bucket_threshold_size = bucket_threshold_size
       @bucket_max_age = bucket_max_age
-      @buckets = {}
       @on_bucket_full_callbacks = []
       @on_bucket_timeout_callbacks = []
       @timers = {}

data/lib/em-bucketer/ordered/database/hash.rb

@@ -6,6 +6,10 @@ module EventMachine::Bucketer
       module Hash
         private
 
+        def setup_db
+          @buckets = {}
+        end
+
         def pop_all_from_db(bucket_id, &blk)
           EM::Completion.new.tap do |c|
             c.callback(&blk) if block_given?

data/lib/em-bucketer/ordered/database/redis.rb

@@ -0,0 +1,120 @@
+require 'em-bucketer/ordered/database'
+require 'em-hiredis'
+
+module EventMachine::Bucketer
+  module Ordered::Database
+    module Redis
+      private
+
+      def setup_db
+        redis.register_script(:lpopn, <<-END)
+          local r = redis.call('lrange', KEYS[1], 0, ARGV[1] - 1)
+          redis.call('ltrim', KEYS[1], ARGV[1], -1)
+          return r
+        END
+        redis.register_script(:lpopa, <<-END)
+          local r = redis.call('lrange', KEYS[1], 0, - 1)
+          redis.call('del', KEYS[1])
+          return r
+        END
+      end
+
+      def bucket_size_from_db(bucket_id, &blk)
+        EM::Completion.new.tap do |c|
+          c.callback(&blk) if block_given?
+          redis.llen(redis_key(bucket_id)).callback do |len|
+            c.succeed len.to_i
+          end.errback do |e|
+            c.fail e
+          end
+        end
+      end
+
+      def add_item_to_db(bucket_id, item, &blk)
+        EM::Completion.new.tap do |c|
+          c.callback(&blk) if block_given?
+          redis.rpush(redis_key(bucket_id), Marshal.dump(item)).callback do
+            add_to_known_buckets(bucket_id).callback do
+              c.succeed
+            end.errback do |e|
+              c.fail e
+            end
+          end.errback do |e|
+            c.fail e
+          end
+        end
+      end
+
+      def pop_all_from_db(bucket_id, &blk)
+        EM::Completion.new.tap do |c|
+          c.callback(&blk) if block_given?
+          redis.lpopa([redis_key(bucket_id)]).callback do |data|
+            bucket = data.map { |d| Marshal.load(d) }
+            c.succeed bucket
+          end.errback do |e|
+            c.fail e
+          end
+        end
+      end
+
+      def pop_count_from_db(bucket_id, count, &blk)
+        EM::Completion.new.tap do |c|
+          c.callback(&blk) if block_given?
+          redis.lpopn([redis_key(bucket_id)], count).callback do |data|
+            bucket = data.map { |d| Marshal.load(d) }
+            c.succeed bucket
+          end.errback do |e|
+            c.fail e
+          end
+        end
+      end
+
+      def get_bucket_from_db(bucket_id, &blk)
+        EM::Completion.new.tap do |c|
+          c.callback(&blk) if block_given?
+          redis.lrange(redis_key(bucket_id), 0, -1).callback do |data|
+            bucket = data.map { |d| Marshal.load(d) }
+            c.succeed bucket
+          end.errback do |e|
+            c.fail e
+          end
+        end
+      end
+
+      def empty_bucket_in_db(bucket_id, &blk)
+        EM::Completion.new.tap do |c|
+          c.callback(&blk) if block_given?
+          redis.del(redis_key(bucket_id)).callback do
+            remove_from_known_buckets(bucket_id).callback do
+              c.succeed
+            end.errback do |e|
+              c.fail e
+            end
+          end.errback do |e|
+            c.fail e
+          end
+        end
+      end
+
+      def known_buckets(&blk)
+        redis.smembers(redis_known_buckets_key, &blk)
+      end
+
+      def add_to_known_buckets(bucket_id, &blk)
+        redis.sadd(redis_known_buckets_key, bucket_id, &blk)
+      end
+
+      def remove_from_known_buckets(bucket_id, &blk)
+        redis.srem(redis_known_buckets_key, bucket_id, &blk)
+      end
+
+      def redis_key(bucket_id)
+        "em_bucketer_ordered:#{redis_prefix}:#{bucket_id}"
+      end
+
+      def redis_known_buckets_key
+        "em_bucketer_ordered_known_buckets:#{redis_prefix}"
+      end
+    end
+  end
+end

data/lib/em-bucketer/ordered/in_memory.rb

@@ -19,6 +19,7 @@ module EventMachine::Bucketer
     # can remain before the on_bucket_timed_out is called
     def initialize(bucket_threshold_size: BUCKET_THRESHOLD_SIZE_DEFAULT, bucket_max_age: BUCKET_MAX_AGE_DEFAULT)
       setup(bucket_threshold_size, bucket_max_age)
+      setup_db
     end
   end
 end

data/lib/em-bucketer/ordered/redis.rb

@@ -0,0 +1,69 @@
+require 'eventmachine'
+require 'em-bucketer/ordered/database/redis'
+require 'em-bucketer/ordered/base'
+
+module EventMachine::Bucketer
+  class Ordered::Redis
+    include Ordered::Database::Redis
+    include Ordered::Base
+
+    BUCKET_THRESHOLD_SIZE_DEFAULT = 1000
+    BUCKET_MAX_AGE_DEFAULT = 3600
+
+    # Creates a new redis Bucketer with the requested
+    # configurations.
+    # *NOTE* The redis bucketer uses Marshal to store
+    # the objects in redis. This puts limitations on
+    # the data that cannot be stored in these buckets.
+    # For example you cannot store an object that
+    # references a proc as an instance variable.
+    #
+    # The redis bucketer also sets all timers on
+    # startup for buckets already in the redis
+    # database. This ensures that even if your
+    # app is restarted the previous timers will still
+    # get set and you won't ever lose a bucket.
+    #
+    # @param redis_prefix [String] The prefix for the
+    # bucket in redis. This is necessary because you
+    # may want to have multiple bucketers using one
+    # redis instance and you don't want them conflicting.
+    # Also this can't just be random because the whole
+    # point of the redis bucketer is that you can restart
+    # your app and get back the same bucketer without any
+    # data loss.
+    # @param bucket_threshold_size [Integer] the max size of the bucket
+    # after which the on_bucket_full callback is called
+    # @param bucket_max_age [Integer] max number of seconds a bucket
+    # can remain before the on_bucket_timed_out is called
+    def initialize(redis_prefix, bucket_threshold_size: BUCKET_THRESHOLD_SIZE_DEFAULT, bucket_max_age: BUCKET_MAX_AGE_DEFAULT)
+      @redis = EM::Hiredis.connect
+      @redis_prefix = redis_prefix
+      setup(bucket_threshold_size, bucket_max_age)
+      setup_db
+      set_timers
+    end
+
+    def set_timers
+      known_buckets.callback do |bucket_ids|
+        bucket_ids.each do |bucket_id|
+          add_timer_if_first(bucket_id)
+        end
+      end.errback do |e|
+        # I think this is okay since it will only happen when
+        # you are initializing the bucketer so it will hopefully
+        # bring the issue to your attention on startup. I
+        # couldn't actually pass this error back through to
+        # anyone so I needed to raise it. It is also a bad
+        # exception since it means you are not properly reloading
+        # your buckets from redis
+        raise e
+      end
+    end
+
+    private
+
+    # Used by Database::Redis
+    attr_reader :redis_prefix, :redis
+  end
+end

data/lib/em-bucketer/ordered.rb

@@ -5,4 +5,4 @@ end
 
 require 'eventmachine'
 require 'em-bucketer/ordered/in_memory'
-#require 'em-bucketer/ordered/redis'
+require 'em-bucketer/ordered/redis'

data/lib/em-bucketer/redis.rb

@@ -40,6 +40,7 @@ module EventMachine::Bucketer
       @redis = EM::Hiredis.connect
       @redis_prefix = redis_prefix
       setup(bucket_threshold_size, bucket_max_age)
+      setup_db
       set_timers
     end
 

data/lib/em-bucketer/version.rb

@@ -1,5 +1,5 @@
 module EventMachine
   module Bucketer
-    VERSION = "0.2.0"
+    VERSION = "0.2.1"
   end
 end

data/spec/redis_ordered_spec.rb

@@ -0,0 +1,18 @@
+require 'spec_helper'
+require 'em-bucketer/ordered'
+require 'redis'
+
+describe EventMachine::Bucketer::Ordered::Redis do
+  let(:prefix) { "test_prefix" }
+  before(:each) do
+    redis = Redis.new
+    redis.keys("em_bucketer_ordered:#{prefix}:*").each do |k|
+      redis.del(k)
+    end
+    redis.del("em_bucketer_ordered_known_buckets:#{prefix}")
+  end
+
+  it_behaves_like "an ordered bucketer" do
+    let(:bucketer) { EM::Bucketer::Ordered::Redis.new(prefix, :bucket_threshold_size => 5) }
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: em-bucketer
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.1
 platform: ruby
 authors:
 - Richard Heycock
@@ -147,13 +147,16 @@ files:
 - lib/em-bucketer/ordered/base.rb
 - lib/em-bucketer/ordered/database.rb
 - lib/em-bucketer/ordered/database/hash.rb
+- lib/em-bucketer/ordered/database/redis.rb
 - lib/em-bucketer/ordered/in_memory.rb
+- lib/em-bucketer/ordered/redis.rb
 - lib/em-bucketer/redis.rb
 - lib/em-bucketer/version.rb
 - spec/em_bucketer_examples.rb
 - spec/em_bucketer_ordered_examples.rb
 - spec/in_memory_ordered_spec.rb
 - spec/in_memory_spec.rb
+- spec/redis_ordered_spec.rb
 - spec/redis_spec.rb
 - spec/spec_helper.rb
 - spec/spec_methods.rb
@@ -187,6 +190,7 @@ test_files:
 - spec/em_bucketer_ordered_examples.rb
 - spec/in_memory_ordered_spec.rb
 - spec/in_memory_spec.rb
+- spec/redis_ordered_spec.rb
 - spec/redis_spec.rb
 - spec/spec_helper.rb
 - spec/spec_methods.rb