RubyGems - hyperloglog-redis - Versions diffs - 1.0.0 → 2.0.0 - Mend

hyperloglog-redis 1.0.0 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (14) hide show

data/Gemfile +1 -0
data/Gemfile.lock +2 -0
data/HISTORY.md +25 -6
data/README.md +124 -25
data/VERSION +1 -1
data/hyperloglog-redis.gemspec +10 -4
data/lib/algorithm.rb +69 -0
data/lib/counter.rb +40 -0
data/lib/hyperloglog-redis.rb +3 -1
data/lib/time_series_counter.rb +80 -0
data/spec/hyper_log_log_spec.rb +199 -197
data/spec/time_series_counter_spec.rb +216 -0
metadata +28 -14
data/lib/hyper_log_log.rb +0 -96

data/spec/hyper_log_log_spec.rb CHANGED

@@ -1,224 +1,226 @@
 require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
 describe HyperLogLog do
-  it "doesn't change its count when it sees values that it's already seen" do
-    redis = Redis.new
-    counter = HyperLogLog.new(redis, 10)
-    test_set = (1..100).map{ |x| x.to_s }
-    test_set.each{ |value| counter.add("mycounter", value) }
-    original_estimate = counter.count("mycounter")
-    5.times do
-      test_set.each do |value|
-        counter.add("mycounter", value)
-        counter.count("mycounter").should == original_estimate
+  [HyperLogLog::Counter, HyperLogLog::TimeSeriesCounter].each do |counter_type|
+    it "doesn't change its count when it sees values that it's already seen" do
+      redis = Redis.new
+      counter = counter_type.new(redis, 10)
+      test_set = (1..100).map{ |x| x.to_s }
+      test_set.each{ |value| counter.add("mycounter", value) }
+      original_estimate = counter.count("mycounter")
+      5.times do
+        test_set.each do |value|
+          counter.add("mycounter", value)
+          counter.count("mycounter").should == original_estimate
+        end
       end
     end
-  end
-  it "can maintain more than one logically distinct counter" do
-    redis = Redis.new
-    counter = HyperLogLog.new(redis, 10)
-    other_estimate = counter.count("counter2")
-    (1..100).each do |i|
-      counter.add("counter1", i.to_s)
-      counter.count("counter2").should == other_estimate
+    it "can maintain more than one logically distinct counter" do
+      redis = Redis.new
+      counter = counter_type.new(redis, 10)
+      other_estimate = counter.count("counter2")
+      (1..100).each do |i|
+        counter.add("counter1", i.to_s)
+        counter.count("counter2").should == other_estimate
+      end
+      other_estimate = counter.count("counter1")
+      (101..200).each do |i|
+        counter.add("counter2", i.to_s)
+        counter.count("counter1").should == other_estimate
+      end
+      other_estimate = counter.count("counter2")
+      (201..300).each do |i|
+        counter.add("counter1", i.to_s)
+        counter.count("counter2").should == other_estimate
+      end
+      counter.count("counter1").should > 100
+      counter.count("counter2").should > 50
+      counter.count("counter1").should > counter.count("counter2")
     end
-    other_estimate = counter.count("counter1")
-    (101..200).each do |i|
-      counter.add("counter2", i.to_s)
-      counter.count("counter1").should == other_estimate
+    it "can exactly count small sets" do
+      redis = Redis.new
+      counter = counter_type.new(redis, 11)
+      10.times { |i| counter.add("mycounter", i.to_s) }
+      counter.count("mycounter").should == 10
     end
-    other_estimate = counter.count("counter2")
-    (201..300).each do |i|
-      counter.add("counter1", i.to_s)
-      counter.count("counter2").should == other_estimate
-    end
-    counter.count("counter1").should > 100
-    counter.count("counter2").should > 50
-    counter.count("counter1").should > counter.count("counter2")
-  end
-  it "can exactly count small sets" do
-    redis = Redis.new
-    counter = HyperLogLog.new(redis, 11)
-    10.times { |i| counter.add("mycounter", i.to_s) }
-    counter.count("mycounter").should == 10
-  end
-  it "can exactly count small unions" do
-    redis = Redis.new
-    counter = HyperLogLog.new(redis, 11)
-    (1..8).each { |i| counter.add("mycounter1", i.to_s) }
-    (5..12).each { |i| counter.add("mycounter2", i.to_s) }
-    counter.union("mycounter1", "mycounter2").should == 12
-  end
-  it "can exactly count small intersections" do
-    redis = Redis.new
-    counter = HyperLogLog.new(redis, 11)
-    (1..8).each { |i| counter.add("mycounter1", i.to_s) }
-    (5..12).each { |i| counter.add("mycounter2", i.to_s) }
-    counter.intersection("mycounter1", "mycounter2").should == 4
-  end
-  it "can store unions for querying later" do
-    redis = Redis.new
-    counter = HyperLogLog.new(redis, 11)
-    (1..10).each { |i| counter.add("mycounter1", i.to_s) }
-    (5..15).each { |i| counter.add("mycounter2", i.to_s) }
-    (15..25).each { |i| counter.add("mycounter3", i.to_s) }
-    (20..50).each { |i| counter.add("mycounter4", i.to_s) }
-    counter.union_store("aggregate_counter", "mycounter1", "mycounter2", "mycounter3", "mycounter4")
-    counter.union("mycounter1", "mycounter2", "mycounter3", "mycounter4").should == counter.count("aggregate_counter")
-  end
-  # With parameter b, HyperLogLog should produce estimates that have
-  # relative error of 1.04 / Math.sqrt(2 ** b). Of course, this analysis
-  # is based on assumptions that aren't necessarily true in practice and
-  # the observed relative error will depend on the distribution of data
-  # we receive as well as the interaction of the murmur hash implementation
-  # with that data. Keeping that in mind, the following spec makes sure
-  # that in the process of adding 1000 values to a set, HyperLogLog only
-  # gives bad estimates (more than twice the expected relative error) in
-  # less than 1% of the cases and never gives very bad estimates (more than
-  # three times the expected relative error.)
-  #
-  # It's fine to fudge these numbers a little if the implementation changes,
-  # since you can clearly find a different set of values that make this test
-  # fail even without changing the implementation. But it should serve as a
-  # good indication that there aren't any logical errors in the HyperLogLog
-  # implementation, since it exercises all of the cases in HyperLogLog's
-  # count method except for the correction for very large set sizes.
-  it "produces acceptable estimates for counts" do
-    max_items = 1000
-    redis = Redis.new
-    (6..16).each do |b|
-      counter = HyperLogLog.new(redis, b)
-      redis.del('mycounter')
+    it "can exactly count small unions" do
+      redis = Redis.new
+      counter = counter_type.new(redis, 11)
+      (1..8).each { |i| counter.add("mycounter1", i.to_s) }
+      (5..12).each { |i| counter.add("mycounter2", i.to_s) }
+      counter.union(["mycounter1", "mycounter2"]).should == 12
+    end
+    it "can exactly count small intersections" do
+      redis = Redis.new
+      counter = counter_type.new(redis, 11)
+      (1..8).each { |i| counter.add("mycounter1", i.to_s) }
+      (5..12).each { |i| counter.add("mycounter2", i.to_s) }
+      counter.intersection(["mycounter1", "mycounter2"]).should == 4
+    end
+    it "can store unions for querying later" do
+      redis = Redis.new
+      counter = counter_type.new(redis, 11)
+      (1..10).each { |i| counter.add("mycounter1", i.to_s) }
+      (5..15).each { |i| counter.add("mycounter2", i.to_s) }
+      (15..25).each { |i| counter.add("mycounter3", i.to_s) }
+      (20..50).each { |i| counter.add("mycounter4", i.to_s) }
+      counter.union_store("aggregate_counter", ["mycounter1", "mycounter2", "mycounter3", "mycounter4"])
+      counter.union(["mycounter1", "mycounter2", "mycounter3", "mycounter4"]).should == counter.count("aggregate_counter")
+    end
+    # With parameter b, HyperLogLog should produce estimates that have
+    # relative error of 1.04 / Math.sqrt(2 ** b). Of course, this analysis
+    # is based on assumptions that aren't necessarily true in practice and
+    # the observed relative error will depend on the distribution of data
+    # we receive as well as the interaction of the murmur hash implementation
+    # with that data. Keeping that in mind, the following spec makes sure
+    # that in the process of adding 1000 values to a set, HyperLogLog only
+    # gives bad estimates (more than twice the expected relative error) in
+    # less than 1% of the cases and never gives very bad estimates (more than
+    # three times the expected relative error.)
+    #
+    # It's fine to fudge these numbers a little if the implementation changes,
+    # since you can clearly find a different set of values that make this test
+    # fail even without changing the implementation. But it should serve as a
+    # good indication that there aren't any logical errors in the HyperLogLog
+    # implementation, since it exercises all of the cases in HyperLogLog's
+    # count method except for the correction for very large set sizes.
+    it "produces acceptable estimates for counts" do
+      max_items = 1000
+      redis = Redis.new
+      (6..16).each do |b|
+        counter = counter_type.new(redis, b)
+        redis.del('mycounter')
+        bad_estimates = 0
+        very_bad_estimates = 0
+        expected_relative_error = 1.04 / Math.sqrt(2 ** b)
+        max_items.times do |i|
+          value = Digest::MD5.hexdigest("value#{i}")
+          counter.add("mycounter", value)
+          actual = i + 1
+          approximate = counter.count("mycounter")
+          relative_error = (actual - approximate).abs / Float(actual)
+          bad_estimates += 1 if relative_error > expected_relative_error * 2
+          very_bad_estimates += 1 if relative_error > expected_relative_error * 3
+        end
+        bad_estimates.should < max_items / 100.00
+        very_bad_estimates.should == 0
+      end
+    end
+    it "produces acceptable estimates for unions with few elements in common" do
+      b, max_items = 10, 2000
+      counter = counter_type.new(Redis.new, b)
       bad_estimates = 0
       very_bad_estimates = 0
       expected_relative_error = 1.04 / Math.sqrt(2 ** b)
       max_items.times do |i|
-        value = Digest::MD5.hexdigest("value#{i}")
-        counter.add("mycounter", value)
-        actual = i + 1
-        approximate = counter.count("mycounter")
+        value1 = Digest::MD5.hexdigest("value#{i}")
+        counter.add("mycounter1", value1)
+        value2 = Digest::MD5.hexdigest("value#{i}incounter2")
+        counter.add("mycounter2", value2)
+        value3 = Digest::MD5.hexdigest("this is value#{i}")
+        counter.add("mycounter3", value3)
+        actual = 3 * (i + 1)
+        approximate = counter.union(["mycounter1", "mycounter2", "mycounter3"])
         relative_error = (actual - approximate).abs / Float(actual)
         bad_estimates += 1 if relative_error > expected_relative_error * 2
         very_bad_estimates += 1 if relative_error > expected_relative_error * 3
       end
-      bad_estimates.should < max_items / 100.00
+      bad_estimates.should < (3 * max_items) / 100.00
       very_bad_estimates.should == 0
     end
-  end
-  it "produces acceptable estimates for unions with few elements in common" do
-    b, max_items = 10, 2000
-    counter = HyperLogLog.new(Redis.new, b)
-    bad_estimates = 0
-    very_bad_estimates = 0
-    expected_relative_error = 1.04 / Math.sqrt(2 ** b)
-    max_items.times do |i|
-      value1 = Digest::MD5.hexdigest("value#{i}")
-      counter.add("mycounter1", value1)
-      value2 = Digest::MD5.hexdigest("value#{i}incounter2")
-      counter.add("mycounter2", value2)
-      value3 = Digest::MD5.hexdigest("this is value#{i}")
-      counter.add("mycounter3", value3)
-      actual = 3 * (i + 1)
-      approximate = counter.union("mycounter1", "mycounter2", "mycounter3")
-      relative_error = (actual - approximate).abs / Float(actual)
-      bad_estimates += 1 if relative_error > expected_relative_error * 2
-      very_bad_estimates += 1 if relative_error > expected_relative_error * 3
-    end
-    bad_estimates.should < (3 * max_items) / 100.00
-    very_bad_estimates.should == 0
-  end
-  it "produces acceptable estimates for unions with many elements in common" do
-    b, max_items, intersection_size = 10, 1000, 2000
-    counter = HyperLogLog.new(Redis.new, b)
-    bad_estimates = 0
-    very_bad_estimates = 0
-    expected_relative_error = 1.04 / Math.sqrt(2 ** b)
-    intersection_size.times do |i|
-      value = Digest::MD5.hexdigest("test#{i}value")
-      ['mycounter1', 'mycounter2', 'mycounter3'].each do |counter_name|
-        counter.add(counter_name, value)
+    it "produces acceptable estimates for unions with many elements in common" do
+      b, max_items, intersection_size = 10, 1000, 2000
+      counter = counter_type.new(Redis.new, b)
+      bad_estimates = 0
+      very_bad_estimates = 0
+      expected_relative_error = 1.04 / Math.sqrt(2 ** b)
+      intersection_size.times do |i|
+        value = Digest::MD5.hexdigest("test#{i}value")
+        ['mycounter1', 'mycounter2', 'mycounter3'].each do |counter_name|
+          counter.add(counter_name, value)
+        end
       end
+      max_items.times do |i|
+        value1 = Digest::MD5.hexdigest("value#{i}")
+        counter.add("mycounter1", value1)
+        value2 = Digest::MD5.hexdigest("value#{i}isincounter2")
+        counter.add("mycounter2", value2)
+        value3 = Digest::MD5.hexdigest("this is value#{i}")
+        counter.add("mycounter3", value3)
+        actual = 3 * (i + 1) + intersection_size
+        approximate = counter.union(["mycounter1", "mycounter2", "mycounter3"])
+        relative_error = (actual - approximate).abs / Float(actual)
+        bad_estimates += 1 if relative_error > expected_relative_error * 2
+        very_bad_estimates += 1 if relative_error > expected_relative_error * 3
+      end
+      bad_estimates.should < ((3 * max_items) + intersection_size) / 100.00
+      very_bad_estimates.should == 0
     end
-    max_items.times do |i|
-      value1 = Digest::MD5.hexdigest("value#{i}")
-      counter.add("mycounter1", value1)
-      value2 = Digest::MD5.hexdigest("value#{i}isincounter2")
-      counter.add("mycounter2", value2)
-      value3 = Digest::MD5.hexdigest("this is value#{i}")
-      counter.add("mycounter3", value3)
-      actual = 3 * (i + 1) + intersection_size
-      approximate = counter.union("mycounter1", "mycounter2", "mycounter3")
-      relative_error = (actual - approximate).abs / Float(actual)
-      bad_estimates += 1 if relative_error > expected_relative_error * 2
-      very_bad_estimates += 1 if relative_error > expected_relative_error * 3
-    end
-    bad_estimates.should < ((3 * max_items) + intersection_size) / 100.00
-    very_bad_estimates.should == 0
-  end
-  # There are no good theoretical guarantees that I know of for arbitrary
-  # intersection estimation, since it's expessed as the sum of unions of
-  # HyperLogLog counters, but it tends to work okay in practice, as seen below.
-  it "produces decent estimates for intersections" do
-    b, max_items = 6, 1000
-    counter = HyperLogLog.new(Redis.new, b)
-    expected_relative_error = 1.04 / Math.sqrt(2 ** b)
-    max_items.times do |i|
-      value1 = Digest::MD5.hexdigest("first-value#{i}")
-      value2 = Digest::MD5.hexdigest("second-value#{i}")
-      value3 = Digest::MD5.hexdigest("third-value#{i}")
-      value4 = Digest::MD5.hexdigest("fourth-value#{i}")
-      counter.add("mycounter1", value1)
-      counter.add("mycounter2", value2)
-      counter.add("mycounter3", value3)
-      counter.add("mycounter4", value4)
-      [value1, value2, value3, value4].each{ |value| counter.add("mycounter5", value) }
-    end
-    small_counters = ['mycounter1', 'mycounter2', 'mycounter3', 'mycounter4']
+    # There are no good theoretical guarantees that I know of for arbitrary
+    # intersection estimation, since it's expessed as the sum of unions of
+    # HyperLogLog counters, but it tends to work okay in practice, as seen below.
-    small_counters.each do |counter_name|
-      intersection_estimate = counter.intersection(counter_name, 'mycounter5')
-      intersection_estimate.should > 0
-      (intersection_estimate - counter.count(counter_name)).abs.should < max_items * expected_relative_error
-    end
-    [2,3].each do |intersection_size|
-      small_counters.combination(intersection_size).each do |counter_names|
-        intersection_estimate = counter.intersection(*counter_names)
-        intersection_estimate.should >= 0
-        intersection_estimate.should < intersection_size * max_items * expected_relative_error
+    it "produces decent estimates for intersections" do
+      b, max_items = 6, 1000
+      counter = counter_type.new(Redis.new, b)
+      expected_relative_error = 1.04 / Math.sqrt(2 ** b)
+      max_items.times do |i|
+        value1 = Digest::MD5.hexdigest("first-value#{i}")
+        value2 = Digest::MD5.hexdigest("second-value#{i}")
+        value3 = Digest::MD5.hexdigest("third-value#{i}")
+        value4 = Digest::MD5.hexdigest("fourth-value#{i}")
+        counter.add("mycounter1", value1)
+        counter.add("mycounter2", value2)
+        counter.add("mycounter3", value3)
+        counter.add("mycounter4", value4)
+        [value1, value2, value3, value4].each{ |value| counter.add("mycounter5", value) }
       end
-    end
-    100.times do |i|
-      value = Digest::MD5.hexdigest("somethingintheintersection#{i}")
-      small_counters.each { |counter_name| counter.add(counter_name, value) }
-    end
-    [2,3,4].each do |intersection_size|
-      small_counters.combination(intersection_size).each do |counter_names|
-        intersection_estimate = counter.intersection(*counter_names)
-        intersection_estimate.should >= 0
-        (intersection_estimate - 100).abs.should < intersection_size * (max_items + 100) * expected_relative_error
+      small_counters = ['mycounter1', 'mycounter2', 'mycounter3', 'mycounter4']
+      small_counters.each do |counter_name|
+        intersection_estimate = counter.intersection([counter_name, 'mycounter5'])
+        intersection_estimate.should > 0
+        (intersection_estimate - counter.count(counter_name)).abs.should < max_items * expected_relative_error
+      end
+      [2,3].each do |intersection_size|
+        small_counters.combination(intersection_size).each do |counter_names|
+          intersection_estimate = counter.intersection(counter_names)
+          intersection_estimate.should >= 0
+          intersection_estimate.should < intersection_size * max_items * expected_relative_error
+        end
       end
+      100.times do |i|
+        value = Digest::MD5.hexdigest("somethingintheintersection#{i}")
+        small_counters.each { |counter_name| counter.add(counter_name, value) }
+      end
+      [2,3,4].each do |intersection_size|
+        small_counters.combination(intersection_size).each do |counter_names|
+          intersection_estimate = counter.intersection(counter_names)
+          intersection_estimate.should >= 0
+          (intersection_estimate - 100).abs.should < intersection_size * (max_items + 100) * expected_relative_error
+        end
+      end
     end
   end
 end

data/spec/time_series_counter_spec.rb ADDED

@@ -0,0 +1,216 @@
+require 'securerandom'
+require 'timecop'
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+MINUTES=60
+HOURS=MINUTES*60
+DAYS=HOURS*24
+WEEKS=DAYS*7
+describe HyperLogLog::TimeSeriesCounter do
+  before(:each) do
+    @b = 11
+    @redis = Redis.new
+    @counter = HyperLogLog::TimeSeriesCounter.new(@redis, @b)
+    @expected_relative_error = 1.04 / Math.sqrt(2 ** @b)
+    def counter_should_equal(counter_val, expected_val, relative_error_base=nil)
+      (counter_val - expected_val).abs.should <= (relative_error_base || expected_val) * @expected_relative_error
+    end
+  end
+  it "can estimate cardinalities from any particular point in time until the present" do
+    Timecop.travel(Time.now - 2 * WEEKS) do
+      (0..100).each { |i| @counter.add('mycounter', "item#{i}") }
+    end
+    Timecop.travel(Time.now - 1 * WEEKS) do
+      (100..200).each { |i| @counter.add('mycounter', "item#{i}") }
+    end
+    Timecop.travel(Time.now - 6 * DAYS) do
+      (0..100).each { |i| @counter.add('mycounter', "item#{i}") }
+    end
+    Timecop.travel(Time.now - 5 * DAYS) do
+      (100..200).each { |i| @counter.add('mycounter', "item#{i}") }
+    end
+    Timecop.travel(Time.now - 4 * DAYS) do
+      (200..250).each { |i| @counter.add('mycounter', "item#{i}") }
+    end
+    counter_should_equal(@counter.count('mycounter'), 250)
+    counter_should_equal(@counter.count('mycounter', Time.now.to_i - 3 * WEEKS), 250)
+    counter_should_equal(@counter.count('mycounter', Time.now.to_i - 1 * WEEKS - 3 * DAYS), 250)
+    counter_should_equal(@counter.count('mycounter', Time.now.to_i - 1 * WEEKS), 250)
+    counter_should_equal(@counter.count('mycounter', Time.now.to_i - 5 * DAYS - 12 * HOURS), 150, 250)
+    counter_should_equal(@counter.count('mycounter', Time.now.to_i - 4 * DAYS - 12 * HOURS), 50, 250)
+    counter_should_equal(@counter.count('mycounter', Time.now.to_i - 3 * DAYS), 0, 250)
+  end
+  it "can estimate unions from any particular point in time until the present" do
+    Timecop.travel(Time.now - 2 * WEEKS) do
+      (0..100).each { |i| @counter.add('mycounter1', "item#{i}") }
+    end
+    Timecop.travel(Time.now - 1 * WEEKS) do
+      (100..200).each { |i| @counter.add('mycounter2', "item#{i}") }
+    end
+    Timecop.travel(Time.now - 6 * DAYS) do
+      (0..100).each { |i| @counter.add('mycounter1', "item#{i}") }
+    end
+    Timecop.travel(Time.now - 5 * DAYS) do
+      (100..200).each { |i| @counter.add('mycounter2', "item#{i}") }
+    end
+    Timecop.travel(Time.now - 4 * DAYS) do
+      (200..250).each { |i| @counter.add('mycounter1', "item#{i}") }
+    end
+    counter_should_equal(@counter.union(['mycounter1', 'mycounter2']), 250)
+    counter_should_equal(@counter.union(['mycounter1', 'mycounter2'], Time.now.to_i - 3 * WEEKS), 250)
+    counter_should_equal(@counter.union(['mycounter1', 'mycounter2'], Time.now.to_i - 1 * WEEKS - 3 * DAYS), 250)
+    counter_should_equal(@counter.union(['mycounter1', 'mycounter2'], Time.now.to_i - 1 * WEEKS), 250)
+    counter_should_equal(@counter.union(['mycounter1', 'mycounter2'], Time.now.to_i - 5 * DAYS - 12 * HOURS), 150, 250)
+    counter_should_equal(@counter.union(['mycounter1', 'mycounter2'], Time.now.to_i - 4 * DAYS - 12 * HOURS), 50, 250)
+    counter_should_equal(@counter.union(['mycounter1', 'mycounter2'], Time.now.to_i - 3 * DAYS), 0, 250)
+  end
+  it "can estimate intersections from any particular point in time until the present" do
+    Timecop.travel(Time.now - 2 * WEEKS) do
+      (0..100).each { |i| @counter.add('mycounter1', "item#{i}") }
+    end
+    Timecop.travel(Time.now - 1 * WEEKS) do
+      (100..200).each { |i| @counter.add('mycounter2', "item#{i}") }
+    end
+    Timecop.travel(Time.now - 6 * DAYS) do
+      (0..100).each { |i| @counter.add('mycounter2', "item#{i}") }
+    end
+    Timecop.travel(Time.now - 5 * DAYS) do
+      (100..200).each { |i| @counter.add('mycounter1', "item#{i}") }
+    end
+    Timecop.travel(Time.now - 4 * DAYS) do
+      (200..250).each { |i| @counter.add('mycounter1', "item#{i}") }
+    end
+    Timecop.travel(Time.now - 3 * DAYS) do
+      (200..250).each { |i| @counter.add('mycounter2', "item#{i}") }
+    end
+    counter_should_equal(@counter.intersection(['mycounter1', 'mycounter2']), 250)
+    counter_should_equal(@counter.intersection(['mycounter1', 'mycounter2'], Time.now.to_i - 3 * WEEKS), 250)
+    counter_should_equal(@counter.intersection(['mycounter1', 'mycounter2'], Time.now.to_i - 1 * WEEKS - 3 * DAYS), 150, 250)
+    counter_should_equal(@counter.intersection(['mycounter1', 'mycounter2'], Time.now.to_i - 6 * DAYS - 12 * HOURS), 50, 250)
+    counter_should_equal(@counter.intersection(['mycounter1', 'mycounter2'], Time.now.to_i - 5 * DAYS - 12 * HOURS), 50, 250)
+    counter_should_equal(@counter.intersection(['mycounter1', 'mycounter2'], Time.now.to_i - 4 * DAYS - 12 * HOURS), 50, 250)
+    counter_should_equal(@counter.intersection(['mycounter1', 'mycounter2'], Time.now.to_i - 3 * DAYS - 12 * HOURS), 0, 250)
+    counter_should_equal(@counter.intersection(['mycounter1', 'mycounter2'], Time.now.to_i - 2 * DAYS), 0, 250)
+  end
+  it "can use union_store to store snapshots of counters at particular points in time" do
+    Timecop.travel(Time.now - 2 * WEEKS) do
+      (0..100).each { |i| @counter.add('mycounter1', "item#{i}") }
+    end
+    Timecop.travel(Time.now - 1 * WEEKS) do
+      (100..200).each { |i| @counter.add('mycounter2', "item#{i}") }
+    end
+    Timecop.travel(Time.now - 6 * DAYS) do
+      (0..100).each { |i| @counter.add('mycounter2', "item#{i}") }
+    end
+    Timecop.travel(Time.now - 5 * DAYS) do
+      (100..200).each { |i| @counter.add('mycounter1', "item#{i}") }
+    end
+    Timecop.travel(Time.now - 4 * DAYS) do
+      (200..250).each { |i| @counter.add('mycounter1', "item#{i}") }
+    end
+    Timecop.travel(Time.now - 3 * DAYS) do
+      (200..250).each { |i| @counter.add('mycounter2', "item#{i}") }
+    end
+    @counter.union_store('counter1_1_week_ago', ['mycounter1'], Time.now.to_i - 1 * WEEKS)
+    @counter.union_store('counter2_5_days_ago', ['mycounter2'], Time.now.to_i - 5 * DAYS)
+    counter_should_equal(@counter.union(['counter1_1_week_ago', 'counter2_5_days_ago']), 150, 250)
+  end
+  it "allows you to override the time an event is registered when it's added" do
+    (0..1000).each { |i| @counter.add('mycounter', "item#{i}", Time.now.to_i - 3 * WEEKS) }
+    (1000..2000).each { |i| @counter.add('mycounter', "item#{i}", Time.now.to_i - 2 * WEEKS) }
+    (2000..3000).each { |i| @counter.add('mycounter', "item#{i}", Time.now.to_i - 1 * WEEKS) }
+    (3000..4000).each { |i| @counter.add('mycounter', "item#{i}") }
+    counter_should_equal(@counter.count('mycounter', Time.now.to_i - 4 * WEEKS), 4000)
+    counter_should_equal(@counter.count('mycounter', Time.now.to_i - 2 * WEEKS - 3 * DAYS), 3000)
+    counter_should_equal(@counter.count('mycounter', Time.now.to_i - 1 * WEEKS - 3 * DAYS), 2000)
+    counter_should_equal(@counter.count('mycounter', Time.now.to_i - 3 * DAYS), 1000)
+  end
+  it "doesn't screw up more recent counts when items are injected with earlier timestamp overrides" do
+    Timecop.travel(Time.now - 3 * WEEKS) do
+      (0..1000).each { |i| @counter.add('mycounter', "item#{i}") }
+    end
+    Timecop.travel(Time.now - 2 * WEEKS) do
+      (1000..2000).each { |i| @counter.add('mycounter', "item#{i}") }
+    end
+    Timecop.travel(Time.now - 1 * WEEKS) do
+      (2000..3000).each { |i| @counter.add('mycounter', "item#{i}") }
+    end
+    Timecop.travel(Time.now - 2 * DAYS) do
+      (1000..2000).each { |i| @counter.add('mycounter', "item#{i}") }
+    end
+    counter_should_equal(@counter.count('mycounter', Time.now.to_i - 4 * WEEKS), 3000)
+    counter_should_equal(@counter.count('mycounter', Time.now.to_i - 2 * WEEKS - 3 * DAYS), 2000)
+    counter_should_equal(@counter.count('mycounter', Time.now.to_i - 1 * WEEKS - 3 * DAYS), 2000)
+    counter_should_equal(@counter.count('mycounter', Time.now.to_i - 3 * DAYS), 1000)
+    counter_should_equal(@counter.count('mycounter', Time.now.to_i - 1 * DAYS), 0)
+    # Shouldn't change counts, since they're updates to counts that happen later
+    # than the time we're trying to inject
+    (1000..2000).each { |i| @counter.add('mycounter', "item#{i}", Time.now.to_i - 1 * WEEKS) }
+    counter_should_equal(@counter.count('mycounter', Time.now.to_i - 4 * WEEKS), 3000)
+    counter_should_equal(@counter.count('mycounter', Time.now.to_i - 2 * WEEKS - 3 * DAYS), 2000)
+    counter_should_equal(@counter.count('mycounter', Time.now.to_i - 1 * WEEKS - 3 * DAYS), 2000)
+    counter_should_equal(@counter.count('mycounter', Time.now.to_i - 3 * DAYS), 1000)
+    counter_should_equal(@counter.count('mycounter', Time.now.to_i - 1 * DAYS), 0)
+    # Should change counts, since they're updates to counts for items we've never
+    # seen before in the past
+    (3000..4000).each { |i| @counter.add('mycounter', "item#{i}", Time.now.to_i - 1 * WEEKS) }
+    counter_should_equal(@counter.count('mycounter', Time.now.to_i - 4 * WEEKS), 4000)
+    counter_should_equal(@counter.count('mycounter', Time.now.to_i - 2 * WEEKS - 3 * DAYS), 3000)
+    counter_should_equal(@counter.count('mycounter', Time.now.to_i - 1 * WEEKS - 3 * DAYS), 3000)
+    counter_should_equal(@counter.count('mycounter', Time.now.to_i - 3 * DAYS), 1000)
+    counter_should_equal(@counter.count('mycounter', Time.now.to_i - 1 * DAYS), 0)
+  end
+  it "can compute deltas over time on events correctly" do
+    # A larger-scale test that simulates user join events and tests that we can get
+    # week-by-week deltas. Generate new user counts according to the following
+    # weekly schedule: 55780 during the first week, 300 more during the next week,
+    # 10 more the next week, etc.
+    schedule = [55780, 300, 10, 4000, 1000, 1000, 5000, 15000, 30000, 3000]
+    schedule.each_with_index do |num_users, i|
+      Timecop.travel(Time.now - (schedule.length * WEEKS) + (i * WEEKS)) do
+        num_users.times do |i|
+          Timecop.travel(Time.now + 2 * HOURS + i) do
+            @counter.add("users", "user#{SecureRandom.uuid}")
+          end
+        end
+      end
+    end
+    actual_total = schedule.reduce(:+)
+    estimated_total = @counter.count("users")
+    (actual_total - estimated_total).abs.should < @expected_relative_error * actual_total
+    # Go through the schedule, computing week-by-week deltas and comparing them to the
+    # scheduled additions.
+    schedule.each_with_index do |users_joined, i|
+      week = schedule.length - 1 - i
+      c = @counter.count('users', Time.now.to_i - (week+1) * WEEKS) - @counter.count('users', Time.now.to_i - week * WEEKS)
+      (users_joined - c).abs.should < @expected_relative_error * schedule.reduce(:+)
+    end
+  end
+end