hyperloglog-redis 0.2.0 → 0.3.0

data/README.md

@@ -16,15 +16,6 @@ instance is used for storing the counters. A simple example:
 
     puts "There are approximately #{counter.count('beatles')} distinct Beatles"
 
-You can also ask for an estimate from multiple counters and you'll get
-an estimate of the size of their union:
-
-    ['joe', 'denny', 'linda', 'jimmy', 'paul'].each do |wing_member|
-      counter.add('wings', wing_member)
-    end
-
-    puts "There are approximately #{counter.count('beatles', 'wings')} people who were in the Beatles or Wings"
-
 Each HyperLogLog counter uses a small, fixed amount of space but can
 estimate the cardinality of any set of up to around a billion values with
 relative error of about 1.04 / Math.sqrt(2 ** b), where b is a parameter
@@ -49,6 +40,34 @@ algorithm"](http://algo.inria.fr/flajolet/Publications/FlFuGaMe07.pdf)
 by Flajolet, Fusy, Gandouet, and Meunier. Our implementation closely 
 follows the program described in Section 4 of that paper.
 
+Unions and intersections
+========================
+
+You can also ask for an estimate of the union from multiple counters:
+
+    ['joe', 'denny', 'linda', 'jimmy', 'paul'].each do |wings_member|
+      counter.add('wings', wings_member)
+    end
+
+    puts "There are approximately #{counter.union('beatles', 'wings')} people who were in the Beatles or Wings"
+
+The same relative error guarantee above applies to unions: a union of
+size N can be estimated to within N * (1.04 / Math.sqrt(2 ** b)) elements,
+regardless of how many HyperLogLog counters that union spans.
+
+Intersections can also be estimated:
+
+    puts "There are approximately #{counter.intersection('beatles', 'wings')} people who were in both the Beatles and Wings"
+
+However, intersections of HyperLogLog counters are calculated indirectly via the
+[inclusion/exclusion principle](http://en.wikipedia.org/wiki/Inclusion%E2%80%93exclusion_principle)
+as a sum of unions and there aren't good theoretical bounds on the error of that sum. In
+practice, the estimates that come out of small intersections tend to follow the
+same relative error patterns, but beware using this type of estimation on large
+intersections, both because the errors can be much larger than those guaranteed
+for unions and the complexity of computing intersections grows exponentially with 
+the number of counters being intersected.
+
 Installation
 ============
 

data/VERSION

@@ -1 +1 @@
-0.2.0
+0.3.0

data/hyperloglog-redis.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "hyperloglog-redis"
-  s.version = "0.2.0"
+  s.version = "0.3.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Aaron Windsor"]
-  s.date = "2012-09-27"
+  s.date = "2012-09-28"
   s.description = "An implementation of the HyperLogLog set cardinality estimation algorithm in Ruby using Redis as a back-end"
   s.email = "aaron.windsor@gmail.com"
   s.extra_rdoc_files = [

data/lib/hyper_log_log.rb

@@ -26,7 +26,19 @@ class HyperLogLog
      @redis.zadd(counter_name, [(max_run_of_zeros || 0), rho(w)].max, function_name)
   end
 
-  def count(*counter_names)
+  def count(counter_name)
+    union_helper([counter_name])
+  end
+
+  def union(*counter_names)
+    union_helper(counter_names)
+  end
+
+  def intersection(*counter_names)
+    [intersection_helper(counter_names, {}), 0].max
+  end
+
+  def union_helper(counter_names)
     all_estimates = counter_names.map{ |counter_name| @redis.zrange(counter_name, 0, -1, {withscores: true}) }
                                  .reduce(:concat)
                                  .group_by{ |value, score| value }
@@ -46,6 +58,15 @@ class HyperLogLog
     end
   end
 
+  def intersection_helper(counter_names, cache)
+    sum = union_helper(counter_names) - (1...counter_names.length).map do |k|
+      ((-1) ** (k + 1)) * counter_names.combination(k).map do |group| 
+        cache[group] ||= intersection_helper(group, cache) 
+      end.inject(0, :+)
+    end.inject(0, :+)
+    ((-1) ** (counter_names.length + 1)) * sum
+  end
+
   # rho(i) is the position of the first 1 in the binary representation of i,
   # reading from most significant to least significant bits. Some examples:
   # rho(1...) = 1, rho(001...) = 3, rho(000...0) = @bits_in_hash + 1

data/spec/hyper_log_log_spec.rb

@@ -95,7 +95,7 @@ describe HyperLogLog do
       value3 = Digest::MD5.hexdigest("this is value#{i}")
       counter.add("mycounter3", value3)
       actual = 3 * (i + 1)
-      approximate = counter.count("mycounter1", "mycounter2", "mycounter3")
+      approximate = counter.union("mycounter1", "mycounter2", "mycounter3")
       relative_error = (actual - approximate).abs / Float(actual)
       bad_estimates += 1 if relative_error > expected_relative_error * 2
       very_bad_estimates += 1 if relative_error > expected_relative_error * 3
@@ -126,7 +126,7 @@ describe HyperLogLog do
       value3 = Digest::MD5.hexdigest("this is value#{i}")
       counter.add("mycounter3", value3)
       actual = 3 * (i + 1) + intersection_size
-      approximate = counter.count("mycounter1", "mycounter2", "mycounter3")
+      approximate = counter.union("mycounter1", "mycounter2", "mycounter3")
       relative_error = (actual - approximate).abs / Float(actual)
       bad_estimates += 1 if relative_error > expected_relative_error * 2
       very_bad_estimates += 1 if relative_error > expected_relative_error * 3
@@ -136,4 +136,56 @@ describe HyperLogLog do
     very_bad_estimates.should == 0
   end
 
+  # There are no good theoretical guarantees that I know of for arbitrary
+  # intersection estimation, since it's expessed as the sum of unions of
+  # HyperLogLog counters, but it tends to work okay in practice, as seen below.
+
+  it "produces decent estimates for intersections" do
+    b, max_items = 6, 1000
+    counter = HyperLogLog.new(Redis.new, b)
+    expected_relative_error = 1.04 / Math.sqrt(2 ** b)
+
+    max_items.times do |i|
+      value1 = Digest::MD5.hexdigest("first-value#{i}")
+      value2 = Digest::MD5.hexdigest("second-value#{i}")
+      value3 = Digest::MD5.hexdigest("third-value#{i}")
+      value4 = Digest::MD5.hexdigest("fourth-value#{i}")
+      counter.add("mycounter1", value1)
+      counter.add("mycounter2", value2)
+      counter.add("mycounter3", value3)
+      counter.add("mycounter4", value4)
+      [value1, value2, value3, value4].each{ |value| counter.add("mycounter5", value) }
+    end
+
+    small_counters = ['mycounter1', 'mycounter2', 'mycounter3', 'mycounter4']
+    
+    small_counters.each do |counter_name|
+      intersection_estimate = counter.intersection(counter_name, 'mycounter5')
+      intersection_estimate.should > 0
+      (intersection_estimate - counter.count(counter_name)).abs.should < max_items * expected_relative_error
+    end
+
+    [2,3].each do |intersection_size|
+      small_counters.combination(intersection_size).each do |counter_names|
+        intersection_estimate = counter.intersection(*counter_names)
+        intersection_estimate.should >= 0
+        intersection_estimate.should < intersection_size * max_items * expected_relative_error
+      end
+    end
+
+    100.times do |i|
+      value = Digest::MD5.hexdigest("somethingintheintersection#{i}")
+      small_counters.each { |counter_name| counter.add(counter_name, value) }
+    end
+
+    [2,3,4].each do |intersection_size|
+      small_counters.combination(intersection_size).each do |counter_names|
+        intersection_estimate = counter.intersection(*counter_names)
+        intersection_estimate.should >= 0
+        (intersection_estimate - 100).abs.should < intersection_size * (max_items + 100) * expected_relative_error
+      end
+    end
+
+  end
+
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: hyperloglog-redis
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-09-27 00:00:00.000000000Z
+date: 2012-09-28 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: murmurhash3
-  requirement: &2172688860 !ruby/object:Gem::Requirement
+  requirement: &2157248980 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,10 +21,10 @@ dependencies:
         version: 0.1.3
   type: :runtime
   prerelease: false
-  version_requirements: *2172688860
+  version_requirements: *2157248980
 - !ruby/object:Gem::Dependency
   name: redis
-  requirement: &2172688380 !ruby/object:Gem::Requirement
+  requirement: &2157248500 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -32,10 +32,10 @@ dependencies:
         version: 3.0.1
   type: :runtime
   prerelease: false
-  version_requirements: *2172688380
+  version_requirements: *2157248500
 - !ruby/object:Gem::Dependency
   name: jeweler
-  requirement: &2172687900 !ruby/object:Gem::Requirement
+  requirement: &2157248020 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -43,10 +43,10 @@ dependencies:
         version: 1.8.4
   type: :development
   prerelease: false
-  version_requirements: *2172687900
+  version_requirements: *2157248020
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &2172687420 !ruby/object:Gem::Requirement
+  requirement: &2157247540 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -54,10 +54,10 @@ dependencies:
         version: 0.9.2.2
   type: :development
   prerelease: false
-  version_requirements: *2172687420
+  version_requirements: *2157247540
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &2172686940 !ruby/object:Gem::Requirement
+  requirement: &2157247060 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -65,7 +65,7 @@ dependencies:
         version: 2.11.0
   type: :development
   prerelease: false
-  version_requirements: *2172686940
+  version_requirements: *2157247060
 description: An implementation of the HyperLogLog set cardinality estimation algorithm
   in Ruby using Redis as a back-end
 email: aaron.windsor@gmail.com
@@ -103,7 +103,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -1980876148320618453
+      hash: 1810415248006536813
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements: