hyperloglog-redis 0.3.0 → 1.0.0

data/HISTORY.md

@@ -0,0 +1,17 @@
+## 1.0.0 (10/26/2012)
+
+* Changed the underlying storage from Redis sorted sets to Redis hashes. This
+  is a breaking change, if you have existing counters stored from earlier
+  versions of this library, you can upgrade them with something like the
+  following method:
+
+      def upgrade(counter, redis)
+        return if redis.type(counter) == "hash"
+        values = redis.zrange(counter, 0, -1, {withscores: true})
+        redis.del(counter)
+        values.each { |key, value| redis.hset(counter, key, value.to_i) }
+      end
+
+* Added union_store command, which stores the results of a union for querying
+  or combining with other sets later
+

data/README.md

@@ -53,7 +53,12 @@ You can also ask for an estimate of the union from multiple counters:
 
 The same relative error guarantee above applies to unions: a union of
 size N can be estimated to within N * (1.04 / Math.sqrt(2 ** b)) elements,
-regardless of how many HyperLogLog counters that union spans.
+regardless of how many HyperLogLog counters that union spans. You can store 
+a unioned counter for querying or combining later with `union_store`:
+
+    counter.union_store('all_beatles_and_wings_members', 'beatles', 'wings')
+    
+    puts "There are approximately #{counter.count('all_beatles_and_wings_members'}} people who were in the Beatles or Wings"
 
 Intersections can also be estimated:
 

data/VERSION

@@ -1 +1 @@
-0.3.0
+1.0.0

data/hyperloglog-redis.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "hyperloglog-redis"
-  s.version = "0.3.0"
+  s.version = "1.0.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Aaron Windsor"]
-  s.date = "2012-09-28"
+  s.date = "2012-10-26"
   s.description = "An implementation of the HyperLogLog set cardinality estimation algorithm in Ruby using Redis as a back-end"
   s.email = "aaron.windsor@gmail.com"
   s.extra_rdoc_files = [
@@ -21,6 +21,7 @@ Gem::Specification.new do |s|
     ".rspec",
     "Gemfile",
     "Gemfile.lock",
+    "HISTORY.md",
     "LICENSE.txt",
     "README.md",
     "Rakefile",

data/lib/hyper_log_log.rb

@@ -19,30 +19,47 @@ class HyperLogLog
   end
 
   def add(counter_name, value)
-     hash = MurmurHash3::V32.murmur3_32_str_hash(value)
-     function_name = (hash % @m).to_s
-     w = hash / @m
-     max_run_of_zeros = @redis.zscore(counter_name, function_name)
-     @redis.zadd(counter_name, [(max_run_of_zeros || 0), rho(w)].max, function_name)
+    hash = MurmurHash3::V32.murmur3_32_str_hash(value)
+    function_name = hash % @m
+    w = hash / @m
+    existing_value = (@redis.hget(counter_name, function_name) || 0).to_i
+    new_value = [existing_value, rho(w)].max
+    @redis.hset(counter_name, function_name, new_value) if new_value > existing_value
   end
 
+  # Estimate the cardinality of a single set
   def count(counter_name)
     union_helper([counter_name])
   end
 
+  # Estimate the cardinality of the union of several sets
   def union(*counter_names)
     union_helper(counter_names)
   end
 
+  # Store the union of several sets in *destination* so that it can be used as 
+  # a HyperLogLog counter later.
+  def union_store(destination, *counter_names)
+    raw_union(counter_names).each do |key, count|
+      @redis.hset(destination, key, count)
+    end
+  end
+
+  # Estimate the cardinality of the intersection of several sets. We do this by 
+  # using the principle of inclusion and exclusion to represent the size of the
+  # intersection as the alternating sum of an exponential number of 
+  # cardinalities of unions of smaller sets.
   def intersection(*counter_names)
-    [intersection_helper(counter_names, {}), 0].max
+    icount = (1..counter_names.length).map do |k|
+      counter_names.combination(k).map do |group|
+        ((k % 2 == 0) ? -1 : 1) * union_helper(group)
+      end.inject(0, :+)
+    end.inject(0, :+)
+    [icount, 0].max
   end
 
   def union_helper(counter_names)
-    all_estimates = counter_names.map{ |counter_name| @redis.zrange(counter_name, 0, -1, {withscores: true}) }
-                                 .reduce(:concat)
-                                 .group_by{ |value, score| value }
-                                 .map{ |group, counters| 2 ** -counters.map{ |x| x.last }.max }
+    all_estimates = raw_union(counter_names).map{ |value, score| 2 ** -score }
     estimate_sum = all_estimates.reduce(:+) || 0
     estimate = @alpha * @m * @m * ((estimate_sum + @m - all_estimates.length) ** -1)
     if estimate <= 2.5 * @m
@@ -58,13 +75,11 @@ class HyperLogLog
     end
   end
 
-  def intersection_helper(counter_names, cache)
-    sum = union_helper(counter_names) - (1...counter_names.length).map do |k|
-      ((-1) ** (k + 1)) * counter_names.combination(k).map do |group| 
-        cache[group] ||= intersection_helper(group, cache) 
-      end.inject(0, :+)
-    end.inject(0, :+)
-    ((-1) ** (counter_names.length + 1)) * sum
+  def raw_union(counter_names)
+    counter_names.map{ |counter_name| @redis.hgetall(counter_name).map{ |x,y| [x, y.to_i] } }
+                 .reduce(:concat)
+                 .group_by{ |key, count| key }
+                 .map{ |key, counters| [key, counters.map{ |x| x.last }.max] }
   end
 
   # rho(i) is the position of the first 1 in the binary representation of i,
@@ -77,4 +92,5 @@ class HyperLogLog
       @bits_in_hash - Math.log(i, 2).floor
     end
   end
+
 end

data/spec/hyper_log_log_spec.rb

@@ -39,6 +39,39 @@ describe HyperLogLog do
     counter.count("counter1").should > counter.count("counter2")
   end
 
+  it "can exactly count small sets" do
+    redis = Redis.new
+    counter = HyperLogLog.new(redis, 11)
+    10.times { |i| counter.add("mycounter", i.to_s) }
+    counter.count("mycounter").should == 10
+  end
+
+  it "can exactly count small unions" do
+    redis = Redis.new
+    counter = HyperLogLog.new(redis, 11)
+    (1..8).each { |i| counter.add("mycounter1", i.to_s) }
+    (5..12).each { |i| counter.add("mycounter2", i.to_s) }
+    counter.union("mycounter1", "mycounter2").should == 12
+  end
+
+  it "can exactly count small intersections" do
+    redis = Redis.new
+    counter = HyperLogLog.new(redis, 11)
+    (1..8).each { |i| counter.add("mycounter1", i.to_s) }
+    (5..12).each { |i| counter.add("mycounter2", i.to_s) }
+    counter.intersection("mycounter1", "mycounter2").should == 4
+  end
+
+  it "can store unions for querying later" do
+    redis = Redis.new
+    counter = HyperLogLog.new(redis, 11)
+    (1..10).each { |i| counter.add("mycounter1", i.to_s) }
+    (5..15).each { |i| counter.add("mycounter2", i.to_s) }
+    (15..25).each { |i| counter.add("mycounter3", i.to_s) }
+    (20..50).each { |i| counter.add("mycounter4", i.to_s) }
+    counter.union_store("aggregate_counter", "mycounter1", "mycounter2", "mycounter3", "mycounter4")
+    counter.union("mycounter1", "mycounter2", "mycounter3", "mycounter4").should == counter.count("aggregate_counter")
+  end
 
   # With parameter b, HyperLogLog should produce estimates that have
   # relative error of 1.04 / Math.sqrt(2 ** b). Of course, this analysis

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: hyperloglog-redis
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 1.0.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-09-28 00:00:00.000000000Z
+date: 2012-10-26 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: murmurhash3
-  requirement: &2157248980 !ruby/object:Gem::Requirement
+  requirement: &2172774780 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,10 +21,10 @@ dependencies:
         version: 0.1.3
   type: :runtime
   prerelease: false
-  version_requirements: *2157248980
+  version_requirements: *2172774780
 - !ruby/object:Gem::Dependency
   name: redis
-  requirement: &2157248500 !ruby/object:Gem::Requirement
+  requirement: &2172774300 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -32,10 +32,10 @@ dependencies:
         version: 3.0.1
   type: :runtime
   prerelease: false
-  version_requirements: *2157248500
+  version_requirements: *2172774300
 - !ruby/object:Gem::Dependency
   name: jeweler
-  requirement: &2157248020 !ruby/object:Gem::Requirement
+  requirement: &2172773820 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -43,10 +43,10 @@ dependencies:
         version: 1.8.4
   type: :development
   prerelease: false
-  version_requirements: *2157248020
+  version_requirements: *2172773820
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &2157247540 !ruby/object:Gem::Requirement
+  requirement: &2172773340 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -54,10 +54,10 @@ dependencies:
         version: 0.9.2.2
   type: :development
   prerelease: false
-  version_requirements: *2157247540
+  version_requirements: *2172773340
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &2157247060 !ruby/object:Gem::Requirement
+  requirement: &2172789220 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -65,7 +65,7 @@ dependencies:
         version: 2.11.0
   type: :development
   prerelease: false
-  version_requirements: *2157247060
+  version_requirements: *2172789220
 description: An implementation of the HyperLogLog set cardinality estimation algorithm
   in Ruby using Redis as a back-end
 email: aaron.windsor@gmail.com
@@ -79,6 +79,7 @@ files:
 - .rspec
 - Gemfile
 - Gemfile.lock
+- HISTORY.md
 - LICENSE.txt
 - README.md
 - Rakefile
@@ -103,7 +104,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 1810415248006536813
+      hash: 2426569210961737114
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements: