hyperloglog-redis 0.1.0 → 0.2.0

data/README.md

@@ -14,7 +14,16 @@ instance is used for storing the counters. A simple example:
       counter.add('beatles', beatle)
     end
 
-    puts "There are approximately #{counter.count('beatles')} distinct beatles!"
+    puts "There are approximately #{counter.count('beatles')} distinct Beatles"
+
+You can also ask for an estimate from multiple counters and you'll get
+an estimate of the size of their union:
+
+    ['joe', 'denny', 'linda', 'jimmy', 'paul'].each do |wing_member|
+      counter.add('wings', wing_member)
+    end
+
+    puts "There are approximately #{counter.count('beatles', 'wings')} people who were in the Beatles or Wings"
 
 Each HyperLogLog counter uses a small, fixed amount of space but can
 estimate the cardinality of any set of up to around a billion values with

data/VERSION

@@ -1 +1 @@
-0.1.0
+0.2.0

data/hyperloglog-redis.gemspec

@@ -0,0 +1,64 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = "hyperloglog-redis"
+  s.version = "0.2.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Aaron Windsor"]
+  s.date = "2012-09-27"
+  s.description = "An implementation of the HyperLogLog set cardinality estimation algorithm in Ruby using Redis as a back-end"
+  s.email = "aaron.windsor@gmail.com"
+  s.extra_rdoc_files = [
+    "LICENSE.txt",
+    "README.md"
+  ]
+  s.files = [
+    ".document",
+    ".rspec",
+    "Gemfile",
+    "Gemfile.lock",
+    "LICENSE.txt",
+    "README.md",
+    "Rakefile",
+    "VERSION",
+    "hyperloglog-redis.gemspec",
+    "lib/hyper_log_log.rb",
+    "lib/hyperloglog-redis.rb",
+    "spec/hyper_log_log_spec.rb",
+    "spec/spec_helper.rb"
+  ]
+  s.homepage = "http://github.com/aaw/hyperloglog-redis"
+  s.licenses = ["MIT"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = "1.8.10"
+  s.summary = "An implementation of the HyperLogLog set cardinality estimation algorithm in Ruby using Redis as a back-end"
+
+  if s.respond_to? :specification_version then
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<murmurhash3>, ["~> 0.1.3"])
+      s.add_runtime_dependency(%q<redis>, ["~> 3.0.1"])
+      s.add_development_dependency(%q<jeweler>, ["~> 1.8.4"])
+      s.add_development_dependency(%q<rake>, ["~> 0.9.2.2"])
+      s.add_development_dependency(%q<rspec>, ["~> 2.11.0"])
+    else
+      s.add_dependency(%q<murmurhash3>, ["~> 0.1.3"])
+      s.add_dependency(%q<redis>, ["~> 3.0.1"])
+      s.add_dependency(%q<jeweler>, ["~> 1.8.4"])
+      s.add_dependency(%q<rake>, ["~> 0.9.2.2"])
+      s.add_dependency(%q<rspec>, ["~> 2.11.0"])
+    end
+  else
+    s.add_dependency(%q<murmurhash3>, ["~> 0.1.3"])
+    s.add_dependency(%q<redis>, ["~> 3.0.1"])
+    s.add_dependency(%q<jeweler>, ["~> 1.8.4"])
+    s.add_dependency(%q<rake>, ["~> 0.9.2.2"])
+    s.add_dependency(%q<rspec>, ["~> 2.11.0"])
+  end
+end
+

data/lib/hyper_log_log.rb

@@ -26,9 +26,12 @@ class HyperLogLog
      @redis.zadd(counter_name, [(max_run_of_zeros || 0), rho(w)].max, function_name)
   end
 
-  def count(counter_name)
-    all_estimates = @redis.zrange(counter_name, 0, -1, {withscores: true})
-    estimate_sum = all_estimates.map{ |f, score| 2 ** -score }.reduce(:+) || 0
+  def count(*counter_names)
+    all_estimates = counter_names.map{ |counter_name| @redis.zrange(counter_name, 0, -1, {withscores: true}) }
+                                 .reduce(:concat)
+                                 .group_by{ |value, score| value }
+                                 .map{ |group, counters| 2 ** -counters.map{ |x| x.last }.max }
+    estimate_sum = all_estimates.reduce(:+) || 0
     estimate = @alpha * @m * @m * ((estimate_sum + @m - all_estimates.length) ** -1)
     if estimate <= 2.5 * @m
       if all_estimates.length == @m

data/spec/hyper_log_log_spec.rb

@@ -58,7 +58,7 @@ describe HyperLogLog do
   # implementation, since it exercises all of the cases in HyperLogLog's
   # count method except for the correction for very large set sizes.
 
-  it "produces acceptable estimates" do
+  it "produces acceptable estimates for counts" do
     max_items = 1000
     redis = Redis.new
     (6..16).each do |b|
@@ -81,4 +81,59 @@ describe HyperLogLog do
     end
   end
 
+  it "produces acceptable estimates for unions with few elements in common" do
+    b, max_items = 10, 2000
+    counter = HyperLogLog.new(Redis.new, b)
+    bad_estimates = 0
+    very_bad_estimates = 0
+    expected_relative_error = 1.04 / Math.sqrt(2 ** b)
+    max_items.times do |i|
+      value1 = Digest::MD5.hexdigest("value#{i}")
+      counter.add("mycounter1", value1)
+      value2 = Digest::MD5.hexdigest("value#{i}incounter2")
+      counter.add("mycounter2", value2)
+      value3 = Digest::MD5.hexdigest("this is value#{i}")
+      counter.add("mycounter3", value3)
+      actual = 3 * (i + 1)
+      approximate = counter.count("mycounter1", "mycounter2", "mycounter3")
+      relative_error = (actual - approximate).abs / Float(actual)
+      bad_estimates += 1 if relative_error > expected_relative_error * 2
+      very_bad_estimates += 1 if relative_error > expected_relative_error * 3
+    end
+    bad_estimates.should < (3 * max_items) / 100.00
+    very_bad_estimates.should == 0
+  end
+
+  it "produces acceptable estimates for unions with many elements in common" do
+    b, max_items, intersection_size = 10, 1000, 2000
+    counter = HyperLogLog.new(Redis.new, b)
+    bad_estimates = 0
+    very_bad_estimates = 0
+    expected_relative_error = 1.04 / Math.sqrt(2 ** b)
+
+    intersection_size.times do |i|
+      value = Digest::MD5.hexdigest("test#{i}value")
+      ['mycounter1', 'mycounter2', 'mycounter3'].each do |counter_name|
+        counter.add(counter_name, value)
+      end
+    end
+
+    max_items.times do |i|
+      value1 = Digest::MD5.hexdigest("value#{i}")
+      counter.add("mycounter1", value1)
+      value2 = Digest::MD5.hexdigest("value#{i}isincounter2")
+      counter.add("mycounter2", value2)
+      value3 = Digest::MD5.hexdigest("this is value#{i}")
+      counter.add("mycounter3", value3)
+      actual = 3 * (i + 1) + intersection_size
+      approximate = counter.count("mycounter1", "mycounter2", "mycounter3")
+      relative_error = (actual - approximate).abs / Float(actual)
+      bad_estimates += 1 if relative_error > expected_relative_error * 2
+      very_bad_estimates += 1 if relative_error > expected_relative_error * 3
+    end
+
+    bad_estimates.should < ((3 * max_items) + intersection_size) / 100.00
+    very_bad_estimates.should == 0
+  end
+
 end

data/spec/spec_helper.rb

@@ -23,4 +23,7 @@ RSpec.configure do |config|
   config.before(:each) do
     Redis.new.flushdb
   end
+  config.after(:each) do
+    Redis.new.flushdb
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: hyperloglog-redis
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-09-07 00:00:00.000000000Z
+date: 2012-09-27 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: murmurhash3
-  requirement: &2159101840 !ruby/object:Gem::Requirement
+  requirement: &2172688860 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,10 +21,10 @@ dependencies:
         version: 0.1.3
   type: :runtime
   prerelease: false
-  version_requirements: *2159101840
+  version_requirements: *2172688860
 - !ruby/object:Gem::Dependency
   name: redis
-  requirement: &2159101360 !ruby/object:Gem::Requirement
+  requirement: &2172688380 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -32,10 +32,10 @@ dependencies:
         version: 3.0.1
   type: :runtime
   prerelease: false
-  version_requirements: *2159101360
+  version_requirements: *2172688380
 - !ruby/object:Gem::Dependency
   name: jeweler
-  requirement: &2159100880 !ruby/object:Gem::Requirement
+  requirement: &2172687900 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -43,10 +43,10 @@ dependencies:
         version: 1.8.4
   type: :development
   prerelease: false
-  version_requirements: *2159100880
+  version_requirements: *2172687900
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &2159100400 !ruby/object:Gem::Requirement
+  requirement: &2172687420 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -54,10 +54,10 @@ dependencies:
         version: 0.9.2.2
   type: :development
   prerelease: false
-  version_requirements: *2159100400
+  version_requirements: *2172687420
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &2159099920 !ruby/object:Gem::Requirement
+  requirement: &2172686940 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -65,7 +65,7 @@ dependencies:
         version: 2.11.0
   type: :development
   prerelease: false
-  version_requirements: *2159099920
+  version_requirements: *2172686940
 description: An implementation of the HyperLogLog set cardinality estimation algorithm
   in Ruby using Redis as a back-end
 email: aaron.windsor@gmail.com
@@ -83,6 +83,7 @@ files:
 - README.md
 - Rakefile
 - VERSION
+- hyperloglog-redis.gemspec
 - lib/hyper_log_log.rb
 - lib/hyperloglog-redis.rb
 - spec/hyper_log_log_spec.rb
@@ -102,7 +103,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 4418438529594871493
+      hash: -1980876148320618453
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements: