hyll 0.1.0

data/examples/advance.rb

@@ -0,0 +1,258 @@
+# frozen_string_literal: true
+
+require "hyll"
+
+# ADVANCED USAGE EXAMPLES
+
+# Define User class outside of method scope
+class User
+  attr_reader :id, :email
+
+  def initialize(id, email)
+    @id = id
+    @email = email
+  end
+
+  # Override to_s for proper hashing
+  def to_s
+    "User:#{@id}:#{@email}"
+  end
+end
+
+# Example 1: Estimating Intersection Size Between Sets
+def intersection_example
+  puts "=== Intersection Estimation Example ==="
+
+  # Create two HyperLogLog counters with higher precision for better accuracy
+  hll1 = Hyll.new(precision: 14)
+  hll2 = Hyll.new(precision: 14)
+
+  # Add elements with controlled overlap (30% overlap)
+  total_items = 100_000
+  overlap = (total_items * 0.3).to_i
+
+  # First set: 0 to 99,999
+  total_items.times { |i| hll1.add("item-#{i}") }
+
+  # Second set: 70,000 to 169,999 (30,000 items overlap)
+  total_items.times { |i| hll2.add("item-#{i + total_items - overlap}") }
+
+  # Create union by merging (make copy first to avoid modifying original)
+  union = hll1.to_enhanced
+  union.merge(hll2)
+
+  # Calculate intersection using inclusion-exclusion principle with standard estimates
+  estimate1 = hll1.cardinality
+  estimate2 = hll2.cardinality
+  union_estimate = union.cardinality
+  intersection_estimate = estimate1 + estimate2 - union_estimate
+
+  # Alternative calculation using maximum likelihood estimation for better accuracy
+  mle_estimate1 = hll1.maximum_likelihood_cardinality
+  mle_estimate2 = hll2.maximum_likelihood_cardinality
+  mle_union_estimate = union.maximum_likelihood_cardinality
+  mle_intersection_estimate = mle_estimate1 + mle_estimate2 - mle_union_estimate
+
+  # Apply bias correction for intersection estimates
+  # Intersection estimates often have larger relative error than individual cardinality estimates
+  correction_factor = 0.95 # Slight adjustment to reduce overestimation
+  corrected_intersection = intersection_estimate * correction_factor
+
+  # Calculate average of different estimation methods for better results
+  combined_intersection = (corrected_intersection + mle_intersection_estimate) / 2.0
+
+  puts "Set A cardinality: #{estimate1.round} (MLE: #{mle_estimate1.round})"
+  puts "Set B cardinality: #{estimate2.round} (MLE: #{mle_estimate2.round})"
+  puts "Union cardinality: #{union_estimate.round} (MLE: #{mle_union_estimate.round})"
+  puts "Estimated intersection:"
+  puts "  - Basic: #{intersection_estimate.round}"
+  puts "  - MLE: #{mle_intersection_estimate.round}"
+  puts "  - Corrected: #{corrected_intersection.round}"
+  puts "  - Combined: #{combined_intersection.round}"
+  puts "Actual intersection: #{overlap}"
+  puts "Error rates:"
+  puts "  - Basic: #{((intersection_estimate - overlap).abs / overlap * 100).round(2)}%"
+  puts "  - MLE: #{((mle_intersection_estimate - overlap).abs / overlap * 100).round(2)}%"
+  puts "  - Corrected: #{((corrected_intersection - overlap).abs / overlap * 100).round(2)}%"
+  puts "  - Combined: #{((combined_intersection - overlap).abs / overlap * 100).round(2)}%"
+  puts "\n"
+end
+
+# Example 2: Working with Custom Data Types
+def custom_data_types_example
+  puts "=== Custom Data Types Example ==="
+
+  # Create HLL counter
+  hll = Hyll.new
+
+  # Add custom objects
+  users = []
+  1000.times do |i|
+    # Some users will have the same email to simulate duplicates
+    email = "user#{i % 800}@example.com"
+    users << User.new(i, email)
+  end
+
+  # Add all users
+  users.each { |user| hll.add(user) }
+
+  # Check cardinality - should be close to 800 (unique emails)
+  puts "Added #{users.size} users with #{users.map(&:email).uniq.size} unique emails"
+  puts "HyperLogLog estimate: #{hll.cardinality.round}"
+  puts "\n"
+
+  # Track unique emails by domain
+  domains = {}
+
+  users.each do |user|
+    domain = user.email.split("@").last
+    domains[domain] ||= Hyll.new
+    domains[domain].add(user.email)
+  end
+
+  domains.each do |domain, counter|
+    puts "Domain #{domain}: ~#{counter.cardinality.round} unique emails"
+  end
+end
+
+# Example 3: Monitoring Stream Cardinality with Time Windows
+def time_window_example
+  puts "=== Time Window Monitoring Example ==="
+
+  # Create counters for different time windows
+  minute_counter = Hyll.new
+  hour_counter = Hyll.new
+  day_counter = Hyll.new
+
+  # Simulate time windows with different event rates
+  # For simplicity, we'll compress time in this example
+
+  puts "Simulating a stream of events with varying rates..."
+
+  # Simulate a day's worth of data
+  # Each "minute" has a different number of events
+  24.times do |hour|
+    puts "Hour #{hour}:"
+
+    # Reset minute counter each hour
+    minute_counter.reset
+
+    60.times do |minute|
+      # Generate some data for this minute
+      # Use time of day to vary the rate (busier during work hours)
+      base_rate = 100
+      time_factor = if (9..17).include?(hour)
+                      10  # Work hours - 10x more traffic
+                    elsif (18..22).include?(hour)
+                      5   # Evening - 5x more traffic
+                    else
+                      1   # Late night/early morning - base traffic
+                    end
+
+      # Add some randomness
+      rate = (base_rate * time_factor * (0.5 + rand)).to_i
+
+      # Add unique events for this minute
+      # Some IDs will repeat across minutes/hours to simulate returning users
+      rate.times do |i|
+        # Event ID combines hour, minute and unique ID
+        # We'll make some IDs repeat to simulate returning users
+        event_id = "user-#{(hour * 60 + minute + i) % 10_000}"
+
+        minute_counter.add(event_id)
+        hour_counter.add(event_id)
+        day_counter.add(event_id)
+      end
+
+      # Every 15 minutes, print stats
+      next unless minute % 15 == 14
+
+      puts "  Minute #{minute + 1} - Unique users in last:"
+      puts "    - Minute: #{minute_counter.cardinality.round}"
+      puts "    - Hour: #{hour_counter.cardinality.round}"
+      puts "    - Day so far: #{day_counter.cardinality.round}"
+    end
+
+    # Reset hour counter at end of day
+    hour_counter.reset unless hour == 23
+  end
+
+  puts "Simulation complete. Total unique users for the day: #{day_counter.cardinality.round}"
+  puts "\n"
+end
+
+# Example 4: Advanced Serialization and Storage
+def serialization_example
+  puts "=== Advanced Serialization Example ==="
+
+  # Create and populate HLL
+  hll = Hyll.new
+  puts "Adding 1 million items..."
+  1_000_000.times { |i| hll.add("user-#{i}") }
+
+  # Serialize to different formats
+  binary = hll.serialize
+
+  # Simulate storing in a database (Base64 encoded)
+  require "base64"
+  base64_string = Base64.strict_encode64(binary)
+
+  puts "Original cardinality: #{hll.cardinality.round}"
+  puts "Binary serialized size: #{binary.bytesize} bytes"
+  puts "Base64 serialized size: #{base64_string.bytesize} bytes"
+
+  # Demonstrate storage efficiency
+  puts "Storage efficiency: #{(1_000_000 * 8 / binary.bytesize).round}x compression ratio"
+
+  # Simulate retrieving and deserializing
+  retrieved_binary = Base64.strict_decode64(base64_string)
+  retrieved_hll = Hyll.deserialize(retrieved_binary)
+
+  puts "Retrieved cardinality: #{retrieved_hll.cardinality.round}"
+  puts "\n"
+
+  # Convert to enhanced format for interoperability
+  enhanced_hll = hll.to_enhanced
+  enhanced_binary = enhanced_hll.serialize
+
+  puts "Enhanced format serialized size: #{enhanced_binary.bytesize} bytes"
+  puts "\n"
+end
+
+# Example 5: Benchmark Different Precision Levels
+def precision_benchmark
+  puts "=== Precision Benchmark Example ==="
+
+  # Create HLLs with different precision levels
+  precisions = [6, 8, 10, 12, 14]
+  hlls = precisions.map { |p| Hyll.new(precision: p) }
+
+  # Number of unique elements to add
+  num_elements = 1_000_000
+
+  puts "Benchmarking with #{num_elements} unique elements"
+  puts "Precision | Memory (bytes) | Estimate | Error (%)"
+  puts "----------|----------------|----------|----------"
+
+  precisions.each_with_index do |precision, i|
+    # Add elements
+    num_elements.times { |j| hlls[i].add("element-#{j}") }
+
+    # Calculate statistics
+    serialized = hlls[i].serialize
+    memory_used = serialized.bytesize
+    estimate = hlls[i].cardinality
+    error_percent = ((estimate - num_elements).abs / num_elements.to_f * 100).round(2)
+
+    puts format("%9d | %14d | %8d | %9.2f", precision, memory_used, estimate.round, error_percent)
+  end
+end
+
+# Run all examples
+intersection_example
+custom_data_types_example
+time_window_example
+serialization_example
+precision_benchmark
+
+puts "Advanced examples completed!"

data/examples/basic.rb

@@ -0,0 +1,161 @@
+# frozen_string_literal: true
+
+require "hyll"
+
+# BASIC USAGE EXAMPLES
+
+# Example 1: Basic Counting
+puts "Example 1: Basic counting of unique elements"
+counter = Hyll::HyperLogLog.new
+100.times { |i| counter.add(i) }
+puts "Added 100 unique numbers. Estimated count: #{counter.count}"
+puts "Raw cardinality estimate: #{counter.cardinality}"
+puts "\n"
+
+# Example 2: MLE Estimation
+puts "Example 2: Using Maximum Likelihood Estimation"
+counter = Hyll::HyperLogLog.new
+1000.times { |i| counter.add(i) }
+puts "Standard estimation: #{counter.cardinality}"
+puts "MLE estimation: #{counter.mle_cardinality}"
+puts "\n"
+
+# Example 3: Custom Precision
+puts "Example 3: Setting custom precision"
+# Lower precision (less memory, less accuracy)
+counter_low = Hyll::HyperLogLog.new(6)
+# Higher precision (more memory, more accuracy)
+counter_high = Hyll::HyperLogLog.new(14)
+
+10_000.times do |i|
+  counter_low.add(i)
+  counter_high.add(i)
+end
+
+puts "Low precision (2^6 registers): #{counter_low.count}"
+puts "High precision (2^14 registers): #{counter_high.count}"
+puts "\n"
+
+# Example 4: Adding non-integer elements
+puts "Example 4: Counting different data types"
+counter = Hyll::HyperLogLog.new
+%w[apple banana cherry apple durian].each { |fruit| counter.add(fruit) }
+puts "Unique fruit count: #{counter.count}" # Should be approximately 4
+
+counter.reset
+["user1@example.com", "user2@example.com", "USER1@example.com"].each { |email| counter.add(email.downcase) }
+puts "Unique email count (case insensitive): #{counter.count}" # Should be approximately 2
+puts "\n"
+
+# ADVANCED USAGE EXAMPLES
+
+# Example 5: Merging counters
+puts "Example 5: Merging counters"
+counter1 = Hyll::HyperLogLog.new
+counter2 = Hyll::HyperLogLog.new
+
+# Add some unique elements to each counter
+100.times { |i| counter1.add("item-#{i}") }
+100.times { |i| counter2.add("item-#{i + 50}") } # 50 overlapping items
+
+puts "Counter 1 estimate: #{counter1.count}"
+puts "Counter 2 estimate: #{counter2.count}"
+
+# Merge counter2 into counter1
+counter1.merge(counter2)
+puts "Merged counter estimate: #{counter1.count}" # Should be approximately 150
+puts "\n"
+
+# Example 6: Serialization
+puts "Example 6: Serializing and deserializing"
+original = Hyll::HyperLogLog.new
+1000.times { |i| original.add(i) }
+
+# Serialize the counter
+serialized = original.serialize
+puts "Serialized size: #{serialized.bytesize} bytes"
+
+# Deserialize
+deserialized = Hyll::HyperLogLog.deserialize(serialized)
+puts "Original count: #{original.count}"
+puts "Deserialized count: #{deserialized.count}"
+puts "\n"
+
+# Example 7: Using EnhancedHyperLogLog
+puts "Example 7: Using EnhancedHyperLogLog"
+enhanced_counter = Hyll::EnhancedHyperLogLog.new(10)
+10_000.times { |i| enhanced_counter.add(i) }
+puts "EnhancedHyperLogLog count: #{enhanced_counter.count}"
+
+# Convert standard HLL to EnhancedHLL
+standard = Hyll::HyperLogLog.new(10)
+10_000.times { |i| standard.add(i) }
+enhanced_converted = standard.to_enhanced
+puts "Standard HLL converted to Enhanced: #{enhanced_converted.count}"
+
+# Convert EnhancedHLL back to standard HLL
+standard_again = enhanced_counter.to_hll
+puts "Enhanced converted back to standard: #{standard_again.count}"
+puts "\n"
+
+# Example 8: Batch Adding
+puts "Example 8: Batch adding elements"
+counter = Hyll::HyperLogLog.new
+elements = (1..10_000).to_a
+start_time = Time.now
+counter.add_all(elements)
+end_time = Time.now
+puts "Added 10,000 elements in batch: #{counter.count}"
+puts "Time taken: #{end_time - start_time} seconds"
+puts "\n"
+
+# Example 9: Dealing with large datasets
+puts "Example 9: Memory efficiency with large datasets"
+counter = Hyll::HyperLogLog.new(12) # 2^12 = 4096 registers
+puts "Memory usage for 100 million elements is roughly the same as for 1000 elements"
+puts "Sparse representation used until #{Hyll::Constants::DEFAULT_SPARSE_THRESHOLD} elements are added"
+
+# Simulate adding 1000 elements and check memory footprint
+1000.times { |i| counter.add(i) }
+puts "Estimated memory for 1000 elements: #{counter.serialize.bytesize} bytes"
+
+# Example 10: Estimating intersection size
+puts "Example 10: Estimating intersection size"
+set_a = Hyll::HyperLogLog.new
+set_b = Hyll::HyperLogLog.new
+
+# Add elements to both sets with some overlap
+1000.times { |i| set_a.add("item-#{i}") }
+1000.times { |i| set_b.add("item-#{i + 500}") } # 500 overlapping items
+
+# Create a union set by merging
+union = set_a.to_enhanced # Make a copy first
+union.merge(set_b)
+
+# Estimate intersection using inclusion-exclusion principle
+# |A ∩ B| = |A| + |B| - |A ∪ B|
+intersection_size = set_a.count + set_b.count - union.count
+puts "Set A size: #{set_a.count}"
+puts "Set B size: #{set_b.count}"
+puts "Union size: #{union.count}"
+puts "Estimated intersection size: #{intersection_size} (actual: 500)"
+puts "\n"
+
+# Example 11: Streaming data application
+puts "Example 11: Streaming data application"
+puts "HyperLogLog is perfect for streaming applications where you can't store all data:"
+
+counter = Hyll::HyperLogLog.new
+puts "Imagine processing a stream of user IDs from web logs..."
+# Simulate stream processing
+10_000.times do
+  # In a real stream, you'd process each item as it arrives
+  user_id = rand(5000) # Simulate about 5000 unique users
+  counter.add(user_id)
+
+  # Periodically report statistics without storing all IDs
+  puts "Processed #{user_id} records, estimated unique users: #{counter.count}" if (user_id % 2500).zero?
+end
+
+puts "Final unique user estimate: #{counter.count}"
+puts "All this with minimal memory usage and O(1) update time!"