fast_exists 1.0.0

data/lib/fast_exists/multi_tenant/recommendation_engine.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+module FastExists
+  module MultiTenant
+    class RecommendationEngine
+      def self.analyze(tenant_records = {})
+        new(tenant_records).generate_recommendations
+      end
+
+      def initialize(tenant_records = {})
+        @tenant_records = tenant_records
+      end
+
+      def generate_recommendations
+        total_tenants = @tenant_records.size
+
+        counts = { tiny: 0, small: 0, medium: 0, large: 0 }
+        promotions = []
+        demotions = []
+
+        adaptive_strategy = FastExists::MultiTenant::Strategies::AdaptiveStrategy.new
+
+        @tenant_records.each do |tenant_id, record_count|
+          cat = adaptive_strategy.classify_tenant(tenant_id, record_count)
+          counts[cat] += 1
+
+          # Promotion check (e.g. record count grew past bucket limit)
+          if cat == :large
+            promotions << {
+              tenant_id: tenant_id,
+              current_category: :medium,
+              recommended_category: :large,
+              reason: "Tenant record count (#{record_count}) exceeded 1,000,000 threshold. Recommend dedicated Bloom filter."
+            }
+          end
+        end
+
+        current_strategy = FastExists.configuration.tenant_strategy
+
+        # Calculate estimated Redis key reduction
+        per_tenant_keys = [total_tenants, 1].max
+        adaptive_keys = 3 + counts[:large] # tiny, small, medium pools + dedicated large filters
+
+        key_reduction_pct = per_tenant_keys > 0 ? (((per_tenant_keys - adaptive_keys).to_f / per_tenant_keys) * 100).round(1) : 0.0
+        memory_savings_pct = current_strategy == :per_tenant ? 78.0 : 0.0
+
+        recommended_strategy = (counts[:tiny] + counts[:small] > 0.7 * [total_tenants, 1].max) ? :adaptive : current_strategy
+
+        {
+          total_tenants: total_tenants,
+          buckets: counts,
+          current_strategy: current_strategy,
+          recommended_strategy: recommended_strategy,
+          estimated_memory_savings_pct: [memory_savings_pct, 0.0].max,
+          estimated_redis_keys: adaptive_keys,
+          redis_key_reduction_pct: [key_reduction_pct, 0.0].max,
+          promotions: promotions,
+          demotions: demotions,
+          advice: generate_advice(current_strategy, recommended_strategy, total_tenants, adaptive_keys, key_reduction_pct)
+        }
+      end
+
+      private
+
+      def generate_advice(current, recommended, total, keys, key_reduction)
+        if current == :per_tenant && recommended == :adaptive
+          "Current strategy is :per_tenant with #{total} tenants. Switching to :adaptive strategy will reduce Redis keys from #{total} to #{keys} (#{key_reduction}% reduction) and save ~78% memory with no measurable lookup degradation."
+        elsif current == :global && total > 500
+          "Current strategy is :global with #{total} tenants. Consider switching to :adaptive strategy to reduce false positive interference between large and small tenants."
+        else
+          "Current strategy :#{current} is optimal for #{total} active tenants."
+        end
+      end
+    end
+  end
+end

data/lib/fast_exists/multi_tenant/strategies/adaptive_strategy.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+module FastExists
+  module MultiTenant
+    module Strategies
+      class AdaptiveStrategy < BaseStrategy
+        attr_reader :pools, :per_tenant_strategy
+
+        def initialize(options = {})
+          super
+          @thresholds = options[:thresholds] || FastExists.configuration.tenant_thresholds
+          @pools = {
+            tiny: FastExists::MultiTenant::Pool.new(name: "tiny", expected_elements: 200_000),
+            small: FastExists::MultiTenant::Pool.new(name: "small", expected_elements: 1_000_000),
+            medium: FastExists::MultiTenant::Pool.new(name: "medium", expected_elements: 5_000_000)
+          }
+          @per_tenant_strategy = FastExists::MultiTenant::Strategies::PerTenantStrategy.new(options)
+          @tenant_classification_cache = {}
+          @mutex = Mutex.new
+        end
+
+        def add(tenant_id, attribute, value)
+          category = tenant_category(tenant_id)
+          if category == :large
+            @per_tenant_strategy.add(tenant_id, attribute, value)
+          else
+            pool = @pools[category] || @pools[:tiny]
+            pool.add(tenant_id, attribute, value)
+          end
+        end
+
+        def contains?(tenant_id, attribute, value)
+          category = tenant_category(tenant_id)
+          if category == :large
+            @per_tenant_strategy.contains?(tenant_id, attribute, value)
+          else
+            pool = @pools[category] || @pools[:tiny]
+            pool.contains?(tenant_id, attribute, value)
+          end
+        end
+
+        def classify_tenant(_tenant_id, record_count)
+          tiny_max = @thresholds[:tiny] || 10_000
+          small_max = @thresholds[:small] || 100_000
+          medium_max = @thresholds[:medium] || 1_000_000
+
+          case record_count
+          when 0...tiny_max then :tiny
+          when tiny_max...small_max then :small
+          when small_max...medium_max then :medium
+          else :large
+          end
+        end
+
+        def set_tenant_category(tenant_id, category)
+          @mutex.synchronize do
+            @tenant_classification_cache[tenant_id.to_s] = category.to_sym
+          end
+        end
+
+        def tenant_category(tenant_id)
+          @mutex.synchronize do
+            @tenant_classification_cache[tenant_id.to_s] || :tiny
+          end
+        end
+
+        def stats
+          @mutex.synchronize do
+            {
+              strategy: :adaptive,
+              classified_tenants: @tenant_classification_cache.size,
+              distribution: @tenant_classification_cache.values.tally,
+              pools: @pools.transform_values(&:stats),
+              dedicated_filters: @per_tenant_strategy.stats
+            }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/fast_exists/multi_tenant/strategies/base_strategy.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module FastExists
+  module MultiTenant
+    module Strategies
+      class BaseStrategy
+        attr_reader :options
+
+        def initialize(options = {})
+          @options = options
+        end
+
+        def add(tenant_id, attribute, value)
+          raise NotImplementedError, "#{self.class.name}#add is not implemented"
+        end
+
+        def contains?(tenant_id, attribute, value)
+          raise NotImplementedError, "#{self.class.name}#contains? is not implemented"
+        end
+
+        def classify_tenant(tenant_id, record_count)
+          :default
+        end
+
+        def stats
+          { strategy: self.class.name }
+        end
+      end
+    end
+  end
+end

data/lib/fast_exists/multi_tenant/strategies/global_strategy.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module FastExists
+  module MultiTenant
+    module Strategies
+      class GlobalStrategy < BaseStrategy
+        def initialize(options = {})
+          super
+          @backend = FastExists.backends.fetch(
+            FastExists.configuration.backend,
+            namespace: "global",
+            expected_elements: options[:expected_elements] || FastExists.configuration.expected_elements,
+            false_positive_rate: options[:false_positive_rate] || FastExists.configuration.false_positive_rate
+          )
+        end
+
+        def add(tenant_id, attribute, value)
+          composite_key = "#{tenant_id}:#{attribute}:#{value}"
+          @backend.add(composite_key)
+        end
+
+        def contains?(tenant_id, attribute, value)
+          composite_key = "#{tenant_id}:#{attribute}:#{value}"
+          @backend.contains?(composite_key)
+        end
+
+        def classify_tenant(_tenant_id, _record_count)
+          :global
+        end
+
+        def stats
+          @backend.stats.merge(strategy: :global, key: FastExists::MultiTenant::KeyLayout.global_key)
+        end
+      end
+    end
+  end
+end

data/lib/fast_exists/multi_tenant/strategies/per_tenant_strategy.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module FastExists
+  module MultiTenant
+    module Strategies
+      class PerTenantStrategy < BaseStrategy
+        def initialize(options = {})
+          super
+          @tenant_backends = {}
+          @mutex = Mutex.new
+        end
+
+        def add(tenant_id, attribute, value)
+          backend = tenant_backend_for(tenant_id)
+          composite_key = "#{attribute}:#{value}"
+          backend.add(composite_key)
+        end
+
+        def contains?(tenant_id, attribute, value)
+          backend = tenant_backend_for(tenant_id)
+          composite_key = "#{attribute}:#{value}"
+          backend.contains?(composite_key)
+        end
+
+        def classify_tenant(_tenant_id, _record_count)
+          :per_tenant
+        end
+
+        def stats
+          @mutex.synchronize do
+            {
+              strategy: :per_tenant,
+              active_tenant_filters: @tenant_backends.size,
+              tenant_keys: @tenant_backends.keys.map { |t| FastExists::MultiTenant::KeyLayout.tenant_key(t) }
+            }
+          end
+        end
+
+        private
+
+        def tenant_backend_for(tenant_id)
+          @mutex.synchronize do
+            @tenant_backends[tenant_id.to_s] ||= FastExists.backends.fetch(
+              FastExists.configuration.backend,
+              namespace: "tenant:#{tenant_id}",
+              expected_elements: options[:expected_elements] || FastExists.configuration.expected_elements,
+              false_positive_rate: options[:false_positive_rate] || FastExists.configuration.false_positive_rate
+            )
+          end
+        end
+      end
+    end
+  end
+end

data/lib/fast_exists/multi_tenant/strategies/shared_strategy.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module FastExists
+  module MultiTenant
+    module Strategies
+      class SharedStrategy < BaseStrategy
+        attr_reader :pools
+
+        def initialize(options = {})
+          super
+          @pools = {
+            small: FastExists::MultiTenant::Pool.new(name: "small", expected_elements: 500_000),
+            medium: FastExists::MultiTenant::Pool.new(name: "medium", expected_elements: 2_000_000),
+            large: FastExists::MultiTenant::Pool.new(name: "large", expected_elements: 10_000_000)
+          }
+        end
+
+        def add(tenant_id, attribute, value)
+          pool = pool_for(tenant_id)
+          pool.add(tenant_id, attribute, value)
+        end
+
+        def contains?(tenant_id, attribute, value)
+          pool = pool_for(tenant_id)
+          pool.contains?(tenant_id, attribute, value)
+        end
+
+        def classify_tenant(_tenant_id, record_count)
+          case record_count
+          when 0..100_000 then :small
+          when 100_001..1_000_000 then :medium
+          else :large
+          end
+        end
+
+        def stats
+          {
+            strategy: :shared,
+            pools: @pools.transform_values(&:stats)
+          }
+        end
+
+        private
+
+        def pool_for(tenant_id)
+          # Default to medium pool if unclassified
+          @pools[:medium]
+        end
+      end
+    end
+  end
+end

data/lib/fast_exists/optimizer/ai_advisor.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module FastExists
+  module Optimizer
+    class AiAdvisor
+      def self.analyze(model_class = nil, attribute = nil)
+        stats = FastExists.stats
+        db_count = model_class ? (model_class.count rescue 100_000) : 100_000
+
+        recommended_fp = stats[:false_positive_rate] > 0.01 ? 0.001 : 0.0005
+        recommended_capacity = (db_count * 1.5).ceil
+
+        # Optimal m calculation: m = - (n * ln(p)) / (ln(2)^2)
+        optimal_bits = (-(recommended_capacity * Math.log(recommended_fp)) / (Math.log(2)**2)).ceil
+        optimal_hashes = [((optimal_bits.to_f / recommended_capacity) * Math.log(2)).round, 1].max
+
+        recommended_backend = case db_count
+                              when 0..100_000 then :memory
+                              when 100_001..5_000_000 then :redis
+                              else :redis_bloom
+                              end
+
+        {
+          current_stats: stats,
+          recommendations: {
+            false_positive_rate: recommended_fp,
+            expected_elements: recommended_capacity,
+            optimal_bit_size: optimal_bits,
+            optimal_hash_count: optimal_hashes,
+            recommended_backend: recommended_backend,
+            estimated_memory_mb: (optimal_bits / 8.0 / 1_048_576.0).round(2),
+            advice: generate_advice(stats, db_count, recommended_backend)
+          }
+        }
+      end
+
+      private
+
+      def self.generate_advice(stats, db_count, backend)
+        adv = []
+        if stats[:false_positive_rate] > 0.01
+          adv << "False positive rate is high (#{stats[:false_positive_rate] * 100}%). Increase expected_elements or reduce false_positive_rate target."
+        end
+        if stats[:hit_ratio] > 0.8
+          adv << "High Bloom filter hit ratio (#{stats[:hit_ratio] * 100}%). Most lookups are positive. Check if existence queries are necessary before writing."
+        end
+        adv << "Recommended backend for dataset of #{db_count} records is :#{backend}."
+        adv.join(" ")
+      end
+    end
+  end
+end

data/lib/fast_exists/probabilistic/count_min_sketch.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+require "zlib"
+require "digest"
+
+module FastExists
+  module Probabilistic
+    class CountMinSketch
+      attr_reader :width, :depth
+
+      def initialize(epsilon: 0.001, confidence: 0.99)
+        @width = (Math::E / epsilon).ceil
+        @depth = Math.log(1.0 / (1.0 - confidence)).ceil
+        @table = Array.new(@depth) { Array.new(@width, 0) }
+        @mutex = Mutex.new
+      end
+
+      def add(element, count = 1)
+        key = element.to_s
+        indexes = hash_indexes(key)
+
+        @mutex.synchronize do
+          indexes.each_with_index do |col, row|
+            @table[row][col] += count
+          end
+        end
+        true
+      end
+
+      def estimate(element)
+        key = element.to_s
+        indexes = hash_indexes(key)
+
+        @mutex.synchronize do
+          indexes.each_with_index.map { |col, row| @table[row][col] }.min
+        end
+      end
+
+      def clear
+        @mutex.synchronize do
+          @table.each { |row| row.fill(0) }
+        end
+        true
+      end
+
+      def stats
+        {
+          type: :count_min_sketch,
+          width: @width,
+          depth: @depth,
+          memory_cells: @width * @depth
+        }
+      end
+
+      private
+
+      def hash_indexes(key)
+        h1 = Zlib.crc32(key)
+        h2 = Digest::SHA256.hexdigest(key)[0..7].hex
+
+        Array.new(@depth) do |i|
+          (h1 + i * h2) % @width
+        end
+      end
+    end
+  end
+end

data/lib/fast_exists/probabilistic/cuckoo.rb

@@ -0,0 +1,128 @@
+# frozen_string_literal: true
+
+require "zlib"
+require "digest"
+
+module FastExists
+  module Probabilistic
+    class Cuckoo
+      attr_reader :capacity, :bucket_size, :fingerprint_size, :count
+
+      def initialize(capacity: 10_000, bucket_size: 4, max_kicks: 500)
+        @capacity = capacity
+        @bucket_size = bucket_size
+        @max_kicks = max_kicks
+        @num_buckets = (capacity.to_f / bucket_size).ceil
+        @num_buckets = 1 if @num_buckets < 1
+        @buckets = Array.new(@num_buckets) { [] }
+        @count = 0
+        @mutex = Mutex.new
+      end
+
+      def add(element)
+        fp = fingerprint(element.to_s)
+        i1 = hash_index(element.to_s)
+        i2 = alt_index(i1, fp)
+
+        @mutex.synchronize do
+          if @buckets[i1].size < @bucket_size
+            @buckets[i1] << fp
+            @count += 1
+            return true
+          end
+
+          if @buckets[i2].size < @bucket_size
+            @buckets[i2] << fp
+            @count += 1
+            return true
+          end
+
+          # Kicking eviction loop
+          curr_idx = rand(2) == 0 ? i1 : i2
+          curr_fp = fp
+
+          @max_kicks.times do
+            evicted_fp = @buckets[curr_idx][rand(@buckets[curr_idx].size)]
+            idx_in_bucket = @buckets[curr_idx].index(evicted_fp)
+            @buckets[curr_idx][idx_in_bucket] = curr_fp
+
+            curr_fp = evicted_fp
+            curr_idx = alt_index(curr_idx, curr_fp)
+
+            if @buckets[curr_idx].size < @bucket_size
+              @buckets[curr_idx] << curr_fp
+              @count += 1
+              return true
+            end
+          end
+
+          raise CapacityExceededError, "Cuckoo filter is full"
+        end
+      end
+
+      def contains?(element)
+        fp = fingerprint(element.to_s)
+        i1 = hash_index(element.to_s)
+        i2 = alt_index(i1, fp)
+
+        @mutex.synchronize do
+          @buckets[i1].include?(fp) || @buckets[i2].include?(fp)
+        end
+      end
+
+      def delete(element)
+        fp = fingerprint(element.to_s)
+        i1 = hash_index(element.to_s)
+        i2 = alt_index(i1, fp)
+
+        @mutex.synchronize do
+          if (idx = @buckets[i1].index(fp))
+            @buckets[i1].delete_at(idx)
+            @count -= 1
+            return true
+          elsif (idx = @buckets[i2].index(fp))
+            @buckets[i2].delete_at(idx)
+            @count -= 1
+            return true
+          end
+        end
+        false
+      end
+
+      def clear
+        @mutex.synchronize do
+          @buckets.each(&:clear)
+          @count = 0
+        end
+        true
+      end
+
+      def stats
+        @mutex.synchronize do
+          {
+            type: :cuckoo,
+            inserted_items: @count,
+            capacity: @capacity,
+            buckets: @num_buckets,
+            bucket_size: @bucket_size
+          }
+        end
+      end
+
+      private
+
+      def fingerprint(key)
+        Digest::MD5.digest(key)[0..1]
+      end
+
+      def hash_index(key)
+        Zlib.crc32(key) % @num_buckets
+      end
+
+      def alt_index(index, fp)
+        fp_hash = Zlib.crc32(fp)
+        (index ^ fp_hash) % @num_buckets
+      end
+    end
+  end
+end

data/lib/fast_exists/probabilistic/hyper_log_log.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+require "zlib"
+require "digest"
+
+module FastExists
+  module Probabilistic
+    class HyperLogLog
+      attr_reader :p, :m
+
+      def initialize(p: 14)
+        raise InvalidArgumentError, "p must be between 4 and 16" if p < 4 || p > 16
+
+        @p = p
+        @m = 1 << p
+        @registers = Array.new(@m, 0)
+        @alpha = calculate_alpha(@m)
+        @mutex = Mutex.new
+      end
+
+      def add(element)
+        hash = Digest::SHA256.hexdigest(element.to_s)[0..15].hex
+        idx = hash >> (64 - @p)
+        w = hash & ((1 << (64 - @p)) - 1)
+        rho = leading_zeros(w, 64 - @p) + 1
+
+        @mutex.synchronize do
+          @registers[idx] = [@registers[idx], rho].max
+        end
+        true
+      end
+
+      def count
+        @mutex.synchronize do
+          raw_estimate = @alpha * (@m**2) * (1.0 / @registers.sum { |r| 2.0**(-r) })
+
+          if raw_estimate <= 2.5 * @m
+            zeros = @registers.count(0)
+            zeros > 0 ? @m * Math.log(@m.to_f / zeros) : raw_estimate
+          else
+            raw_estimate
+          end.round
+        end
+      end
+
+      def clear
+        @mutex.synchronize do
+          @registers.fill(0)
+        end
+        true
+      end
+
+      def stats
+        {
+          type: :hyper_log_log,
+          registers: @m,
+          precision_p: @p,
+          estimated_cardinality: count
+        }
+      end
+
+      private
+
+      def leading_zeros(val, bit_len)
+        return bit_len if val == 0
+        clz = 0
+        (bit_len - 1).downto(0) do |bit|
+          break if (val & (1 << bit)) != 0
+          clz += 1
+        end
+        clz
+      end
+
+      def calculate_alpha(m)
+        case m
+        when 16 then 0.673
+        when 32 then 0.697
+        when 64 then 0.709
+        else 0.7213 / (1.0 + 1.079 / m)
+        end
+      end
+    end
+  end
+end

data/lib/fast_exists/railtie.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+if defined?(Rails::Railtie)
+  module FastExists
+    class Railtie < Rails::Railtie
+      initializer "fast_exists.active_record" do
+        ActiveSupport.on_load(:active_record) do
+          include FastExists::ActiveRecord::Extension
+        end
+      end
+
+      rake_tasks do
+        load File.expand_path("tasks/fast_exists.rake", __dir__)
+      end
+    end
+  end
+end