splitclient-rb 7.3.4.pre.rc2-java → 7.3.5.pre.rc3-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7ea2b0463e8a3c2f13075209d9e0cb8b189696b3
-  data.tar.gz: 601571ffb3a015fbf8dcb468577b05caced467c7
+  metadata.gz: 186060ea5e282d91426d8c220d060fe34cf8a316
+  data.tar.gz: 2217aae3d87f3dd8ba0a09128ba5d7a064fb1fae
 SHA512:
-  metadata.gz: c1eaa3106070a63477fa3951ff55a0c24c4287d379384da95852fb34ddc2a20d61557d3282e40f4c56d14f915f6bd615223b8830a8bd0d2b9afd36d32331f83e
-  data.tar.gz: bde0cdac66e0bdeb0e5e02fc8e77ef8c834c56a767fdf197c398add674e1dbf1e19c683feda9384ab5ce2f80571d0c633e50e8417ab2cd3c0dfab8b7005b1cb3
+  metadata.gz: e3e69c550554845b3b27ae2132a86c25e2d2f0b49c001baacdea33feec117f8ce5778ff004ccfe44388841b384d9a086db43eeb89997e2159503361adda3525b
+  data.tar.gz: fdc0ec71a389b8f48f9d941cca7bf2571d3e91cd85ee0f1ac03413e53ac8ea4367eb5d8d00b414c5649262f88cba4be8e70726d38d67cbf2f0a8c813ee830fa6

data/.github/workflows/update-license-year.yml

@@ -0,0 +1,45 @@
+name: Update License Year
+
+on:
+  schedule:
+    - cron: "0 3 1 1 *" # 03:00 AM on January 1
+
+permissions:
+  contents: write
+  pull-requests: write
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      
+      - name: Set Current year
+        run: "echo CURRENT=$(date +%Y) >> $GITHUB_ENV"
+      
+      - name: Set Previous Year
+        run: "echo PREVIOUS=$(($CURRENT-1)) >> $GITHUB_ENV"
+
+      - name: Update LICENSE
+        uses: jacobtomlinson/gha-find-replace@v2
+        with:
+          find: ${{ env.PREVIOUS }}
+          replace: ${{ env.CURRENT }}
+          include: "LICENSE"
+          regex: false
+
+      - name: Commit files
+        run: |
+          git config user.name 'github-actions[bot]'
+          git config user.email 'github-actions[bot]@users.noreply.github.com'
+          git commit -m "Updated License Year" -a
+
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@v3
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          title: Update License Year
+          branch: update-license

data/.rubocop.yml

@@ -26,7 +26,7 @@ Metrics/ParameterLists:
     - lib/splitclient-rb/engine/sync_manager.rb
 
 Metrics/LineLength:
-  Max: 130
+  Max: 135
   Exclude:
     - spec/sse/**/*
     - spec/integrations/**/*
@@ -35,6 +35,7 @@ Metrics/LineLength:
     - spec/telemetry/synchronizer_spec.rb
     - spec/splitclient/split_config_spec.rb
     - spec/engine/push_manager_spec.rb
+    - spec/cache/senders/impressions_sender_adapter_spec.rb
 
 Style/BracesAroundHashParameters:
   Exclude:

data/CHANGES.txt

@@ -1,6 +1,10 @@
 CHANGES
 
-7.3.3 (Jan 28, 2021)
+7.3.4 (Feb 21, 2022)
+- Updated streaming events architecture with a new queue logic.
+- Fixed redis integration Pipelining command deprecation warning.
+
+7.3.3 (Jan 28, 2022)
 - Fixed edge cases where the sdk lost streaming connection.
 - Updated default auth service url to https://auth.split.io/api/v2/auth
 - Updated dependencies:

data/LICENSE

@@ -1,4 +1,4 @@
-Copyright © 2021 Split Software, Inc.
+Copyright © 2022 Split Software, Inc.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

data/lib/splitclient-rb/cache/filter/bloom_filter.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+require 'bitarray'
+
+module SplitIoClient
+  module Cache
+    module Filter
+      class BloomFilter
+        def initialize(capacity, false_positive_probability = 0.001)
+          @capacity = capacity.round
+          m = best_m(capacity, false_positive_probability)
+          @ba = BitArray.new(m.round)
+          @k = best_k(capacity)
+        end
+
+        def add(string)
+          return false if contains?(string)
+
+          positions = hashes(string)
+          
+          positions.each { |position| @ba[position] = 1 }
+
+          true
+        end
+      
+        def contains?(string)
+          !hashes(string).any? { |ea| @ba[ea] == 0 }
+        end
+
+        def clear
+          @ba.size.times { |i| @ba[i] = 0 }
+        end
+      
+        private
+
+        # m is the required number of bits in the array
+        def best_m(capacity, false_positive_probability)
+          -(capacity * Math.log(false_positive_probability)) / (Math.log(2) ** 2)
+        end
+
+        # k is the number of hash functions that minimizes the probability of false positives
+        def best_k(capacity)
+          (Math.log(2) * (@ba.size / capacity)).round
+        end
+
+        def hashes(data)
+          m = @ba.size
+          h = Digest::MD5.hexdigest(data.to_s).to_i(16)
+          x = h % m
+          h /= m
+          y = h % m
+          h /= m
+          z = h % m
+          [x] + 1.upto(@k - 1).collect do |i|
+            x = (x + y) % m
+            y = (y + z) % m
+            x
+          end
+        end
+      end
+    end
+  end
+end

data/lib/splitclient-rb/cache/filter/filter_adapter.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module SplitIoClient
+  module Cache
+    module Filter
+      class FilterAdapter
+        def initialize(config, filter)
+          @config = config
+          @filter = filter
+        end
+
+        def add(feature_name, key)
+          @filter.add("#{feature_name}#{key}")
+        rescue StandardError => e
+          @config.log_found_exception(__method__.to_s, e)
+        end
+
+        def contains?(feature_name, key)
+          @filter.contains?("#{feature_name}#{key}")
+        rescue StandardError => e
+          @config.log_found_exception(__method__.to_s, e)
+        end
+
+        def clear
+          @filter.clear
+        rescue StandardError => e
+          @config.log_found_exception(__method__.to_s, e)
+        end
+      end
+    end
+  end
+end

data/lib/splitclient-rb/cache/observers/noop_impression_observer.rb

@@ -0,0 +1,10 @@
+module SplitIoClient
+  module Observers
+    class NoopImpressionObserver
+      def test_and_set(impression)
+        # no-op
+      end
+    end
+  end
+end
+  

data/lib/splitclient-rb/cache/senders/impressions_adapter/memory_sender.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module SplitIoClient
+  module Cache
+    module Senders
+      class MemoryImpressionsSender < ImpressionsSenderAdapter
+        def initialize(config, telemetry_api, impressions_api)
+          @config = config
+          @telemetry_api = telemetry_api
+          @impressions_api = impressions_api
+        end
+
+        def record_uniques_key(uniques)
+          uniques_keys = uniques_formatter(uniques)
+
+          @telemetry_api.record_unique_keys(uniques_keys) unless uniques_keys.nil?
+        rescue StandardError => e
+          @config.log_found_exception(__method__.to_s, e)
+        end
+
+        def record_impressions_count(impressions_count)
+          counts = impressions_count_formatter(impressions_count)
+
+          @impressions_api.post_count(counts) unless counts.nil?
+        rescue StandardError => e
+          @config.log_found_exception(__method__.to_s, e)
+        end
+
+        private 
+
+        def uniques_formatter(uniques)
+          return if uniques.nil? || uniques.empty?
+
+          to_return = { mtks: [] }
+          uniques.each do |key, value|
+            to_return[:mtks] << {
+              f: key,
+              ks: value.to_a
+            }
+          end
+
+          to_return
+        rescue StandardError => error
+          @config.log_found_exception(__method__.to_s, error)
+          nil
+        end
+
+        def impressions_count_formatter(counts)
+          return if counts.nil? || counts.empty?
+
+          formated_counts = {pf: []}
+
+          counts.each do |key, value|              
+            key_splited = key.split('::')
+            
+            formated_counts[:pf] << {
+              f: key_splited[0].to_s, # feature name
+              m: key_splited[1].to_i, # time frame
+              rc: value # count
+            }
+          end
+
+          formated_counts
+        rescue StandardError => error
+          @config.log_found_exception(__method__.to_s, error)
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/splitclient-rb/cache/senders/impressions_adapter/redis_sender.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+module SplitIoClient
+  module Cache
+    module Senders
+      class RedisImpressionsSender < ImpressionsSenderAdapter
+        EXPIRE_SECONDS = 3600
+
+        def initialize(config)
+          @config = config
+          @adapter = @config.impressions_adapter
+        end
+
+        def record_uniques_key(uniques)
+          formatted = uniques_formatter(uniques)
+
+          unless formatted.nil?
+            size = @adapter.add_to_queue(unique_keys_key, formatted)
+            @adapter.expire(unique_keys_key, EXPIRE_SECONDS) if formatted.size == size
+          end
+        rescue StandardError => e
+          @config.log_found_exception(__method__.to_s, e)
+        end
+
+        def record_impressions_count(impressions_count)
+          return if impressions_count.nil? || impressions_count.empty?
+
+          result = @adapter.redis.pipelined do |pipeline|
+            impressions_count.each do |key, value|
+              pipeline.hincrby(impressions_count_key, key, value)
+            end
+            
+            @future = pipeline.hlen(impressions_count_key)
+          end
+
+          expire_impressions_count_key(impressions_count, result)
+        rescue StandardError => e
+          @config.log_found_exception(__method__.to_s, e)
+        end
+
+        private
+
+        def expire_impressions_count_key(impressions_count, pipeline_result)
+          total_count = impressions_count.sum { |_, value| value }
+          hlen = pipeline_result.last
+
+          @adapter.expire(impressions_count_key, EXPIRE_SECONDS) if impressions_count.size == hlen && (pipeline_result.sum - hlen) == total_count
+        end
+
+        def impressions_count_key
+          "#{@config.redis_namespace}.impressions.count"
+        end
+
+        def unique_keys_key
+          "#{@config.redis_namespace}.uniquekeys"
+        end
+
+        def uniques_formatter(uniques)
+          return if uniques.nil? || uniques.empty?
+
+          to_return = []
+          uniques.each do |key, value|
+            to_return << {
+              f: key,
+              k: value.to_a
+            }
+          end
+
+          to_return
+        rescue StandardError => error
+          @config.log_found_exception(__method__.to_s, error)
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/splitclient-rb/cache/senders/impressions_count_sender.rb

@@ -4,12 +4,10 @@ module SplitIoClient
   module Cache
     module Senders
       class ImpressionsCountSender
-        COUNTER_REFRESH_RATE_SECONDS = 1800
-
-        def initialize(config, impression_counter, impressions_api)
+        def initialize(config, impression_counter, impressions_sender_adapter)
           @config = config
           @impression_counter = impression_counter
-          @impressions_api = impressions_api
+          @impressions_sender_adapter = impressions_sender_adapter
         end
 
         def call
@@ -22,13 +20,11 @@ module SplitIoClient
           @config.threads[:impressions_count_sender] = Thread.new do
             begin
               @config.logger.info('Starting impressions count service')
-
               loop do
-                post_impressions_count
-
-                sleep(COUNTER_REFRESH_RATE_SECONDS)
+                sleep(@config.counter_refresh_rate)
+                post_impressions_count                
               end
-            rescue SplitIoClient::SDKShutdownException
+            rescue SplitIoClient::SDKShutdownException              
               post_impressions_count
 
               @config.logger.info('Posting impressions count due to shutdown')
@@ -37,27 +33,7 @@ module SplitIoClient
         end
 
         def post_impressions_count
-          @impressions_api.post_count(formatter(@impression_counter.pop_all))
-        rescue StandardError => error
-          @config.log_found_exception(__method__.to_s, error)
-        end
-
-        def formatter(counts)
-          return if counts.empty?
-
-          formated_counts = {pf: []}
-
-          counts.each do |key, value|              
-            key_splited = key.split('::')
-            
-            formated_counts[:pf] << {
-              f: key_splited[0].to_s, # feature name
-              m: key_splited[1].to_i, # time frame
-              rc: value # count
-            }
-          end
-
-          formated_counts
+          @impressions_sender_adapter.record_impressions_count(@impression_counter.pop_all)
         rescue StandardError => error
           @config.log_found_exception(__method__.to_s, error)
         end

data/lib/splitclient-rb/cache/senders/impressions_sender_adapter.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module SplitIoClient
+  module Cache
+    module Senders
+      class ImpressionsSenderAdapter
+        extend Forwardable
+        def_delegators :@sender, :record_uniques_key, :record_impressions_count
+
+        def initialize(config, telemetry_api, impressions_api)
+          @sender = case config.telemetry_adapter.class.to_s
+                    when 'SplitIoClient::Cache::Adapters::RedisAdapter'
+                      Cache::Senders::RedisImpressionsSender.new(config)
+                    else
+                      Cache::Senders::MemoryImpressionsSender.new(config, telemetry_api, impressions_api)
+                    end
+        end
+      end
+    end
+  end
+end

data/lib/splitclient-rb/engine/api/telemetry_api.rb

@@ -17,6 +17,14 @@ module SplitIoClient
         post_telemetry("#{@config.telemetry_service_url}/metrics/usage", stats, 'stats')
       end
 
+      def record_unique_keys(uniques)
+        return if uniques[:mtks].empty?
+
+        post_telemetry("#{@config.telemetry_service_url}/mtks/ss", uniques, 'mtks')
+      rescue StandardError => e
+        @config.log_found_exception(__method__.to_s, e)
+      end
+
       private
 
       def post_telemetry(url, obj, method)

data/lib/splitclient-rb/engine/common/impressions_manager.rb

@@ -4,65 +4,75 @@ module SplitIoClient
   module Engine
     module Common
       class ImpressionManager
-        def initialize(config, impressions_repository, impression_counter, telemetry_runtime_producer)
+        def initialize(config,
+                       impressions_repository,
+                       impression_counter,
+                       telemetry_runtime_producer,
+                       impression_observer,
+                       unique_keys_tracker,
+                       impression_router)
           @config = config
           @impressions_repository = impressions_repository
           @impression_counter = impression_counter
-          @impression_observer = SplitIoClient::Observers::ImpressionObserver.new
+          @impression_observer = impression_observer
           @telemetry_runtime_producer = telemetry_runtime_producer
+          @unique_keys_tracker = unique_keys_tracker
+          @impression_router = impression_router
         end
 
-        # added param time for test
         def build_impression(matching_key, bucketing_key, split_name, treatment, params = {})
           impression_data = impression_data(matching_key, bucketing_key, split_name, treatment, params[:time])
 
-          impression_data[:pt] = @impression_observer.test_and_set(impression_data) unless redis?
-
-          @impression_counter.inc(split_name, impression_data[:m]) if optimized? && !redis?
+          begin
+            case @config.impressions_mode
+            when :debug #  In DEBUG mode we should calculate the pt only.
+              impression_data[:pt] = @impression_observer.test_and_set(impression_data)
+            when :none # In NONE mode we should track the total amount of evaluations and the unique keys.
+              @impression_counter.inc(split_name, impression_data[:m])
+              @unique_keys_tracker.track(split_name, matching_key)
+            else # In OPTIMIZED mode we should track the total amount of evaluations and deduplicate the impressions.
+              impression_data[:pt] = @impression_observer.test_and_set(impression_data)
+              @impression_counter.inc(split_name, impression_data[:m])
+            end
+          rescue StandardError => e
+            @config.log_found_exception(__method__.to_s, e)
+          end
 
           impression(impression_data, params[:attributes])
-        rescue StandardError => e
-          @config.log_found_exception(__method__.to_s, e)
         end
 
         def track(impressions)
           return if impressions.empty?
 
-          impression_router.add_bulk(impressions)
-
-          dropped = 0
-          queued = 0
-          dedupe = 0
-
-          if optimized? && !redis?
-            optimized_impressions = impressions.select { |imp| should_queue_impression?(imp[:i]) }
-
-            unless optimized_impressions.empty?
-              dropped = @impressions_repository.add_bulk(optimized_impressions)
-              dedupe = impressions.length - optimized_impressions.length
-              queued = optimized_impressions.length - dropped
+          stats = { dropped: 0, queued: 0, dedupe: 0 }
+          begin
+            case @config.impressions_mode
+            when :none
+              return
+            when :debug
+              track_debug_mode(impressions, stats)
+            when :optimized
+              track_optimized_mode(impressions, stats)
             end
-          else
-            dropped = @impressions_repository.add_bulk(impressions)
-            queued = impressions.length - dropped
+          rescue StandardError => e
+            @config.log_found_exception(__method__.to_s, e)
+          ensure
+            record_stats(stats)
+            @impression_router.add_bulk(impressions)
           end
-
-          record_stats(queued, dropped, dedupe)
-        rescue StandardError => e
-          @config.log_found_exception(__method__.to_s, e)
         end
 
         private
 
-        def record_stats(queued, dropped, dedupe)
+        def record_stats(stats)
           return if redis?
 
           imp_queued = Telemetry::Domain::Constants::IMPRESSIONS_QUEUED
           imp_dropped = Telemetry::Domain::Constants::IMPRESSIONS_DROPPED
           imp_dedupe = Telemetry::Domain::Constants::IMPRESSIONS_DEDUPE
-          @telemetry_runtime_producer.record_impressions_stats(imp_queued, queued) unless queued.zero?
-          @telemetry_runtime_producer.record_impressions_stats(imp_dropped, dropped) unless dropped.zero?
-          @telemetry_runtime_producer.record_impressions_stats(imp_dedupe, dedupe) unless dedupe.zero?
+          @telemetry_runtime_producer.record_impressions_stats(imp_queued, stats[:queued]) unless stats[:queued].zero?
+          @telemetry_runtime_producer.record_impressions_stats(imp_dropped, stats[:dropped]) unless stats[:dropped].zero?
+          @telemetry_runtime_producer.record_impressions_stats(imp_dedupe, stats[:dedupe]) unless stats[:dedupe].zero?
         end
 
         # added param time for test
@@ -91,10 +101,6 @@ module SplitIoClient
           @config.labels_enabled ? label : nil
         end
 
-        def optimized?
-          @config.impressions_mode == :optimized
-        end
-
         def should_queue_impression?(impression)
           impression[:pt].nil? ||
             (ImpressionCounter.truncate_time_frame(impression[:pt]) != ImpressionCounter.truncate_time_frame(impression[:m]))
@@ -108,10 +114,19 @@ module SplitIoClient
           @config.impressions_adapter.class.to_s == 'SplitIoClient::Cache::Adapters::RedisAdapter'
         end
 
-        def impression_router
-          @impression_router ||= SplitIoClient::ImpressionRouter.new(@config)
-        rescue StandardError => error
-          @config.log_found_exception(__method__.to_s, error)
+        def track_debug_mode(impressions, stats)
+          stats[:dropped] = @impressions_repository.add_bulk(impressions)
+          stats[:queued] = impressions.length - stats[:dropped]
+        end
+
+        def track_optimized_mode(impressions, stats)
+          optimized_impressions = impressions.select { |imp| should_queue_impression?(imp[:i]) }
+
+          return if optimized_impressions.empty?
+
+          stats[:dropped] = @impressions_repository.add_bulk(optimized_impressions)
+          stats[:dedupe] = impressions.length - optimized_impressions.length
+          stats[:queued] = optimized_impressions.length - stats[:dropped]
         end
       end
     end

data/lib/splitclient-rb/engine/common/noop_impressions_counter.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require 'concurrent'
+
+module SplitIoClient
+  module Engine
+    module Common
+      class NoopImpressionCounter
+        def inc(split_name, time_frame)
+          # no-op
+        end
+
+        def pop_all
+          # no-op
+        end
+
+        def make_key(split_name, time_frame)
+          # no-op
+        end
+
+        def self.truncate_time_frame(timestamp_ms)
+          # no-op
+        end
+      end
+    end
+  end
+end

data/lib/splitclient-rb/engine/impressions/noop_unique_keys_tracker.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module SplitIoClient
+  module Engine
+    module Impressions
+      class NoopUniqueKeysTracker
+        def call
+          # no-op
+        end
+
+        def track(feature_name, key)
+          # no-op
+        end
+      end
+    end
+  end
+end

data/lib/splitclient-rb/engine/impressions/unique_keys_tracker.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+module SplitIoClient
+  module Engine
+    module Impressions
+      class UniqueKeysTracker
+        INTERVAL_TO_CLEAR_LONG_TERM_CACHE = 86_400 # 24 hours
+
+        def initialize(config,
+                       filter_adapter,
+                       sender_adapter,
+                       cache)
+          @config = config
+          @filter_adapter = filter_adapter
+          @sender_adapter = sender_adapter
+          @cache = cache
+          @cache_max_size = config.unique_keys_cache_max_size
+          @max_bulk_size = config.unique_keys_bulk_size
+          @semaphore = Mutex.new
+        end
+
+        def call
+          @config.threads[:unique_keys_sender] = Thread.new { send_bulk_data_thread }
+          @config.threads[:clear_filter] = Thread.new { clear_filter_thread }
+        end
+
+        def track(feature_name, key)
+          return false if @filter_adapter.contains?(feature_name, key)
+
+          @filter_adapter.add(feature_name, key)
+
+          add_or_update(feature_name, key)
+
+          send_bulk_data if @cache.size >= @cache_max_size
+
+          true
+        rescue StandardError => e
+          @config.log_found_exception(__method__.to_s, e)
+          false
+        end
+
+        private
+
+        def send_bulk_data_thread
+          @config.logger.info('Starting Unique Keys Tracker.') if @config.debug_enabled
+          loop do
+            sleep(@config.unique_keys_refresh_rate)
+            send_bulk_data
+          end
+        rescue SplitIoClient::SDKShutdownException
+          send_bulk_data
+          @config.logger.info('Posting unique keys due to shutdown')
+        end
+
+        def clear_filter_thread
+          loop do
+            sleep(INTERVAL_TO_CLEAR_LONG_TERM_CACHE)
+            @config.logger.debug('Starting task to clean the filter cache.') if @config.debug_enabled
+            @filter_adapter.clear
+          end
+        rescue SplitIoClient::SDKShutdownException
+          @filter_adapter.clear
+        end
+
+        def add_or_update(feature_name, key)
+          if @cache[feature_name].nil?
+            @cache[feature_name] = Set.new([key])
+          else
+            @cache[feature_name].add(key)
+          end
+        end
+
+        def send_bulk_data
+          @semaphore.synchronize do
+            return if @cache.empty?
+
+            uniques = @cache.clone
+            @cache.clear
+
+            if uniques.size <= @max_bulk_size
+              @sender_adapter.record_uniques_key(uniques)
+              return
+            end
+
+            bulks = SplitIoClient::Utilities.split_bulk_to_send(uniques, uniques.size / @max_bulk_size)
+
+            bulks.each do |b|
+              @sender_adapter.record_uniques_key(b)
+            end
+          end
+        rescue StandardError => e
+          @config.log_found_exception(__method__.to_s, e)
+        end
+      end
+    end
+  end
+end

data/lib/splitclient-rb/engine/matchers/combining_matcher.rb

@@ -52,9 +52,8 @@ module SplitIoClient
     # @return [boolean] match value for combiner delegates
     def eval_and(args)
       # Convert all keys to symbols
-      if args && args[:attributes]
-        args[:attributes] = args[:attributes].each_with_object({}) { |(k, v), memo| memo[k.to_sym] = v }
-      end
+      args[:attributes] = args[:attributes].each_with_object({}) { |(k, v), memo| memo[k.to_sym] = v } if args && args[:attributes]
+
       @matchers.all? do |matcher|
         if match_with_key?(matcher)
           matcher.match?(value: args[:matching_key])

data/lib/splitclient-rb/engine/sync_manager.rb

@@ -30,6 +30,11 @@ module SplitIoClient
         PhusionPassenger.on_event(:starting_worker_process) { |forked| start_thread if forked } if defined?(PhusionPassenger)
       end
 
+      def start_consumer
+        start_consumer_thread
+        PhusionPassenger.on_event(:starting_worker_process) { |forked| start_consumer_thread if forked } if defined?(PhusionPassenger)
+      end
+
       private
 
       def start_thread
@@ -55,6 +60,14 @@ module SplitIoClient
         end
       end
 
+      def start_consumer_thread
+        @config.threads[:start_sdk_consumer] = Thread.new do
+          @status_manager.ready!
+          @telemetry_synchronizer.synchronize_config
+          @synchronizer.start_periodic_data_recording
+        end
+      end
+
       def process_subsystem_ready
         @synchronizer.stop_periodic_fetch
         @synchronizer.sync_all

data/lib/splitclient-rb/engine/synchronizer.rb

@@ -12,7 +12,6 @@ module SplitIoClient
 
       def initialize(
         repositories,
-        api_key,
         config,
         params
       )
@@ -20,13 +19,14 @@ module SplitIoClient
         @segments_repository = repositories[:segments]
         @impressions_repository = repositories[:impressions]
         @events_repository = repositories[:events]
-        @api_key = api_key
         @config = config
         @split_fetcher = params[:split_fetcher]
         @segment_fetcher = params[:segment_fetcher]
-        @impressions_api = SplitIoClient::Api::Impressions.new(@api_key, @config, params[:telemetry_runtime_producer])
+        @impressions_api = params[:impressions_api]
         @impression_counter = params[:imp_counter]
         @telemetry_synchronizer = params[:telemetry_synchronizer]
+        @impressions_sender_adapter = params[:impressions_sender_adapter]
+        @unique_keys_tracker = params[:unique_keys_tracker]
       end
 
       def sync_all(asynchronous = true)
@@ -42,10 +42,14 @@ module SplitIoClient
       end
 
       def start_periodic_data_recording
-        impressions_sender
-        events_sender
+        unless @config.consumer?
+          impressions_sender
+          events_sender
+          start_telemetry_sync_task
+        end
+        
         impressions_count_sender
-        start_telemetry_sync_task
+        start_unique_keys_tracker_task
       end
 
       def start_periodic_fetch
@@ -170,7 +174,7 @@ module SplitIoClient
 
       # Starts thread which loops constantly and sends impressions to the Split API
       def impressions_sender
-        ImpressionsSender.new(@impressions_repository, @config, @impressions_api).call
+        ImpressionsSender.new(@impressions_repository, @config, @impressions_api).call unless @config.impressions_mode == :none
       end
 
       # Starts thread which loops constantly and sends events to the Split API
@@ -180,13 +184,17 @@ module SplitIoClient
 
       # Starts thread which loops constantly and sends impressions count to the Split API
       def impressions_count_sender
-        ImpressionsCountSender.new(@config, @impression_counter, @impressions_api).call
+        ImpressionsCountSender.new(@config, @impression_counter, @impressions_sender_adapter).call unless @config.impressions_mode == :debug
       end
 
       def start_telemetry_sync_task
         Telemetry::SyncTask.new(@config, @telemetry_synchronizer).call
       end
 
+      def start_unique_keys_tracker_task
+        @unique_keys_tracker.call
+      end
+
       def sync_result(success, remaining_attempts, segment_names = nil)
         { success: success, remaining_attempts: remaining_attempts, segment_names: segment_names }
       end

data/lib/splitclient-rb/split_config.rb

@@ -111,6 +111,12 @@ module SplitIoClient
       @telemetry_refresh_rate = SplitConfig.init_telemetry_refresh_rate(opts[:telemetry_refresh_rate])
       @telemetry_service_url = opts[:telemetry_service_url] || SplitConfig.default_telemetry_service_url
 
+      @unique_keys_refresh_rate = SplitConfig.default_unique_keys_refresh_rate(@cache_adapter)
+      @unique_keys_cache_max_size = SplitConfig.default_unique_keys_cache_max_size
+      @unique_keys_bulk_size = SplitConfig.default_unique_keys_bulk_size(@cache_adapter)
+
+      @counter_refresh_rate = SplitConfig.default_counter_refresh_rate(@cache_adapter)
+
       @sdk_start_time = Time.now
 
       @on_demand_fetch_retry_delay_seconds = SplitConfig.default_on_demand_fetch_retry_delay_seconds
@@ -284,6 +290,18 @@ module SplitIoClient
     attr_accessor :on_demand_fetch_retry_delay_seconds    
     attr_accessor :on_demand_fetch_max_retries
 
+    attr_accessor :unique_keys_refresh_rate
+    attr_accessor :unique_keys_cache_max_size
+    attr_accessor :unique_keys_bulk_size
+
+    attr_accessor :counter_refresh_rate
+
+    def self.default_counter_refresh_rate(adapter)
+      return 300 if adapter == :redis # Send bulk impressions count - Refresh rate: 5 min.
+
+      1800 # Send bulk impressions count - Refresh rate: 30 min.
+    end
+
     def self.default_on_demand_fetch_retry_delay_seconds
       0.05
     end
@@ -302,6 +320,8 @@ module SplitIoClient
       case impressions_mode
       when :debug
         return :debug
+      when :none
+        return :none
       else
         @logger.error('You passed an invalid impressions_mode, impressions_mode should be one of the following values: :debug or :optimized. Defaulting to :optimized mode') unless impressions_mode == :optimized
         return :optimized
@@ -468,6 +488,22 @@ module SplitIoClient
       3600      
     end
 
+    def self.default_unique_keys_refresh_rate(adapter)
+      return 300 if adapter == :redis
+
+      900
+    end
+
+    def self.default_unique_keys_cache_max_size
+      30000
+    end
+
+    def self.default_unique_keys_bulk_size(adapter)
+      return 2000 if adapter == :redis
+
+      5000
+    end
+
     def self.default_telemetry_service_url
       'https://telemetry.split.io/api/v1'
     end

data/lib/splitclient-rb/split_factory.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module SplitIoClient
   class SplitFactory
     ROOT_PROCESS_ID = Process.pid
@@ -34,8 +36,10 @@ module SplitIoClient
 
       build_telemetry_components
       build_repositories
-      build_impressions_components
       build_telemetry_synchronizer
+      build_impressions_sender_adapter
+      build_unique_keys_tracker
+      build_impressions_components
 
       @status_manager = Engine::StatusManager.new(@config)
 
@@ -49,8 +53,10 @@ module SplitIoClient
       return start_localhost_components if @config.localhost_mode
 
       if @config.consumer?
-        @status_manager.ready!
-        @telemetry_synchronizer.synchronize_config
+        build_synchronizer
+        build_sync_manager
+        
+        @sync_manager.start_consumer
         return
       end
       
@@ -59,7 +65,7 @@ module SplitIoClient
       build_streaming_components
       build_sync_manager
 
-      @sync_manager.start
+      @sync_manager.start if @config.valid_mode
     end
 
     def stop!
@@ -121,10 +127,10 @@ module SplitIoClient
     def validate_api_key
       if(@api_key.nil?)
         @config.logger.error('Factory Instantiation: you passed a nil api_key, api_key must be a non-empty String')
-        @config.valid_mode =  false
+        @config.valid_mode = false
       elsif (@api_key.empty?)
         @config.logger.error('Factory Instantiation: you passed and empty api_key, api_key must be a non-empty String')
-        @config.valid_mode =  false
+        @config.valid_mode = false
       end
     end
 
@@ -167,11 +173,13 @@ module SplitIoClient
         split_fetcher: @split_fetcher,
         segment_fetcher: @segment_fetcher,
         imp_counter: @impression_counter,
-        telemetry_runtime_producer: @runtime_producer,
-        telemetry_synchronizer: @telemetry_synchronizer
+        telemetry_synchronizer: @telemetry_synchronizer,
+        impressions_sender_adapter: @impressions_sender_adapter,
+        impressions_api: @impressions_api,
+        unique_keys_tracker: @unique_keys_tracker
       }
 
-      @synchronizer = Engine::Synchronizer.new(repositories, @api_key, @config, params)
+      @synchronizer = Engine::Synchronizer.new(repositories, @config, params)
     end
 
     def build_streaming_components
@@ -198,13 +206,51 @@ module SplitIoClient
     end
 
     def build_telemetry_synchronizer
-      telemetry_api = Api::TelemetryApi.new(@config, @api_key, @runtime_producer)
-      @telemetry_synchronizer = Telemetry::Synchronizer.new(@config, @telemetry_consumers, @init_producer, repositories, telemetry_api)
+      @telemetry_api = Api::TelemetryApi.new(@config, @api_key, @runtime_producer)
+      @telemetry_synchronizer = Telemetry::Synchronizer.new(@config, @telemetry_consumers, @init_producer, repositories, @telemetry_api)
+    end
+
+    def build_unique_keys_tracker
+      if @config.impressions_mode != :none
+        @unique_keys_tracker = Engine::Impressions::NoopUniqueKeysTracker.new
+        return
+      end
+
+      bf = Cache::Filter::BloomFilter.new(30_000_000)
+      filter_adapter = Cache::Filter::FilterAdapter.new(@config, bf)
+      cache = Concurrent::Hash.new
+      @unique_keys_tracker = Engine::Impressions::UniqueKeysTracker.new(@config, filter_adapter, @impressions_sender_adapter, cache)
+    end
+
+    def build_impressions_observer
+      if (@config.cache_adapter == :redis && @config.impressions_mode != :optimized) ||
+         (@config.cache_adapter == :memory && @config.impressions_mode == :none)
+        @impression_observer = Observers::NoopImpressionObserver.new
+      else
+        @impression_observer = Observers::ImpressionObserver.new
+      end
+    end
+
+    def build_impression_counter
+      case @config.impressions_mode
+      when :debug
+        @impression_counter = Engine::Common::NoopImpressionCounter.new
+      else
+        @impression_counter = Engine::Common::ImpressionCounter.new
+      end
+    end
+
+    def build_impressions_sender_adapter
+      @impressions_api = Api::Impressions.new(@api_key, @config, @runtime_producer)
+      @impressions_sender_adapter = Cache::Senders::ImpressionsSenderAdapter.new(@config, @telemetry_api, @impressions_api)
     end
 
     def build_impressions_components
-      @impression_counter = Engine::Common::ImpressionCounter.new
-      @impressions_manager = Engine::Common::ImpressionManager.new(@config, @impressions_repository, @impression_counter, @runtime_producer)
+      build_impressions_observer
+      build_impression_counter
+
+      impression_router = ImpressionRouter.new(@config)
+      @impressions_manager = Engine::Common::ImpressionManager.new(@config, @impressions_repository, @impression_counter, @runtime_producer, @impression_observer, @unique_keys_tracker, impression_router)
     end
   end
 end

data/lib/splitclient-rb/telemetry/memory/memory_synchronizer.rb

@@ -183,8 +183,10 @@ module SplitIoClient
         case @config.impressions_mode
         when :optimized
           0
-        else
+        when :debug
           1
+        else
+          2
         end
       end
     end

data/lib/splitclient-rb/utilitites.rb

@@ -37,5 +37,17 @@ module SplitIoClient
 
       interval * random_factor
     end
+
+    def split_bulk_to_send(hash, divisions)
+      count = 0
+
+      hash.each_with_object([]) do |key_value, final|
+        final[count % divisions] ||= {}
+        final[count % divisions][key_value[0]] = key_value[1]
+        count += 1
+      end
+    rescue StandardError
+      []
+    end
   end  
 end

data/lib/splitclient-rb/version.rb

@@ -1,3 +1,3 @@
 module SplitIoClient
-  VERSION = '7.3.4.pre.rc2'
+  VERSION = '7.3.5.pre.rc3'
 end

data/lib/splitclient-rb.rb

@@ -12,8 +12,11 @@ require 'splitclient-rb/cache/adapters/memory_adapter'
 require 'splitclient-rb/cache/adapters/redis_adapter'
 require 'splitclient-rb/cache/fetchers/segment_fetcher'
 require 'splitclient-rb/cache/fetchers/split_fetcher'
+require 'splitclient-rb/cache/filter/bloom_filter'
+require 'splitclient-rb/cache/filter/filter_adapter'
 require 'splitclient-rb/cache/hashers/impression_hasher'
 require 'splitclient-rb/cache/observers/impression_observer'
+require 'splitclient-rb/cache/observers/noop_impression_observer'
 require 'splitclient-rb/cache/repositories/repository'
 require 'splitclient-rb/cache/repositories/segments_repository'
 require 'splitclient-rb/cache/repositories/splits_repository'
@@ -28,6 +31,9 @@ require 'splitclient-rb/cache/senders/impressions_sender'
 require 'splitclient-rb/cache/senders/events_sender'
 require 'splitclient-rb/cache/senders/impressions_count_sender'
 require 'splitclient-rb/cache/senders/localhost_repo_cleaner'
+require 'splitclient-rb/cache/senders/impressions_sender_adapter'
+require 'splitclient-rb/cache/senders/impressions_adapter/memory_sender'
+require 'splitclient-rb/cache/senders/impressions_adapter/redis_sender'
 require 'splitclient-rb/cache/stores/localhost_split_builder'
 require 'splitclient-rb/cache/stores/localhost_split_store'
 require 'splitclient-rb/cache/stores/store_utils'
@@ -52,6 +58,7 @@ require 'splitclient-rb/engine/api/events'
 require 'splitclient-rb/engine/api/telemetry_api'
 require 'splitclient-rb/engine/common/impressions_counter'
 require 'splitclient-rb/engine/common/impressions_manager'
+require 'splitclient-rb/engine/common/noop_impressions_counter'
 require 'splitclient-rb/engine/parser/condition'
 require 'splitclient-rb/engine/parser/partition'
 require 'splitclient-rb/engine/parser/evaluator'
@@ -79,6 +86,8 @@ require 'splitclient-rb/engine/matchers/equal_to_boolean_matcher'
 require 'splitclient-rb/engine/matchers/equal_to_matcher'
 require 'splitclient-rb/engine/matchers/matches_string_matcher'
 require 'splitclient-rb/engine/evaluator/splitter'
+require 'splitclient-rb/engine/impressions/noop_unique_keys_tracker'
+require 'splitclient-rb/engine/impressions/unique_keys_tracker'
 require 'splitclient-rb/engine/metrics/binary_search_latency_tracker'
 require 'splitclient-rb/engine/models/split'
 require 'splitclient-rb/engine/models/label'

data/splitclient-rb.gemspec

@@ -50,6 +50,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'timecop', '~> 0.9'
   spec.add_development_dependency 'webmock', '~> 3.14'
 
+  spec.add_runtime_dependency 'bitarray', '~> 1.3'
   spec.add_runtime_dependency 'concurrent-ruby', '~> 1.0'
   spec.add_runtime_dependency 'faraday', '>= 0.8', '< 2.0'
   spec.add_runtime_dependency 'json', '>= 1.8', '< 3.0'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: splitclient-rb
 version: !ruby/object:Gem::Version
-  version: 7.3.4.pre.rc2
+  version: 7.3.5.pre.rc3
 platform: java
 authors:
 - Split Software
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-02-16 00:00:00.000000000 Z
+date: 2022-04-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -192,6 +192,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.14'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  name: bitarray
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -363,6 +377,7 @@ extra_rdoc_files: []
 files:
 - ".github/pull_request_template.md"
 - ".github/workflows/ci.yml"
+- ".github/workflows/update-license-year.yml"
 - ".gitignore"
 - ".rubocop.yml"
 - ".simplecov"
@@ -387,8 +402,11 @@ files:
 - lib/splitclient-rb/cache/adapters/redis_adapter.rb
 - lib/splitclient-rb/cache/fetchers/segment_fetcher.rb
 - lib/splitclient-rb/cache/fetchers/split_fetcher.rb
+- lib/splitclient-rb/cache/filter/bloom_filter.rb
+- lib/splitclient-rb/cache/filter/filter_adapter.rb
 - lib/splitclient-rb/cache/hashers/impression_hasher.rb
 - lib/splitclient-rb/cache/observers/impression_observer.rb
+- lib/splitclient-rb/cache/observers/noop_impression_observer.rb
 - lib/splitclient-rb/cache/repositories/events/memory_repository.rb
 - lib/splitclient-rb/cache/repositories/events/redis_repository.rb
 - lib/splitclient-rb/cache/repositories/events_repository.rb
@@ -400,9 +418,12 @@ files:
 - lib/splitclient-rb/cache/repositories/splits_repository.rb
 - lib/splitclient-rb/cache/routers/impression_router.rb
 - lib/splitclient-rb/cache/senders/events_sender.rb
+- lib/splitclient-rb/cache/senders/impressions_adapter/memory_sender.rb
+- lib/splitclient-rb/cache/senders/impressions_adapter/redis_sender.rb
 - lib/splitclient-rb/cache/senders/impressions_count_sender.rb
 - lib/splitclient-rb/cache/senders/impressions_formatter.rb
 - lib/splitclient-rb/cache/senders/impressions_sender.rb
+- lib/splitclient-rb/cache/senders/impressions_sender_adapter.rb
 - lib/splitclient-rb/cache/senders/localhost_repo_cleaner.rb
 - lib/splitclient-rb/cache/stores/localhost_split_builder.rb
 - lib/splitclient-rb/cache/stores/localhost_split_store.rb
@@ -421,7 +442,10 @@ files:
 - lib/splitclient-rb/engine/back_off.rb
 - lib/splitclient-rb/engine/common/impressions_counter.rb
 - lib/splitclient-rb/engine/common/impressions_manager.rb
+- lib/splitclient-rb/engine/common/noop_impressions_counter.rb
 - lib/splitclient-rb/engine/evaluator/splitter.rb
+- lib/splitclient-rb/engine/impressions/noop_unique_keys_tracker.rb
+- lib/splitclient-rb/engine/impressions/unique_keys_tracker.rb
 - lib/splitclient-rb/engine/matchers/all_keys_matcher.rb
 - lib/splitclient-rb/engine/matchers/between_matcher.rb
 - lib/splitclient-rb/engine/matchers/combiners.rb