splitclient-rb 7.1.4.pre.rc7 → 7.1.4.pre.rc8

data/ext/murmurhash/murmurhash.h

@@ -0,0 +1,94 @@
+#ifndef MURMURHASH_INCLUDED
+#  define MURMURHASH_INCLUDED
+
+#include "ruby.h"
+
+// Microsoft Visual Studio
+
+#if defined(_MSC_VER)
+  #define FORCE_INLINE    __forceinline
+  #include <stdlib.h>
+  #define ROTL32(x,y)     _rotl(x,y)
+  #define ROTL64(x,y)     _rotl64(x,y)
+  #define BIG_CONSTANT(x) (x)
+#else   // defined(_MSC_VER)
+  #define FORCE_INLINE inline __attribute__((always_inline))
+  #define ROTL32(x,y)     rotl32(x,y)
+  #define ROTL64(x,y)     rotl64(x,y)
+  #define BIG_CONSTANT(x) (x##LLU)
+#endif // !defined(_MSC_VER)
+
+#ifdef DYNAMIC_ENDIAN
+  /* for universal binary of NEXTSTEP and MacOS X */
+  /* useless since autoconf 2.63? */
+  static int
+  is_bigendian(void)
+  {
+    static int init = 0;
+    static int endian_value;
+    char *p;
+
+    if (init) return endian_value;
+    init = 1;
+    p = (char*)&init;
+    return endian_value = p[0] ? 0 : 1;
+  }
+#  define BIGENDIAN_P() (is_bigendian())
+#elif defined(WORDS_BIGENDIAN)
+#  define BIGENDIAN_P() 1
+#else
+#  define BIGENDIAN_P() 0
+#endif
+
+#define MURMURHASH_MAGIC 0x5bd1e995
+#define MURMURHASH_MAGIC64A BIG_CONSTANT(0xc6a4a7935bd1e995)
+
+void assign_by_endian_32(uint8_t *digest, uint32_t h);
+void assign_by_endian_64(uint8_t *digest, uint64_t h);
+void assign_by_endian_128(uint8_t*, void*);
+
+uint32_t rotl32(uint32_t, int8_t);
+uint64_t rotl64(uint64_t, int8_t);
+uint32_t getblock32(const uint32_t*, int);
+uint64_t getblock64(const uint64_t*, int);
+uint32_t fmix32(uint32_t);
+uint64_t fmix64(uint64_t);
+uint32_t _murmur_finish32(VALUE, uint32_t (*)(const char*, uint32_t, uint32_t));
+uint64_t _murmur_finish64(VALUE, uint64_t (*)(const char*, uint32_t, uint64_t));
+void _murmur_finish128(VALUE, void*, void (*)(const char*, uint32_t, uint32_t, void*));
+uint32_t _murmur_s_digest32(int, VALUE*, VALUE, uint32_t (*)(const char*, uint32_t, uint32_t));
+uint64_t _murmur_s_digest64(int, VALUE*, VALUE, uint64_t (*)(const char*, uint32_t, uint64_t));
+void _murmur_s_digest128(int, VALUE*, VALUE, void*, void (*)(const char*, uint32_t, uint32_t, void*));
+
+VALUE murmur1_finish(VALUE);
+VALUE murmur1_s_digest(int, VALUE*, VALUE);
+VALUE murmur1_s_rawdigest(int, VALUE*, VALUE);
+VALUE murmur2_finish(VALUE);
+VALUE murmur2_s_digest(int, VALUE*, VALUE);
+VALUE murmur2_s_rawdigest(int, VALUE*, VALUE);
+VALUE murmur2a_finish(VALUE);
+VALUE murmur2a_s_digest(int, VALUE*, VALUE);
+VALUE murmur2a_s_rawdigest(int, VALUE*, VALUE);
+VALUE murmur64a_finish(VALUE);
+VALUE murmur64a_s_digest(int, VALUE*, VALUE);
+VALUE murmur64a_s_rawdigest(int, VALUE*, VALUE);
+VALUE murmur64b_finish(VALUE);
+VALUE murmur64b_s_digest(int, VALUE*, VALUE);
+VALUE murmur64b_s_rawdigest(int, VALUE*, VALUE);
+VALUE murmur_neutral2_finish(VALUE);
+VALUE murmur_neutral2_s_digest(int, VALUE*, VALUE);
+VALUE murmur_neutral2_s_rawdigest(int, VALUE*, VALUE);
+VALUE murmur_aligned2_finish(VALUE);
+VALUE murmur_aligned2_s_digest(int, VALUE*, VALUE);
+VALUE murmur_aligned2_s_rawdigest(int, VALUE*, VALUE);
+VALUE murmur3_x86_32_finish(VALUE);
+VALUE murmur3_x86_32_s_digest(int, VALUE*, VALUE);
+VALUE murmur3_x86_32_s_rawdigest(int, VALUE*, VALUE);
+VALUE murmur3_x86_128_finish(VALUE);
+VALUE murmur3_x86_128_s_digest(int, VALUE*, VALUE);
+VALUE murmur3_x86_128_s_rawdigest(int, VALUE*, VALUE);
+VALUE murmur3_x64_128_finish(VALUE);
+VALUE murmur3_x64_128_s_digest(int, VALUE*, VALUE);
+VALUE murmur3_x64_128_s_rawdigest(int, VALUE*, VALUE);
+
+#endif /* ifndef MURMURHASH_INCLUDED */

data/lib/splitclient-rb.rb

@@ -12,6 +12,8 @@ require 'splitclient-rb/cache/adapters/memory_adapter'
 require 'splitclient-rb/cache/adapters/redis_adapter'
 require 'splitclient-rb/cache/fetchers/segment_fetcher'
 require 'splitclient-rb/cache/fetchers/split_fetcher'
+require 'splitclient-rb/cache/hashers/impression_hasher'
+require 'splitclient-rb/cache/observers/impression_observer'
 require 'splitclient-rb/cache/repositories/repository'
 require 'splitclient-rb/cache/repositories/segments_repository'
 require 'splitclient-rb/cache/repositories/splits_repository'
@@ -28,6 +30,7 @@ require 'splitclient-rb/cache/senders/impressions_formatter'
 require 'splitclient-rb/cache/senders/impressions_sender'
 require 'splitclient-rb/cache/senders/metrics_sender'
 require 'splitclient-rb/cache/senders/events_sender'
+require 'splitclient-rb/cache/senders/impressions_count_sender'
 require 'splitclient-rb/cache/senders/localhost_repo_cleaner'
 require 'splitclient-rb/cache/stores/store_utils'
 require 'splitclient-rb/cache/stores/localhost_split_builder'
@@ -52,6 +55,8 @@ require 'splitclient-rb/engine/api/metrics'
 require 'splitclient-rb/engine/api/segments'
 require 'splitclient-rb/engine/api/splits'
 require 'splitclient-rb/engine/api/events'
+require 'splitclient-rb/engine/common/impressions_counter'
+require 'splitclient-rb/engine/common/impressions_manager'
 require 'splitclient-rb/engine/parser/condition'
 require 'splitclient-rb/engine/parser/partition'
 require 'splitclient-rb/engine/parser/evaluator'
@@ -105,7 +110,7 @@ require 'splitclient-rb/sse/notification_processor'
 require 'splitclient-rb/sse/sse_handler'
 
 # C extension
-#require 'murmurhash/murmurhash_mri'
+require 'murmurhash/murmurhash_mri'
 
 module SplitIoClient
   def self.root

data/lib/splitclient-rb/cache/hashers/impression_hasher.rb

@@ -0,0 +1,34 @@
+module SplitIoClient
+  module Hashers
+    class ImpressionHasher
+      def initialize
+        @murmur_hash_128_64 = case RUBY_PLATFORM
+        when 'java' 
+          Proc.new { |key, seed| Java::MurmurHash3.hash128x64(key, seed) }
+        else
+          Proc.new { |key, seed| Digest::MurmurHashMRI3_x64_128.rawdigest(key, [seed].pack('L')) }
+        end
+      end
+
+      def process(impression)
+        impression_data = "#{unknown_if_null(impression[:k])}"
+        impression_data << ":#{unknown_if_null(impression[:f])}"
+        impression_data << ":#{unknown_if_null(impression[:t])}"
+        impression_data << ":#{unknown_if_null(impression[:r])}"
+        impression_data << ":#{zero_if_null(impression[:c])}"
+        
+        @murmur_hash_128_64.call(impression_data, 0)[0];
+      end
+
+      private
+
+      def unknown_if_null(value)
+        value == nil ? "UNKNOWN" : value
+      end
+
+      def zero_if_null(value)
+        value == nil ? 0 : value
+      end
+    end
+  end
+end

data/lib/splitclient-rb/cache/observers/impression_observer.rb

@@ -0,0 +1,22 @@
+module SplitIoClient
+  module Observers
+    class ImpressionObserver
+      LAST_SEEN_CACHE_SIZE = 500000
+
+      def initialize
+        @cache = LruRedux::TTL::ThreadSafeCache.new(LAST_SEEN_CACHE_SIZE)
+        @impression_hasher = Hashers::ImpressionHasher.new
+      end
+
+      def test_and_set(impression)
+        return if impression.nil?
+        
+        hash = @impression_hasher.process(impression)
+        previous = @cache[hash]
+        @cache[hash] = impression[:m]
+
+        previous.nil? ? nil : [previous, impression[:m]].min
+      end
+    end
+  end
+end

data/lib/splitclient-rb/cache/repositories/impressions/memory_repository.rb

@@ -10,18 +10,10 @@ module SplitIoClient
             @adapter = @config.impressions_adapter
           end
 
-          # Store impression data in the selected adapter
-          def add(matching_key, bucketing_key, split_name, treatment, time)
-            @adapter.add_to_queue(
-              m: metadata,
-              i: impression_data(
-                matching_key,
-                bucketing_key,
-                split_name,
-                treatment,
-                time
-              )
-            )
+          def add_bulk(impressions)
+            impressions.each do |impression|
+              @adapter.add_to_queue(impression)
+            end
           rescue ThreadError # queue is full
             if random_sampler.rand(1..1000) <= 2 # log only 0.2 % of the time
               @config.logger.warn("Dropping impressions. Current size is \
@@ -30,12 +22,6 @@ module SplitIoClient
             end
           end
 
-          def add_bulk(key, bucketing_key, treatments, time)
-            treatments.each do |split_name, treatment|
-              add(key, bucketing_key, split_name, treatment, time)
-            end
-          end
-
           def batch
             return [] if @config.impressions_bulk_size.zero?
 

data/lib/splitclient-rb/cache/repositories/impressions/redis_repository.rb

@@ -12,28 +12,17 @@ module SplitIoClient
             @adapter = @config.impressions_adapter
           end
 
-          def add(matching_key, bucketing_key, split_name, treatment, time)
-            add_bulk(matching_key, bucketing_key, { split_name => treatment }, time)
-          end
-
-          def add_bulk(matching_key, bucketing_key, treatments, time)
-            impressions = treatments.map do |split_name, treatment|
-              {
-                m: metadata,
-                i: impression_data(
-                  matching_key,
-                  bucketing_key,
-                  split_name,
-                  treatment,
-                  time
-                )
-              }.to_json
+          def add_bulk(impressions)
+            impressions_json = impressions.map do |impression|
+              impression.to_json
             end
 
-            impressions_list_size = @adapter.add_to_queue(key, impressions)
+            impressions_list_size = @adapter.add_to_queue(key, impressions_json)
 
             # Synchronizer might not be running
-            @adapter.expire(key, EXPIRE_SECONDS) if impressions.size == impressions_list_size
+            @adapter.expire(key, EXPIRE_SECONDS) if impressions_json.size == impressions_list_size
+          rescue StandardError => e
+            @config.logger.error("Exception while add_bulk_v2: #{e}")
           end
 
           def get_impressions(number_of_impressions = 0)

data/lib/splitclient-rb/cache/repositories/impressions_repository.rb

@@ -6,7 +6,7 @@ module SplitIoClient
       # Repository which forwards impressions interface to the selected adapter
       class ImpressionsRepository < Repository
         extend Forwardable
-        def_delegators :@repository, :add, :add_bulk, :batch, :clear, :empty?
+        def_delegators :@repository, :add_bulk, :batch, :clear, :empty?
 
         def initialize(config)
           super(config)
@@ -17,32 +17,6 @@ module SplitIoClient
                        Repositories::Impressions::RedisRepository.new(@config)
                      end
         end
-
-        protected
-
-        def impression_data(matching_key, bucketing_key, split_name, treatment, timestamp)
-          {
-            k: matching_key,
-            b: bucketing_key,
-            f: split_name,
-            t: treatment[:treatment],
-            r: applied_rule(treatment[:label]),
-            c: treatment[:change_number],
-            m: timestamp
-          }
-        end
-
-        def metadata
-          {
-            s: "#{@config.language}-#{@config.version}",
-            i: @config.machine_ip,
-            n: @config.machine_name
-          }
-        end
-
-        def applied_rule(label)
-          @config.labels_enabled ? label : nil
-        end
       end
     end
   end

data/lib/splitclient-rb/cache/routers/impression_router.rb

@@ -18,24 +18,22 @@ module SplitIoClient
       end
     end
 
-    def add(impression)
-      enqueue(impression)
-    end
-
     def add_bulk(impressions)
-      impressions[:split_names].each do |split_name|
+      return unless @listener
+      impressions.each do |impression|
         enqueue(
-          split_name: split_name.to_s,
-          matching_key: impressions[:matching_key],
-          bucketing_key: impressions[:bucketing_key],
-          time: impressions[:time],
+          split_name: impression[:i][:f],
+          matching_key: impression[:i][:k],
+          bucketing_key: impression[:i][:b],
+          time: impression[:i][:m],
           treatment: {
-            label: impressions[:treatments_labels_change_numbers][split_name.to_sym][:label],
-            treatment: impressions[:treatments_labels_change_numbers][split_name.to_sym][:treatment],
-            change_number: impressions[:treatments_labels_change_numbers][split_name.to_sym][:change_number]
+            label: impression[:i][:r],
+            treatment: impression[:i][:t],
+            change_number: impression[:i][:c]
           },
-          attributes: impressions[:attributes]
-        ) unless impressions[:treatments_labels_change_numbers][split_name.to_sym].nil?
+          previous_time: impression[:i][:pt],
+          attributes: impression[:attributes]
+        ) unless impression.nil?
       end
     end
 

data/lib/splitclient-rb/cache/senders/impressions_count_sender.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+module SplitIoClient
+  module Cache
+    module Senders
+      class ImpressionsCountSender
+        COUNTER_REFRESH_RATE_SECONDS = 1800
+
+        def initialize(config, impression_counter, impressions_api)
+          @config = config
+          @impression_counter = impression_counter
+          @impressions_api = impressions_api
+        end
+
+        def call
+          impressions_count_thread
+
+          if defined?(PhusionPassenger)
+            PhusionPassenger.on_event(:starting_worker_process) do |forked|
+              impressions_count_thread if forked
+            end
+          end
+        end
+
+        private
+
+        def impressions_count_thread
+          @config.threads[:impressions_count_sender] = Thread.new do
+            begin
+              @config.logger.info('Starting impressions count service')
+
+              loop do
+                post_impressions_count
+
+                sleep(COUNTER_REFRESH_RATE_SECONDS)
+              end
+            rescue SplitIoClient::SDKShutdownException
+              post_impressions_count
+
+              @config.logger.info('Posting impressions count due to shutdown')
+            end
+          end
+
+          def post_impressions_count
+            @impressions_api.post_count(formatter(@impression_counter.pop_all))
+          rescue StandardError => error
+            @config.log_found_exception(__method__.to_s, error)
+          end
+
+          def formatter(counts)
+            return if counts.empty?
+
+            formated_counts = {pf: []}
+
+            counts.each do |key, value|              
+              key_splited = key.split('::')
+              
+              formated_counts[:pf] << {
+                f: key_splited[0].to_s, # feature name
+                m: key_splited[1].to_i, # time frame
+                rc: value # count
+              }
+            end
+
+            formated_counts
+          rescue StandardError => error
+            @config.log_found_exception(__method__.to_s, error)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/splitclient-rb/cache/senders/impressions_formatter.rb

@@ -17,12 +17,10 @@ module SplitIoClient
 
           formatted_impressions = unique_features(filtered_impressions).each_with_object([]) do |feature, memo|
             feature_impressions = feature_impressions(filtered_impressions, feature)
-            ip = feature_impressions.first[:m][:i]
             current_impressions = current_impressions(feature_impressions)
             memo << {
-              testName: feature.to_sym,
-              keyImpressions: current_impressions,
-              ip: ip
+              f: feature.to_sym,
+              i: current_impressions
             }
           end
 
@@ -40,12 +38,13 @@ module SplitIoClient
         def current_impressions(feature_impressions)
           feature_impressions.map do |impression|
             {
-              keyName: impression[:i][:k],
-              treatment: impression[:i][:t],
-              time: impression[:i][:m],
-              bucketingKey: impression[:i][:b],
-              label: impression[:i][:r],
-              changeNumber: impression[:i][:c]
+              k: impression[:i][:k],
+              t: impression[:i][:t],
+              m: impression[:i][:m],
+              b: impression[:i][:b],
+              r: impression[:i][:r],
+              c: impression[:i][:c],
+              pt: impression[:i][:pt]
             }
           end
         end
@@ -73,7 +72,8 @@ module SplitIoClient
           "#{impression[:i][:k]}:" \
           "#{impression[:i][:b]}:" \
           "#{impression[:i][:c]}:" \
-          "#{impression[:i][:t]}"
+          "#{impression[:i][:t]}:" \
+          "#{impression[:i][:pt]}"
         end
       end
     end