splitclient-rb 7.1.4.pre.rc7 → 7.1.4.pre.rc8

Sign up to get free protection for your applications and to get access to all the features.
Files changed (31) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGES.txt +1 -1
  3. data/Rakefile +7 -2
  4. data/ext/murmurhash/3_x64_128.c +117 -0
  5. data/ext/murmurhash/3_x86_32.c +88 -0
  6. data/ext/murmurhash/extconf.rb +5 -0
  7. data/ext/murmurhash/murmurhash.c +255 -0
  8. data/ext/murmurhash/murmurhash.h +94 -0
  9. data/lib/splitclient-rb.rb +6 -1
  10. data/lib/splitclient-rb/cache/hashers/impression_hasher.rb +34 -0
  11. data/lib/splitclient-rb/cache/observers/impression_observer.rb +22 -0
  12. data/lib/splitclient-rb/cache/repositories/impressions/memory_repository.rb +4 -18
  13. data/lib/splitclient-rb/cache/repositories/impressions/redis_repository.rb +7 -18
  14. data/lib/splitclient-rb/cache/repositories/impressions_repository.rb +1 -27
  15. data/lib/splitclient-rb/cache/routers/impression_router.rb +12 -14
  16. data/lib/splitclient-rb/cache/senders/impressions_count_sender.rb +73 -0
  17. data/lib/splitclient-rb/cache/senders/impressions_formatter.rb +11 -11
  18. data/lib/splitclient-rb/cache/senders/impressions_sender.rb +3 -3
  19. data/lib/splitclient-rb/clients/split_client.rb +24 -73
  20. data/lib/splitclient-rb/engine/api/impressions.rb +30 -13
  21. data/lib/splitclient-rb/engine/common/impressions_counter.rb +45 -0
  22. data/lib/splitclient-rb/engine/common/impressions_manager.rb +87 -0
  23. data/lib/splitclient-rb/engine/evaluator/splitter.rb +1 -5
  24. data/lib/splitclient-rb/engine/parser/evaluator.rb +0 -4
  25. data/lib/splitclient-rb/engine/sync_manager.rb +5 -6
  26. data/lib/splitclient-rb/engine/synchronizer.rb +9 -1
  27. data/lib/splitclient-rb/split_config.rb +31 -1
  28. data/lib/splitclient-rb/split_factory.rb +5 -2
  29. data/lib/splitclient-rb/version.rb +1 -1
  30. data/splitclient-rb.gemspec +8 -1
  31. metadata +14 -17
@@ -0,0 +1,94 @@
1
+ #ifndef MURMURHASH_INCLUDED
2
+ # define MURMURHASH_INCLUDED
3
+
4
+ #include "ruby.h"
5
+
6
+ // Microsoft Visual Studio
7
+
8
+ #if defined(_MSC_VER)
9
+ #define FORCE_INLINE __forceinline
10
+ #include <stdlib.h>
11
+ #define ROTL32(x,y) _rotl(x,y)
12
+ #define ROTL64(x,y) _rotl64(x,y)
13
+ #define BIG_CONSTANT(x) (x)
14
+ #else // defined(_MSC_VER)
15
+ #define FORCE_INLINE inline __attribute__((always_inline))
16
+ #define ROTL32(x,y) rotl32(x,y)
17
+ #define ROTL64(x,y) rotl64(x,y)
18
+ #define BIG_CONSTANT(x) (x##LLU)
19
+ #endif // !defined(_MSC_VER)
20
+
21
+ #ifdef DYNAMIC_ENDIAN
22
+ /* for universal binary of NEXTSTEP and MacOS X */
23
+ /* useless since autoconf 2.63? */
24
+ static int
25
+ is_bigendian(void)
26
+ {
27
+ static int init = 0;
28
+ static int endian_value;
29
+ char *p;
30
+
31
+ if (init) return endian_value;
32
+ init = 1;
33
+ p = (char*)&init;
34
+ return endian_value = p[0] ? 0 : 1;
35
+ }
36
+ # define BIGENDIAN_P() (is_bigendian())
37
+ #elif defined(WORDS_BIGENDIAN)
38
+ # define BIGENDIAN_P() 1
39
+ #else
40
+ # define BIGENDIAN_P() 0
41
+ #endif
42
+
43
+ #define MURMURHASH_MAGIC 0x5bd1e995
44
+ #define MURMURHASH_MAGIC64A BIG_CONSTANT(0xc6a4a7935bd1e995)
45
+
46
+ void assign_by_endian_32(uint8_t *digest, uint32_t h);
47
+ void assign_by_endian_64(uint8_t *digest, uint64_t h);
48
+ void assign_by_endian_128(uint8_t*, void*);
49
+
50
+ uint32_t rotl32(uint32_t, int8_t);
51
+ uint64_t rotl64(uint64_t, int8_t);
52
+ uint32_t getblock32(const uint32_t*, int);
53
+ uint64_t getblock64(const uint64_t*, int);
54
+ uint32_t fmix32(uint32_t);
55
+ uint64_t fmix64(uint64_t);
56
+ uint32_t _murmur_finish32(VALUE, uint32_t (*)(const char*, uint32_t, uint32_t));
57
+ uint64_t _murmur_finish64(VALUE, uint64_t (*)(const char*, uint32_t, uint64_t));
58
+ void _murmur_finish128(VALUE, void*, void (*)(const char*, uint32_t, uint32_t, void*));
59
+ uint32_t _murmur_s_digest32(int, VALUE*, VALUE, uint32_t (*)(const char*, uint32_t, uint32_t));
60
+ uint64_t _murmur_s_digest64(int, VALUE*, VALUE, uint64_t (*)(const char*, uint32_t, uint64_t));
61
+ void _murmur_s_digest128(int, VALUE*, VALUE, void*, void (*)(const char*, uint32_t, uint32_t, void*));
62
+
63
+ VALUE murmur1_finish(VALUE);
64
+ VALUE murmur1_s_digest(int, VALUE*, VALUE);
65
+ VALUE murmur1_s_rawdigest(int, VALUE*, VALUE);
66
+ VALUE murmur2_finish(VALUE);
67
+ VALUE murmur2_s_digest(int, VALUE*, VALUE);
68
+ VALUE murmur2_s_rawdigest(int, VALUE*, VALUE);
69
+ VALUE murmur2a_finish(VALUE);
70
+ VALUE murmur2a_s_digest(int, VALUE*, VALUE);
71
+ VALUE murmur2a_s_rawdigest(int, VALUE*, VALUE);
72
+ VALUE murmur64a_finish(VALUE);
73
+ VALUE murmur64a_s_digest(int, VALUE*, VALUE);
74
+ VALUE murmur64a_s_rawdigest(int, VALUE*, VALUE);
75
+ VALUE murmur64b_finish(VALUE);
76
+ VALUE murmur64b_s_digest(int, VALUE*, VALUE);
77
+ VALUE murmur64b_s_rawdigest(int, VALUE*, VALUE);
78
+ VALUE murmur_neutral2_finish(VALUE);
79
+ VALUE murmur_neutral2_s_digest(int, VALUE*, VALUE);
80
+ VALUE murmur_neutral2_s_rawdigest(int, VALUE*, VALUE);
81
+ VALUE murmur_aligned2_finish(VALUE);
82
+ VALUE murmur_aligned2_s_digest(int, VALUE*, VALUE);
83
+ VALUE murmur_aligned2_s_rawdigest(int, VALUE*, VALUE);
84
+ VALUE murmur3_x86_32_finish(VALUE);
85
+ VALUE murmur3_x86_32_s_digest(int, VALUE*, VALUE);
86
+ VALUE murmur3_x86_32_s_rawdigest(int, VALUE*, VALUE);
87
+ VALUE murmur3_x86_128_finish(VALUE);
88
+ VALUE murmur3_x86_128_s_digest(int, VALUE*, VALUE);
89
+ VALUE murmur3_x86_128_s_rawdigest(int, VALUE*, VALUE);
90
+ VALUE murmur3_x64_128_finish(VALUE);
91
+ VALUE murmur3_x64_128_s_digest(int, VALUE*, VALUE);
92
+ VALUE murmur3_x64_128_s_rawdigest(int, VALUE*, VALUE);
93
+
94
+ #endif /* ifndef MURMURHASH_INCLUDED */
@@ -12,6 +12,8 @@ require 'splitclient-rb/cache/adapters/memory_adapter'
12
12
  require 'splitclient-rb/cache/adapters/redis_adapter'
13
13
  require 'splitclient-rb/cache/fetchers/segment_fetcher'
14
14
  require 'splitclient-rb/cache/fetchers/split_fetcher'
15
+ require 'splitclient-rb/cache/hashers/impression_hasher'
16
+ require 'splitclient-rb/cache/observers/impression_observer'
15
17
  require 'splitclient-rb/cache/repositories/repository'
16
18
  require 'splitclient-rb/cache/repositories/segments_repository'
17
19
  require 'splitclient-rb/cache/repositories/splits_repository'
@@ -28,6 +30,7 @@ require 'splitclient-rb/cache/senders/impressions_formatter'
28
30
  require 'splitclient-rb/cache/senders/impressions_sender'
29
31
  require 'splitclient-rb/cache/senders/metrics_sender'
30
32
  require 'splitclient-rb/cache/senders/events_sender'
33
+ require 'splitclient-rb/cache/senders/impressions_count_sender'
31
34
  require 'splitclient-rb/cache/senders/localhost_repo_cleaner'
32
35
  require 'splitclient-rb/cache/stores/store_utils'
33
36
  require 'splitclient-rb/cache/stores/localhost_split_builder'
@@ -52,6 +55,8 @@ require 'splitclient-rb/engine/api/metrics'
52
55
  require 'splitclient-rb/engine/api/segments'
53
56
  require 'splitclient-rb/engine/api/splits'
54
57
  require 'splitclient-rb/engine/api/events'
58
+ require 'splitclient-rb/engine/common/impressions_counter'
59
+ require 'splitclient-rb/engine/common/impressions_manager'
55
60
  require 'splitclient-rb/engine/parser/condition'
56
61
  require 'splitclient-rb/engine/parser/partition'
57
62
  require 'splitclient-rb/engine/parser/evaluator'
@@ -105,7 +110,7 @@ require 'splitclient-rb/sse/notification_processor'
105
110
  require 'splitclient-rb/sse/sse_handler'
106
111
 
107
112
  # C extension
108
- #require 'murmurhash/murmurhash_mri'
113
+ require 'murmurhash/murmurhash_mri'
109
114
 
110
115
  module SplitIoClient
111
116
  def self.root
@@ -0,0 +1,34 @@
1
+ module SplitIoClient
2
+ module Hashers
3
+ class ImpressionHasher
4
+ def initialize
5
+ @murmur_hash_128_64 = case RUBY_PLATFORM
6
+ when 'java'
7
+ Proc.new { |key, seed| Java::MurmurHash3.hash128x64(key, seed) }
8
+ else
9
+ Proc.new { |key, seed| Digest::MurmurHashMRI3_x64_128.rawdigest(key, [seed].pack('L')) }
10
+ end
11
+ end
12
+
13
+ def process(impression)
14
+ impression_data = "#{unknown_if_null(impression[:k])}"
15
+ impression_data << ":#{unknown_if_null(impression[:f])}"
16
+ impression_data << ":#{unknown_if_null(impression[:t])}"
17
+ impression_data << ":#{unknown_if_null(impression[:r])}"
18
+ impression_data << ":#{zero_if_null(impression[:c])}"
19
+
20
+ @murmur_hash_128_64.call(impression_data, 0)[0];
21
+ end
22
+
23
+ private
24
+
25
+ def unknown_if_null(value)
26
+ value == nil ? "UNKNOWN" : value
27
+ end
28
+
29
+ def zero_if_null(value)
30
+ value == nil ? 0 : value
31
+ end
32
+ end
33
+ end
34
+ end
@@ -0,0 +1,22 @@
1
+ module SplitIoClient
2
+ module Observers
3
+ class ImpressionObserver
4
+ LAST_SEEN_CACHE_SIZE = 500000
5
+
6
+ def initialize
7
+ @cache = LruRedux::TTL::ThreadSafeCache.new(LAST_SEEN_CACHE_SIZE)
8
+ @impression_hasher = Hashers::ImpressionHasher.new
9
+ end
10
+
11
+ def test_and_set(impression)
12
+ return if impression.nil?
13
+
14
+ hash = @impression_hasher.process(impression)
15
+ previous = @cache[hash]
16
+ @cache[hash] = impression[:m]
17
+
18
+ previous.nil? ? nil : [previous, impression[:m]].min
19
+ end
20
+ end
21
+ end
22
+ end
@@ -10,18 +10,10 @@ module SplitIoClient
10
10
  @adapter = @config.impressions_adapter
11
11
  end
12
12
 
13
- # Store impression data in the selected adapter
14
- def add(matching_key, bucketing_key, split_name, treatment, time)
15
- @adapter.add_to_queue(
16
- m: metadata,
17
- i: impression_data(
18
- matching_key,
19
- bucketing_key,
20
- split_name,
21
- treatment,
22
- time
23
- )
24
- )
13
+ def add_bulk(impressions)
14
+ impressions.each do |impression|
15
+ @adapter.add_to_queue(impression)
16
+ end
25
17
  rescue ThreadError # queue is full
26
18
  if random_sampler.rand(1..1000) <= 2 # log only 0.2 % of the time
27
19
  @config.logger.warn("Dropping impressions. Current size is \
@@ -30,12 +22,6 @@ module SplitIoClient
30
22
  end
31
23
  end
32
24
 
33
- def add_bulk(key, bucketing_key, treatments, time)
34
- treatments.each do |split_name, treatment|
35
- add(key, bucketing_key, split_name, treatment, time)
36
- end
37
- end
38
-
39
25
  def batch
40
26
  return [] if @config.impressions_bulk_size.zero?
41
27
 
@@ -12,28 +12,17 @@ module SplitIoClient
12
12
  @adapter = @config.impressions_adapter
13
13
  end
14
14
 
15
- def add(matching_key, bucketing_key, split_name, treatment, time)
16
- add_bulk(matching_key, bucketing_key, { split_name => treatment }, time)
17
- end
18
-
19
- def add_bulk(matching_key, bucketing_key, treatments, time)
20
- impressions = treatments.map do |split_name, treatment|
21
- {
22
- m: metadata,
23
- i: impression_data(
24
- matching_key,
25
- bucketing_key,
26
- split_name,
27
- treatment,
28
- time
29
- )
30
- }.to_json
15
+ def add_bulk(impressions)
16
+ impressions_json = impressions.map do |impression|
17
+ impression.to_json
31
18
  end
32
19
 
33
- impressions_list_size = @adapter.add_to_queue(key, impressions)
20
+ impressions_list_size = @adapter.add_to_queue(key, impressions_json)
34
21
 
35
22
  # Synchronizer might not be running
36
- @adapter.expire(key, EXPIRE_SECONDS) if impressions.size == impressions_list_size
23
+ @adapter.expire(key, EXPIRE_SECONDS) if impressions_json.size == impressions_list_size
24
+ rescue StandardError => e
25
+ @config.logger.error("Exception while add_bulk_v2: #{e}")
37
26
  end
38
27
 
39
28
  def get_impressions(number_of_impressions = 0)
@@ -6,7 +6,7 @@ module SplitIoClient
6
6
  # Repository which forwards impressions interface to the selected adapter
7
7
  class ImpressionsRepository < Repository
8
8
  extend Forwardable
9
- def_delegators :@repository, :add, :add_bulk, :batch, :clear, :empty?
9
+ def_delegators :@repository, :add_bulk, :batch, :clear, :empty?
10
10
 
11
11
  def initialize(config)
12
12
  super(config)
@@ -17,32 +17,6 @@ module SplitIoClient
17
17
  Repositories::Impressions::RedisRepository.new(@config)
18
18
  end
19
19
  end
20
-
21
- protected
22
-
23
- def impression_data(matching_key, bucketing_key, split_name, treatment, timestamp)
24
- {
25
- k: matching_key,
26
- b: bucketing_key,
27
- f: split_name,
28
- t: treatment[:treatment],
29
- r: applied_rule(treatment[:label]),
30
- c: treatment[:change_number],
31
- m: timestamp
32
- }
33
- end
34
-
35
- def metadata
36
- {
37
- s: "#{@config.language}-#{@config.version}",
38
- i: @config.machine_ip,
39
- n: @config.machine_name
40
- }
41
- end
42
-
43
- def applied_rule(label)
44
- @config.labels_enabled ? label : nil
45
- end
46
20
  end
47
21
  end
48
22
  end
@@ -18,24 +18,22 @@ module SplitIoClient
18
18
  end
19
19
  end
20
20
 
21
- def add(impression)
22
- enqueue(impression)
23
- end
24
-
25
21
  def add_bulk(impressions)
26
- impressions[:split_names].each do |split_name|
22
+ return unless @listener
23
+ impressions.each do |impression|
27
24
  enqueue(
28
- split_name: split_name.to_s,
29
- matching_key: impressions[:matching_key],
30
- bucketing_key: impressions[:bucketing_key],
31
- time: impressions[:time],
25
+ split_name: impression[:i][:f],
26
+ matching_key: impression[:i][:k],
27
+ bucketing_key: impression[:i][:b],
28
+ time: impression[:i][:m],
32
29
  treatment: {
33
- label: impressions[:treatments_labels_change_numbers][split_name.to_sym][:label],
34
- treatment: impressions[:treatments_labels_change_numbers][split_name.to_sym][:treatment],
35
- change_number: impressions[:treatments_labels_change_numbers][split_name.to_sym][:change_number]
30
+ label: impression[:i][:r],
31
+ treatment: impression[:i][:t],
32
+ change_number: impression[:i][:c]
36
33
  },
37
- attributes: impressions[:attributes]
38
- ) unless impressions[:treatments_labels_change_numbers][split_name.to_sym].nil?
34
+ previous_time: impression[:i][:pt],
35
+ attributes: impression[:attributes]
36
+ ) unless impression.nil?
39
37
  end
40
38
  end
41
39
 
@@ -0,0 +1,73 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SplitIoClient
4
+ module Cache
5
+ module Senders
6
+ class ImpressionsCountSender
7
+ COUNTER_REFRESH_RATE_SECONDS = 1800
8
+
9
+ def initialize(config, impression_counter, impressions_api)
10
+ @config = config
11
+ @impression_counter = impression_counter
12
+ @impressions_api = impressions_api
13
+ end
14
+
15
+ def call
16
+ impressions_count_thread
17
+
18
+ if defined?(PhusionPassenger)
19
+ PhusionPassenger.on_event(:starting_worker_process) do |forked|
20
+ impressions_count_thread if forked
21
+ end
22
+ end
23
+ end
24
+
25
+ private
26
+
27
+ def impressions_count_thread
28
+ @config.threads[:impressions_count_sender] = Thread.new do
29
+ begin
30
+ @config.logger.info('Starting impressions count service')
31
+
32
+ loop do
33
+ post_impressions_count
34
+
35
+ sleep(COUNTER_REFRESH_RATE_SECONDS)
36
+ end
37
+ rescue SplitIoClient::SDKShutdownException
38
+ post_impressions_count
39
+
40
+ @config.logger.info('Posting impressions count due to shutdown')
41
+ end
42
+ end
43
+
44
+ def post_impressions_count
45
+ @impressions_api.post_count(formatter(@impression_counter.pop_all))
46
+ rescue StandardError => error
47
+ @config.log_found_exception(__method__.to_s, error)
48
+ end
49
+
50
+ def formatter(counts)
51
+ return if counts.empty?
52
+
53
+ formated_counts = {pf: []}
54
+
55
+ counts.each do |key, value|
56
+ key_splited = key.split('::')
57
+
58
+ formated_counts[:pf] << {
59
+ f: key_splited[0].to_s, # feature name
60
+ m: key_splited[1].to_i, # time frame
61
+ rc: value # count
62
+ }
63
+ end
64
+
65
+ formated_counts
66
+ rescue StandardError => error
67
+ @config.log_found_exception(__method__.to_s, error)
68
+ end
69
+ end
70
+ end
71
+ end
72
+ end
73
+ end
@@ -17,12 +17,10 @@ module SplitIoClient
17
17
 
18
18
  formatted_impressions = unique_features(filtered_impressions).each_with_object([]) do |feature, memo|
19
19
  feature_impressions = feature_impressions(filtered_impressions, feature)
20
- ip = feature_impressions.first[:m][:i]
21
20
  current_impressions = current_impressions(feature_impressions)
22
21
  memo << {
23
- testName: feature.to_sym,
24
- keyImpressions: current_impressions,
25
- ip: ip
22
+ f: feature.to_sym,
23
+ i: current_impressions
26
24
  }
27
25
  end
28
26
 
@@ -40,12 +38,13 @@ module SplitIoClient
40
38
  def current_impressions(feature_impressions)
41
39
  feature_impressions.map do |impression|
42
40
  {
43
- keyName: impression[:i][:k],
44
- treatment: impression[:i][:t],
45
- time: impression[:i][:m],
46
- bucketingKey: impression[:i][:b],
47
- label: impression[:i][:r],
48
- changeNumber: impression[:i][:c]
41
+ k: impression[:i][:k],
42
+ t: impression[:i][:t],
43
+ m: impression[:i][:m],
44
+ b: impression[:i][:b],
45
+ r: impression[:i][:r],
46
+ c: impression[:i][:c],
47
+ pt: impression[:i][:pt]
49
48
  }
50
49
  end
51
50
  end
@@ -73,7 +72,8 @@ module SplitIoClient
73
72
  "#{impression[:i][:k]}:" \
74
73
  "#{impression[:i][:b]}:" \
75
74
  "#{impression[:i][:c]}:" \
76
- "#{impression[:i][:t]}"
75
+ "#{impression[:i][:t]}:" \
76
+ "#{impression[:i][:pt]}"
77
77
  end
78
78
  end
79
79
  end