splitclient-rb 7.3.4.pre.rc2-java → 7.3.5.pre.rc1-java

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 7ea2b0463e8a3c2f13075209d9e0cb8b189696b3
4
- data.tar.gz: 601571ffb3a015fbf8dcb468577b05caced467c7
3
+ metadata.gz: 52ee386fb2542467575c4f58ddfd25b3d33ff33e
4
+ data.tar.gz: 6a6d7cd53ca3d0c7ad30c8fa8d3621c43e9a63d2
5
5
  SHA512:
6
- metadata.gz: c1eaa3106070a63477fa3951ff55a0c24c4287d379384da95852fb34ddc2a20d61557d3282e40f4c56d14f915f6bd615223b8830a8bd0d2b9afd36d32331f83e
7
- data.tar.gz: bde0cdac66e0bdeb0e5e02fc8e77ef8c834c56a767fdf197c398add674e1dbf1e19c683feda9384ab5ce2f80571d0c633e50e8417ab2cd3c0dfab8b7005b1cb3
6
+ metadata.gz: fb3e3c3dac2de8df3994d40bb10ba3dddf1e1ca8ae382a144980ddb8ba61ff76fb4dad4f40ce4755ed0f81e63650f84ca238c9b4c1353db86dfa39891131be88
7
+ data.tar.gz: f68eb21afac8f031a56eecb358d1e79520f38036b44f20f0596e07163b1cf1c4b50263f419b9388d556d2e919f206bffa17bd4f16e7f25276ba6d592487b3268
@@ -0,0 +1,45 @@
1
+ name: Update License Year
2
+
3
+ on:
4
+ schedule:
5
+ - cron: "0 3 1 1 *" # 03:00 AM on January 1
6
+
7
+ permissions:
8
+ contents: write
9
+ pull-requests: write
10
+
11
+ jobs:
12
+ test:
13
+ runs-on: ubuntu-latest
14
+ steps:
15
+ - name: Checkout
16
+ uses: actions/checkout@v2
17
+ with:
18
+ fetch-depth: 0
19
+
20
+ - name: Set Current year
21
+ run: "echo CURRENT=$(date +%Y) >> $GITHUB_ENV"
22
+
23
+ - name: Set Previous Year
24
+ run: "echo PREVIOUS=$(($CURRENT-1)) >> $GITHUB_ENV"
25
+
26
+ - name: Update LICENSE
27
+ uses: jacobtomlinson/gha-find-replace@v2
28
+ with:
29
+ find: ${{ env.PREVIOUS }}
30
+ replace: ${{ env.CURRENT }}
31
+ include: "LICENSE"
32
+ regex: false
33
+
34
+ - name: Commit files
35
+ run: |
36
+ git config user.name 'github-actions[bot]'
37
+ git config user.email 'github-actions[bot]@users.noreply.github.com'
38
+ git commit -m "Updated License Year" -a
39
+
40
+ - name: Create Pull Request
41
+ uses: peter-evans/create-pull-request@v3
42
+ with:
43
+ token: ${{ secrets.GITHUB_TOKEN }}
44
+ title: Update License Year
45
+ branch: update-license
data/.rubocop.yml CHANGED
@@ -26,7 +26,7 @@ Metrics/ParameterLists:
26
26
  - lib/splitclient-rb/engine/sync_manager.rb
27
27
 
28
28
  Metrics/LineLength:
29
- Max: 130
29
+ Max: 135
30
30
  Exclude:
31
31
  - spec/sse/**/*
32
32
  - spec/integrations/**/*
@@ -35,6 +35,7 @@ Metrics/LineLength:
35
35
  - spec/telemetry/synchronizer_spec.rb
36
36
  - spec/splitclient/split_config_spec.rb
37
37
  - spec/engine/push_manager_spec.rb
38
+ - spec/cache/senders/impressions_sender_adapter_spec.rb
38
39
 
39
40
  Style/BracesAroundHashParameters:
40
41
  Exclude:
data/CHANGES.txt CHANGED
@@ -1,6 +1,10 @@
1
1
  CHANGES
2
2
 
3
- 7.3.3 (Jan 28, 2021)
3
+ 7.3.4 (Feb 21, 2022)
4
+ - Updated streaming events architecture with a new queue logic.
5
+ - Fixed redis integration Pipelining command deprecation warning.
6
+
7
+ 7.3.3 (Jan 28, 2022)
4
8
  - Fixed edge cases where the sdk lost streaming connection.
5
9
  - Updated default auth service url to https://auth.split.io/api/v2/auth
6
10
  - Updated dependencies:
data/LICENSE CHANGED
@@ -1,4 +1,4 @@
1
- Copyright © 2021 Split Software, Inc.
1
+ Copyright © 2022 Split Software, Inc.
2
2
 
3
3
  Licensed under the Apache License, Version 2.0 (the "License");
4
4
  you may not use this file except in compliance with the License.
@@ -0,0 +1,32 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SplitIoClient
4
+ module Cache
5
+ module Filter
6
+ class FilterAdapter
7
+ def initialize(config, filter)
8
+ @config = config
9
+ @filter = filter
10
+ end
11
+
12
+ def add(feature_name, key)
13
+ @filter.insert("#{feature_name}#{key}")
14
+ rescue StandardError => e
15
+ @config.log_found_exception(__method__.to_s, e)
16
+ end
17
+
18
+ def contains?(feature_name, key)
19
+ @filter.include?("#{feature_name}#{key}")
20
+ rescue StandardError => e
21
+ @config.log_found_exception(__method__.to_s, e)
22
+ end
23
+
24
+ def clear
25
+ @filter.clear
26
+ rescue StandardError => e
27
+ @config.log_found_exception(__method__.to_s, e)
28
+ end
29
+ end
30
+ end
31
+ end
32
+ end
@@ -0,0 +1,10 @@
1
+ module SplitIoClient
2
+ module Observers
3
+ class NoopImpressionObserver
4
+ def test_and_set(impression)
5
+ # no-op
6
+ end
7
+ end
8
+ end
9
+ end
10
+
@@ -0,0 +1,71 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SplitIoClient
4
+ module Cache
5
+ module Senders
6
+ class MemoryImpressionsSender < ImpressionsSenderAdapter
7
+ def initialize(config, telemetry_api, impressions_api)
8
+ @config = config
9
+ @telemetry_api = telemetry_api
10
+ @impressions_api = impressions_api
11
+ end
12
+
13
+ def record_uniques_key(uniques)
14
+ uniques_keys = uniques_formatter(uniques)
15
+
16
+ @telemetry_api.record_unique_keys(uniques_keys) unless uniques_keys.nil?
17
+ rescue StandardError => e
18
+ @config.log_found_exception(__method__.to_s, e)
19
+ end
20
+
21
+ def record_impressions_count(impressions_count)
22
+ counts = impressions_count_formatter(impressions_count)
23
+
24
+ @impressions_api.post_count(counts) unless counts.nil?
25
+ rescue StandardError => e
26
+ @config.log_found_exception(__method__.to_s, e)
27
+ end
28
+
29
+ private
30
+
31
+ def uniques_formatter(uniques)
32
+ return if uniques.nil? || uniques.empty?
33
+
34
+ to_return = { mtks: [] }
35
+ uniques.each do |key, value|
36
+ to_return[:mtks] << {
37
+ f: key,
38
+ ks: value.to_a
39
+ }
40
+ end
41
+
42
+ to_return
43
+ rescue StandardError => error
44
+ @config.log_found_exception(__method__.to_s, error)
45
+ nil
46
+ end
47
+
48
+ def impressions_count_formatter(counts)
49
+ return if counts.nil? || counts.empty?
50
+
51
+ formated_counts = {pf: []}
52
+
53
+ counts.each do |key, value|
54
+ key_splited = key.split('::')
55
+
56
+ formated_counts[:pf] << {
57
+ f: key_splited[0].to_s, # feature name
58
+ m: key_splited[1].to_i, # time frame
59
+ rc: value # count
60
+ }
61
+ end
62
+
63
+ formated_counts
64
+ rescue StandardError => error
65
+ @config.log_found_exception(__method__.to_s, error)
66
+ nil
67
+ end
68
+ end
69
+ end
70
+ end
71
+ end
@@ -0,0 +1,77 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SplitIoClient
4
+ module Cache
5
+ module Senders
6
+ class RedisImpressionsSender < ImpressionsSenderAdapter
7
+ EXPIRE_SECONDS = 3600
8
+
9
+ def initialize(config)
10
+ @config = config
11
+ @adapter = @config.impressions_adapter
12
+ end
13
+
14
+ def record_uniques_key(uniques)
15
+ formatted = uniques_formatter(uniques)
16
+
17
+ unless formatted.nil?
18
+ size = @adapter.add_to_queue(unique_keys_key, formatted)
19
+ @adapter.expire(unique_keys_key, EXPIRE_SECONDS) if formatted.size == size
20
+ end
21
+ rescue StandardError => e
22
+ @config.log_found_exception(__method__.to_s, e)
23
+ end
24
+
25
+ def record_impressions_count(impressions_count)
26
+ return if impressions_count.nil? || impressions_count.empty?
27
+
28
+ result = @adapter.redis.pipelined do |pipeline|
29
+ impressions_count.each do |key, value|
30
+ pipeline.hincrby(impressions_count_key, key, value)
31
+ end
32
+
33
+ @future = pipeline.hlen(impressions_count_key)
34
+ end
35
+
36
+ expire_impressions_count_key(impressions_count, result)
37
+ rescue StandardError => e
38
+ @config.log_found_exception(__method__.to_s, e)
39
+ end
40
+
41
+ private
42
+
43
+ def expire_impressions_count_key(impressions_count, pipeline_result)
44
+ total_count = impressions_count.sum { |_, value| value }
45
+ hlen = pipeline_result.last
46
+
47
+ @adapter.expire(impressions_count_key, EXPIRE_SECONDS) if impressions_count.size == hlen && (pipeline_result.sum - hlen) == total_count
48
+ end
49
+
50
+ def impressions_count_key
51
+ "#{@config.redis_namespace}.impressions.count"
52
+ end
53
+
54
+ def unique_keys_key
55
+ "#{@config.redis_namespace}.uniquekeys"
56
+ end
57
+
58
+ def uniques_formatter(uniques)
59
+ return if uniques.nil? || uniques.empty?
60
+
61
+ to_return = []
62
+ uniques.each do |key, value|
63
+ to_return << {
64
+ f: key,
65
+ k: value.to_a
66
+ }
67
+ end
68
+
69
+ to_return
70
+ rescue StandardError => error
71
+ @config.log_found_exception(__method__.to_s, error)
72
+ nil
73
+ end
74
+ end
75
+ end
76
+ end
77
+ end
@@ -4,12 +4,10 @@ module SplitIoClient
4
4
  module Cache
5
5
  module Senders
6
6
  class ImpressionsCountSender
7
- COUNTER_REFRESH_RATE_SECONDS = 1800
8
-
9
- def initialize(config, impression_counter, impressions_api)
7
+ def initialize(config, impression_counter, impressions_sender_adapter)
10
8
  @config = config
11
9
  @impression_counter = impression_counter
12
- @impressions_api = impressions_api
10
+ @impressions_sender_adapter = impressions_sender_adapter
13
11
  end
14
12
 
15
13
  def call
@@ -22,13 +20,11 @@ module SplitIoClient
22
20
  @config.threads[:impressions_count_sender] = Thread.new do
23
21
  begin
24
22
  @config.logger.info('Starting impressions count service')
25
-
26
23
  loop do
27
- post_impressions_count
28
-
29
- sleep(COUNTER_REFRESH_RATE_SECONDS)
24
+ sleep(@config.counter_refresh_rate)
25
+ post_impressions_count
30
26
  end
31
- rescue SplitIoClient::SDKShutdownException
27
+ rescue SplitIoClient::SDKShutdownException
32
28
  post_impressions_count
33
29
 
34
30
  @config.logger.info('Posting impressions count due to shutdown')
@@ -37,27 +33,7 @@ module SplitIoClient
37
33
  end
38
34
 
39
35
  def post_impressions_count
40
- @impressions_api.post_count(formatter(@impression_counter.pop_all))
41
- rescue StandardError => error
42
- @config.log_found_exception(__method__.to_s, error)
43
- end
44
-
45
- def formatter(counts)
46
- return if counts.empty?
47
-
48
- formated_counts = {pf: []}
49
-
50
- counts.each do |key, value|
51
- key_splited = key.split('::')
52
-
53
- formated_counts[:pf] << {
54
- f: key_splited[0].to_s, # feature name
55
- m: key_splited[1].to_i, # time frame
56
- rc: value # count
57
- }
58
- end
59
-
60
- formated_counts
36
+ @impressions_sender_adapter.record_impressions_count(@impression_counter.pop_all)
61
37
  rescue StandardError => error
62
38
  @config.log_found_exception(__method__.to_s, error)
63
39
  end
@@ -0,0 +1,21 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SplitIoClient
4
+ module Cache
5
+ module Senders
6
+ class ImpressionsSenderAdapter
7
+ extend Forwardable
8
+ def_delegators :@sender, :record_uniques_key, :record_impressions_count
9
+
10
+ def initialize(config, telemetry_api, impressions_api)
11
+ @sender = case config.telemetry_adapter.class.to_s
12
+ when 'SplitIoClient::Cache::Adapters::RedisAdapter'
13
+ Cache::Senders::RedisImpressionsSender.new(config)
14
+ else
15
+ Cache::Senders::MemoryImpressionsSender.new(config, telemetry_api, impressions_api)
16
+ end
17
+ end
18
+ end
19
+ end
20
+ end
21
+ end
@@ -17,6 +17,14 @@ module SplitIoClient
17
17
  post_telemetry("#{@config.telemetry_service_url}/metrics/usage", stats, 'stats')
18
18
  end
19
19
 
20
+ def record_unique_keys(uniques)
21
+ return if uniques[:mtks].empty?
22
+
23
+ post_telemetry("#{@config.telemetry_service_url}/mtks/ss", uniques, 'mtks')
24
+ rescue StandardError => e
25
+ @config.log_found_exception(__method__.to_s, e)
26
+ end
27
+
20
28
  private
21
29
 
22
30
  def post_telemetry(url, obj, method)
@@ -4,65 +4,75 @@ module SplitIoClient
4
4
  module Engine
5
5
  module Common
6
6
  class ImpressionManager
7
- def initialize(config, impressions_repository, impression_counter, telemetry_runtime_producer)
7
+ def initialize(config,
8
+ impressions_repository,
9
+ impression_counter,
10
+ telemetry_runtime_producer,
11
+ impression_observer,
12
+ unique_keys_tracker,
13
+ impression_router)
8
14
  @config = config
9
15
  @impressions_repository = impressions_repository
10
16
  @impression_counter = impression_counter
11
- @impression_observer = SplitIoClient::Observers::ImpressionObserver.new
17
+ @impression_observer = impression_observer
12
18
  @telemetry_runtime_producer = telemetry_runtime_producer
19
+ @unique_keys_tracker = unique_keys_tracker
20
+ @impression_router = impression_router
13
21
  end
14
22
 
15
- # added param time for test
16
23
  def build_impression(matching_key, bucketing_key, split_name, treatment, params = {})
17
24
  impression_data = impression_data(matching_key, bucketing_key, split_name, treatment, params[:time])
18
25
 
19
- impression_data[:pt] = @impression_observer.test_and_set(impression_data) unless redis?
20
-
21
- @impression_counter.inc(split_name, impression_data[:m]) if optimized? && !redis?
26
+ begin
27
+ case @config.impressions_mode
28
+ when :debug # In DEBUG mode we should calculate the pt only.
29
+ impression_data[:pt] = @impression_observer.test_and_set(impression_data)
30
+ when :none # In NONE mode we should track the total amount of evaluations and the unique keys.
31
+ @impression_counter.inc(split_name, impression_data[:m])
32
+ @unique_keys_tracker.track(split_name, matching_key)
33
+ else # In OPTIMIZED mode we should track the total amount of evaluations and deduplicate the impressions.
34
+ impression_data[:pt] = @impression_observer.test_and_set(impression_data)
35
+ @impression_counter.inc(split_name, impression_data[:m])
36
+ end
37
+ rescue StandardError => e
38
+ @config.log_found_exception(__method__.to_s, e)
39
+ end
22
40
 
23
41
  impression(impression_data, params[:attributes])
24
- rescue StandardError => e
25
- @config.log_found_exception(__method__.to_s, e)
26
42
  end
27
43
 
28
44
  def track(impressions)
29
45
  return if impressions.empty?
30
46
 
31
- impression_router.add_bulk(impressions)
32
-
33
- dropped = 0
34
- queued = 0
35
- dedupe = 0
36
-
37
- if optimized? && !redis?
38
- optimized_impressions = impressions.select { |imp| should_queue_impression?(imp[:i]) }
39
-
40
- unless optimized_impressions.empty?
41
- dropped = @impressions_repository.add_bulk(optimized_impressions)
42
- dedupe = impressions.length - optimized_impressions.length
43
- queued = optimized_impressions.length - dropped
47
+ stats = { dropped: 0, queued: 0, dedupe: 0 }
48
+ begin
49
+ case @config.impressions_mode
50
+ when :none
51
+ return
52
+ when :debug
53
+ track_debug_mode(impressions, stats)
54
+ when :optimized
55
+ track_optimized_mode(impressions, stats)
44
56
  end
45
- else
46
- dropped = @impressions_repository.add_bulk(impressions)
47
- queued = impressions.length - dropped
57
+ rescue StandardError => e
58
+ @config.log_found_exception(__method__.to_s, e)
59
+ ensure
60
+ record_stats(stats)
61
+ @impression_router.add_bulk(impressions)
48
62
  end
49
-
50
- record_stats(queued, dropped, dedupe)
51
- rescue StandardError => e
52
- @config.log_found_exception(__method__.to_s, e)
53
63
  end
54
64
 
55
65
  private
56
66
 
57
- def record_stats(queued, dropped, dedupe)
67
+ def record_stats(stats)
58
68
  return if redis?
59
69
 
60
70
  imp_queued = Telemetry::Domain::Constants::IMPRESSIONS_QUEUED
61
71
  imp_dropped = Telemetry::Domain::Constants::IMPRESSIONS_DROPPED
62
72
  imp_dedupe = Telemetry::Domain::Constants::IMPRESSIONS_DEDUPE
63
- @telemetry_runtime_producer.record_impressions_stats(imp_queued, queued) unless queued.zero?
64
- @telemetry_runtime_producer.record_impressions_stats(imp_dropped, dropped) unless dropped.zero?
65
- @telemetry_runtime_producer.record_impressions_stats(imp_dedupe, dedupe) unless dedupe.zero?
73
+ @telemetry_runtime_producer.record_impressions_stats(imp_queued, stats[:queued]) unless stats[:queued].zero?
74
+ @telemetry_runtime_producer.record_impressions_stats(imp_dropped, stats[:dropped]) unless stats[:dropped].zero?
75
+ @telemetry_runtime_producer.record_impressions_stats(imp_dedupe, stats[:dedupe]) unless stats[:dedupe].zero?
66
76
  end
67
77
 
68
78
  # added param time for test
@@ -91,10 +101,6 @@ module SplitIoClient
91
101
  @config.labels_enabled ? label : nil
92
102
  end
93
103
 
94
- def optimized?
95
- @config.impressions_mode == :optimized
96
- end
97
-
98
104
  def should_queue_impression?(impression)
99
105
  impression[:pt].nil? ||
100
106
  (ImpressionCounter.truncate_time_frame(impression[:pt]) != ImpressionCounter.truncate_time_frame(impression[:m]))
@@ -108,10 +114,19 @@ module SplitIoClient
108
114
  @config.impressions_adapter.class.to_s == 'SplitIoClient::Cache::Adapters::RedisAdapter'
109
115
  end
110
116
 
111
- def impression_router
112
- @impression_router ||= SplitIoClient::ImpressionRouter.new(@config)
113
- rescue StandardError => error
114
- @config.log_found_exception(__method__.to_s, error)
117
+ def track_debug_mode(impressions, stats)
118
+ stats[:dropped] = @impressions_repository.add_bulk(impressions)
119
+ stats[:queued] = impressions.length - stats[:dropped]
120
+ end
121
+
122
+ def track_optimized_mode(impressions, stats)
123
+ optimized_impressions = impressions.select { |imp| should_queue_impression?(imp[:i]) }
124
+
125
+ return if optimized_impressions.empty?
126
+
127
+ stats[:dropped] = @impressions_repository.add_bulk(optimized_impressions)
128
+ stats[:dedupe] = impressions.length - optimized_impressions.length
129
+ stats[:queued] = optimized_impressions.length - stats[:dropped]
115
130
  end
116
131
  end
117
132
  end
@@ -0,0 +1,27 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'concurrent'
4
+
5
+ module SplitIoClient
6
+ module Engine
7
+ module Common
8
+ class NoopImpressionCounter
9
+ def inc(split_name, time_frame)
10
+ # no-op
11
+ end
12
+
13
+ def pop_all
14
+ # no-op
15
+ end
16
+
17
+ def make_key(split_name, time_frame)
18
+ # no-op
19
+ end
20
+
21
+ def self.truncate_time_frame(timestamp_ms)
22
+ # no-op
23
+ end
24
+ end
25
+ end
26
+ end
27
+ end
@@ -0,0 +1,17 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SplitIoClient
4
+ module Engine
5
+ module Impressions
6
+ class NoopUniqueKeysTracker
7
+ def call
8
+ # no-op
9
+ end
10
+
11
+ def track(feature_name, key)
12
+ # no-op
13
+ end
14
+ end
15
+ end
16
+ end
17
+ end
@@ -0,0 +1,97 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SplitIoClient
4
+ module Engine
5
+ module Impressions
6
+ class UniqueKeysTracker
7
+ INTERVAL_TO_CLEAR_LONG_TERM_CACHE = 86_400 # 24 hours
8
+
9
+ def initialize(config,
10
+ filter_adapter,
11
+ sender_adapter,
12
+ cache)
13
+ @config = config
14
+ @filter_adapter = filter_adapter
15
+ @sender_adapter = sender_adapter
16
+ @cache = cache
17
+ @cache_max_size = config.unique_keys_cache_max_size
18
+ @max_bulk_size = config.unique_keys_bulk_size
19
+ @semaphore = Mutex.new
20
+ end
21
+
22
+ def call
23
+ @config.threads[:unique_keys_sender] = Thread.new { send_bulk_data_thread }
24
+ @config.threads[:clear_filter] = Thread.new { clear_filter_thread }
25
+ end
26
+
27
+ def track(feature_name, key)
28
+ return false if @filter_adapter.contains?(feature_name, key)
29
+
30
+ @filter_adapter.add(feature_name, key)
31
+
32
+ add_or_update(feature_name, key)
33
+
34
+ send_bulk_data if @cache.size >= @cache_max_size
35
+
36
+ true
37
+ rescue StandardError => e
38
+ @config.log_found_exception(__method__.to_s, e)
39
+ false
40
+ end
41
+
42
+ private
43
+
44
+ def send_bulk_data_thread
45
+ @config.logger.info('Starting Unique Keys Tracker.') if @config.debug_enabled
46
+ loop do
47
+ sleep(@config.unique_keys_refresh_rate)
48
+ send_bulk_data
49
+ end
50
+ rescue SplitIoClient::SDKShutdownException
51
+ send_bulk_data
52
+ @config.logger.info('Posting unique keys due to shutdown')
53
+ end
54
+
55
+ def clear_filter_thread
56
+ loop do
57
+ sleep(INTERVAL_TO_CLEAR_LONG_TERM_CACHE)
58
+ @config.logger.debug('Starting task to clean the filter cache.') if @config.debug_enabled
59
+ @filter_adapter.clear
60
+ end
61
+ rescue SplitIoClient::SDKShutdownException
62
+ @filter_adapter.clear
63
+ end
64
+
65
+ def add_or_update(feature_name, key)
66
+ if @cache[feature_name].nil?
67
+ @cache[feature_name] = Set.new([key])
68
+ else
69
+ @cache[feature_name].add(key)
70
+ end
71
+ end
72
+
73
+ def send_bulk_data
74
+ @semaphore.synchronize do
75
+ return if @cache.empty?
76
+
77
+ uniques = @cache.clone
78
+ @cache.clear
79
+
80
+ if uniques.size <= @max_bulk_size
81
+ @sender_adapter.record_uniques_key(uniques)
82
+ return
83
+ end
84
+
85
+ bulks = SplitIoClient::Utilities.split_bulk_to_send(uniques, uniques.size / @max_bulk_size)
86
+
87
+ bulks.each do |b|
88
+ @sender_adapter.record_uniques_key(b)
89
+ end
90
+ end
91
+ rescue StandardError => e
92
+ @config.log_found_exception(__method__.to_s, e)
93
+ end
94
+ end
95
+ end
96
+ end
97
+ end
@@ -52,9 +52,8 @@ module SplitIoClient
52
52
  # @return [boolean] match value for combiner delegates
53
53
  def eval_and(args)
54
54
  # Convert all keys to symbols
55
- if args && args[:attributes]
56
- args[:attributes] = args[:attributes].each_with_object({}) { |(k, v), memo| memo[k.to_sym] = v }
57
- end
55
+ args[:attributes] = args[:attributes].each_with_object({}) { |(k, v), memo| memo[k.to_sym] = v } if args && args[:attributes]
56
+
58
57
  @matchers.all? do |matcher|
59
58
  if match_with_key?(matcher)
60
59
  matcher.match?(value: args[:matching_key])
@@ -30,6 +30,11 @@ module SplitIoClient
30
30
  PhusionPassenger.on_event(:starting_worker_process) { |forked| start_thread if forked } if defined?(PhusionPassenger)
31
31
  end
32
32
 
33
+ def start_consumer
34
+ start_consumer_thread
35
+ PhusionPassenger.on_event(:starting_worker_process) { |forked| start_consumer_thread if forked } if defined?(PhusionPassenger)
36
+ end
37
+
33
38
  private
34
39
 
35
40
  def start_thread
@@ -55,6 +60,14 @@ module SplitIoClient
55
60
  end
56
61
  end
57
62
 
63
+ def start_consumer_thread
64
+ @config.threads[:start_sdk_consumer] = Thread.new do
65
+ @status_manager.ready!
66
+ @telemetry_synchronizer.synchronize_config
67
+ @synchronizer.start_periodic_data_recording
68
+ end
69
+ end
70
+
58
71
  def process_subsystem_ready
59
72
  @synchronizer.stop_periodic_fetch
60
73
  @synchronizer.sync_all
@@ -12,7 +12,6 @@ module SplitIoClient
12
12
 
13
13
  def initialize(
14
14
  repositories,
15
- api_key,
16
15
  config,
17
16
  params
18
17
  )
@@ -20,13 +19,14 @@ module SplitIoClient
20
19
  @segments_repository = repositories[:segments]
21
20
  @impressions_repository = repositories[:impressions]
22
21
  @events_repository = repositories[:events]
23
- @api_key = api_key
24
22
  @config = config
25
23
  @split_fetcher = params[:split_fetcher]
26
24
  @segment_fetcher = params[:segment_fetcher]
27
- @impressions_api = SplitIoClient::Api::Impressions.new(@api_key, @config, params[:telemetry_runtime_producer])
25
+ @impressions_api = params[:impressions_api]
28
26
  @impression_counter = params[:imp_counter]
29
27
  @telemetry_synchronizer = params[:telemetry_synchronizer]
28
+ @impressions_sender_adapter = params[:impressions_sender_adapter]
29
+ @unique_keys_tracker = params[:unique_keys_tracker]
30
30
  end
31
31
 
32
32
  def sync_all(asynchronous = true)
@@ -42,10 +42,14 @@ module SplitIoClient
42
42
  end
43
43
 
44
44
  def start_periodic_data_recording
45
- impressions_sender
46
- events_sender
45
+ unless @config.consumer?
46
+ impressions_sender
47
+ events_sender
48
+ start_telemetry_sync_task
49
+ end
50
+
47
51
  impressions_count_sender
48
- start_telemetry_sync_task
52
+ start_unique_keys_tracker_task
49
53
  end
50
54
 
51
55
  def start_periodic_fetch
@@ -170,7 +174,7 @@ module SplitIoClient
170
174
 
171
175
  # Starts thread which loops constantly and sends impressions to the Split API
172
176
  def impressions_sender
173
- ImpressionsSender.new(@impressions_repository, @config, @impressions_api).call
177
+ ImpressionsSender.new(@impressions_repository, @config, @impressions_api).call unless @config.impressions_mode == :none
174
178
  end
175
179
 
176
180
  # Starts thread which loops constantly and sends events to the Split API
@@ -180,13 +184,17 @@ module SplitIoClient
180
184
 
181
185
  # Starts thread which loops constantly and sends impressions count to the Split API
182
186
  def impressions_count_sender
183
- ImpressionsCountSender.new(@config, @impression_counter, @impressions_api).call
187
+ ImpressionsCountSender.new(@config, @impression_counter, @impressions_sender_adapter).call unless @config.impressions_mode == :debug
184
188
  end
185
189
 
186
190
  def start_telemetry_sync_task
187
191
  Telemetry::SyncTask.new(@config, @telemetry_synchronizer).call
188
192
  end
189
193
 
194
+ def start_unique_keys_tracker_task
195
+ @unique_keys_tracker.call
196
+ end
197
+
190
198
  def sync_result(success, remaining_attempts, segment_names = nil)
191
199
  { success: success, remaining_attempts: remaining_attempts, segment_names: segment_names }
192
200
  end
@@ -111,6 +111,12 @@ module SplitIoClient
111
111
  @telemetry_refresh_rate = SplitConfig.init_telemetry_refresh_rate(opts[:telemetry_refresh_rate])
112
112
  @telemetry_service_url = opts[:telemetry_service_url] || SplitConfig.default_telemetry_service_url
113
113
 
114
+ @unique_keys_refresh_rate = SplitConfig.default_unique_keys_refresh_rate(@cache_adapter)
115
+ @unique_keys_cache_max_size = SplitConfig.default_unique_keys_cache_max_size
116
+ @unique_keys_bulk_size = SplitConfig.default_unique_keys_bulk_size(@cache_adapter)
117
+
118
+ @counter_refresh_rate = SplitConfig.default_counter_refresh_rate(@cache_adapter)
119
+
114
120
  @sdk_start_time = Time.now
115
121
 
116
122
  @on_demand_fetch_retry_delay_seconds = SplitConfig.default_on_demand_fetch_retry_delay_seconds
@@ -284,6 +290,18 @@ module SplitIoClient
284
290
  attr_accessor :on_demand_fetch_retry_delay_seconds
285
291
  attr_accessor :on_demand_fetch_max_retries
286
292
 
293
+ attr_accessor :unique_keys_refresh_rate
294
+ attr_accessor :unique_keys_cache_max_size
295
+ attr_accessor :unique_keys_bulk_size
296
+
297
+ attr_accessor :counter_refresh_rate
298
+
299
+ def self.default_counter_refresh_rate(adapter)
300
+ return 300 if adapter == :redis # Send bulk impressions count - Refresh rate: 5 min.
301
+
302
+ 1800 # Send bulk impressions count - Refresh rate: 30 min.
303
+ end
304
+
287
305
  def self.default_on_demand_fetch_retry_delay_seconds
288
306
  0.05
289
307
  end
@@ -302,6 +320,8 @@ module SplitIoClient
302
320
  case impressions_mode
303
321
  when :debug
304
322
  return :debug
323
+ when :none
324
+ return :none
305
325
  else
306
326
  @logger.error('You passed an invalid impressions_mode, impressions_mode should be one of the following values: :debug or :optimized. Defaulting to :optimized mode') unless impressions_mode == :optimized
307
327
  return :optimized
@@ -468,6 +488,22 @@ module SplitIoClient
468
488
  3600
469
489
  end
470
490
 
491
+ def self.default_unique_keys_refresh_rate(adapter)
492
+ return 300 if adapter == :redis
493
+
494
+ 900
495
+ end
496
+
497
+ def self.default_unique_keys_cache_max_size
498
+ 30000
499
+ end
500
+
501
+ def self.default_unique_keys_bulk_size(adapter)
502
+ return 2000 if adapter == :redis
503
+
504
+ 5000
505
+ end
506
+
471
507
  def self.default_telemetry_service_url
472
508
  'https://telemetry.split.io/api/v1'
473
509
  end
@@ -1,3 +1,7 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'bloomfilter-rb'
4
+
1
5
  module SplitIoClient
2
6
  class SplitFactory
3
7
  ROOT_PROCESS_ID = Process.pid
@@ -34,8 +38,10 @@ module SplitIoClient
34
38
 
35
39
  build_telemetry_components
36
40
  build_repositories
37
- build_impressions_components
38
41
  build_telemetry_synchronizer
42
+ build_impressions_sender_adapter
43
+ build_unique_keys_tracker
44
+ build_impressions_components
39
45
 
40
46
  @status_manager = Engine::StatusManager.new(@config)
41
47
 
@@ -49,8 +55,10 @@ module SplitIoClient
49
55
  return start_localhost_components if @config.localhost_mode
50
56
 
51
57
  if @config.consumer?
52
- @status_manager.ready!
53
- @telemetry_synchronizer.synchronize_config
58
+ build_synchronizer
59
+ build_sync_manager
60
+
61
+ @sync_manager.start_consumer
54
62
  return
55
63
  end
56
64
 
@@ -167,11 +175,13 @@ module SplitIoClient
167
175
  split_fetcher: @split_fetcher,
168
176
  segment_fetcher: @segment_fetcher,
169
177
  imp_counter: @impression_counter,
170
- telemetry_runtime_producer: @runtime_producer,
171
- telemetry_synchronizer: @telemetry_synchronizer
178
+ telemetry_synchronizer: @telemetry_synchronizer,
179
+ impressions_sender_adapter: @impressions_sender_adapter,
180
+ impressions_api: @impressions_api,
181
+ unique_keys_tracker: @unique_keys_tracker
172
182
  }
173
183
 
174
- @synchronizer = Engine::Synchronizer.new(repositories, @api_key, @config, params)
184
+ @synchronizer = Engine::Synchronizer.new(repositories, @config, params)
175
185
  end
176
186
 
177
187
  def build_streaming_components
@@ -198,13 +208,51 @@ module SplitIoClient
198
208
  end
199
209
 
200
210
  def build_telemetry_synchronizer
201
- telemetry_api = Api::TelemetryApi.new(@config, @api_key, @runtime_producer)
202
- @telemetry_synchronizer = Telemetry::Synchronizer.new(@config, @telemetry_consumers, @init_producer, repositories, telemetry_api)
211
+ @telemetry_api = Api::TelemetryApi.new(@config, @api_key, @runtime_producer)
212
+ @telemetry_synchronizer = Telemetry::Synchronizer.new(@config, @telemetry_consumers, @init_producer, repositories, @telemetry_api)
213
+ end
214
+
215
+ def build_unique_keys_tracker
216
+ if @config.impressions_mode != :none
217
+ @unique_keys_tracker = Engine::Impressions::NoopUniqueKeysTracker.new
218
+ return
219
+ end
220
+
221
+ bf = BloomFilter::Native.new(size: 95_850_584, hashes: 2)
222
+ filter_adapter = Cache::Filter::FilterAdapter.new(@config, bf)
223
+ cache = Concurrent::Hash.new
224
+ @unique_keys_tracker = Engine::Impressions::UniqueKeysTracker.new(@config, filter_adapter, @impressions_sender_adapter, cache)
225
+ end
226
+
227
+ def build_impressions_observer
228
+ if (@config.cache_adapter == :redis && @config.impressions_mode != :optimized) ||
229
+ (@config.cache_adapter == :memory && @config.impressions_mode == :none)
230
+ @impression_observer = Observers::NoopImpressionObserver.new
231
+ else
232
+ @impression_observer = Observers::ImpressionObserver.new
233
+ end
234
+ end
235
+
236
+ def build_impression_counter
237
+ case @config.impressions_mode
238
+ when :debug
239
+ @impression_counter = Engine::Common::NoopImpressionCounter.new
240
+ else
241
+ @impression_counter = Engine::Common::ImpressionCounter.new
242
+ end
243
+ end
244
+
245
+ def build_impressions_sender_adapter
246
+ @impressions_api = Api::Impressions.new(@api_key, @config, @runtime_producer)
247
+ @impressions_sender_adapter = Cache::Senders::ImpressionsSenderAdapter.new(@config, @telemetry_api, @impressions_api)
203
248
  end
204
249
 
205
250
  def build_impressions_components
206
- @impression_counter = Engine::Common::ImpressionCounter.new
207
- @impressions_manager = Engine::Common::ImpressionManager.new(@config, @impressions_repository, @impression_counter, @runtime_producer)
251
+ build_impressions_observer
252
+ build_impression_counter
253
+
254
+ impression_router = ImpressionRouter.new(@config)
255
+ @impressions_manager = Engine::Common::ImpressionManager.new(@config, @impressions_repository, @impression_counter, @runtime_producer, @impression_observer, @unique_keys_tracker, impression_router)
208
256
  end
209
257
  end
210
258
  end
@@ -183,8 +183,10 @@ module SplitIoClient
183
183
  case @config.impressions_mode
184
184
  when :optimized
185
185
  0
186
- else
186
+ when :debug
187
187
  1
188
+ else
189
+ 2
188
190
  end
189
191
  end
190
192
  end
@@ -37,5 +37,17 @@ module SplitIoClient
37
37
 
38
38
  interval * random_factor
39
39
  end
40
+
41
+ def split_bulk_to_send(hash, divisions)
42
+ count = 0
43
+
44
+ hash.each_with_object([]) do |key_value, final|
45
+ final[count % divisions] ||= {}
46
+ final[count % divisions][key_value[0]] = key_value[1]
47
+ count += 1
48
+ end
49
+ rescue StandardError
50
+ []
51
+ end
40
52
  end
41
53
  end
@@ -1,3 +1,3 @@
1
1
  module SplitIoClient
2
- VERSION = '7.3.4.pre.rc2'
2
+ VERSION = '7.3.5.pre.rc1'
3
3
  end
@@ -12,8 +12,10 @@ require 'splitclient-rb/cache/adapters/memory_adapter'
12
12
  require 'splitclient-rb/cache/adapters/redis_adapter'
13
13
  require 'splitclient-rb/cache/fetchers/segment_fetcher'
14
14
  require 'splitclient-rb/cache/fetchers/split_fetcher'
15
+ require 'splitclient-rb/cache/filter/filter_adapter'
15
16
  require 'splitclient-rb/cache/hashers/impression_hasher'
16
17
  require 'splitclient-rb/cache/observers/impression_observer'
18
+ require 'splitclient-rb/cache/observers/noop_impression_observer'
17
19
  require 'splitclient-rb/cache/repositories/repository'
18
20
  require 'splitclient-rb/cache/repositories/segments_repository'
19
21
  require 'splitclient-rb/cache/repositories/splits_repository'
@@ -28,6 +30,9 @@ require 'splitclient-rb/cache/senders/impressions_sender'
28
30
  require 'splitclient-rb/cache/senders/events_sender'
29
31
  require 'splitclient-rb/cache/senders/impressions_count_sender'
30
32
  require 'splitclient-rb/cache/senders/localhost_repo_cleaner'
33
+ require 'splitclient-rb/cache/senders/impressions_sender_adapter'
34
+ require 'splitclient-rb/cache/senders/impressions_adapter/memory_sender'
35
+ require 'splitclient-rb/cache/senders/impressions_adapter/redis_sender'
31
36
  require 'splitclient-rb/cache/stores/localhost_split_builder'
32
37
  require 'splitclient-rb/cache/stores/localhost_split_store'
33
38
  require 'splitclient-rb/cache/stores/store_utils'
@@ -52,6 +57,7 @@ require 'splitclient-rb/engine/api/events'
52
57
  require 'splitclient-rb/engine/api/telemetry_api'
53
58
  require 'splitclient-rb/engine/common/impressions_counter'
54
59
  require 'splitclient-rb/engine/common/impressions_manager'
60
+ require 'splitclient-rb/engine/common/noop_impressions_counter'
55
61
  require 'splitclient-rb/engine/parser/condition'
56
62
  require 'splitclient-rb/engine/parser/partition'
57
63
  require 'splitclient-rb/engine/parser/evaluator'
@@ -79,6 +85,8 @@ require 'splitclient-rb/engine/matchers/equal_to_boolean_matcher'
79
85
  require 'splitclient-rb/engine/matchers/equal_to_matcher'
80
86
  require 'splitclient-rb/engine/matchers/matches_string_matcher'
81
87
  require 'splitclient-rb/engine/evaluator/splitter'
88
+ require 'splitclient-rb/engine/impressions/noop_unique_keys_tracker'
89
+ require 'splitclient-rb/engine/impressions/unique_keys_tracker'
82
90
  require 'splitclient-rb/engine/metrics/binary_search_latency_tracker'
83
91
  require 'splitclient-rb/engine/models/split'
84
92
  require 'splitclient-rb/engine/models/label'
@@ -50,6 +50,7 @@ Gem::Specification.new do |spec|
50
50
  spec.add_development_dependency 'timecop', '~> 0.9'
51
51
  spec.add_development_dependency 'webmock', '~> 3.14'
52
52
 
53
+ spec.add_runtime_dependency 'bloomfilter-rb', '~> 2.1'
53
54
  spec.add_runtime_dependency 'concurrent-ruby', '~> 1.0'
54
55
  spec.add_runtime_dependency 'faraday', '>= 0.8', '< 2.0'
55
56
  spec.add_runtime_dependency 'json', '>= 1.8', '< 3.0'
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: splitclient-rb
3
3
  version: !ruby/object:Gem::Version
4
- version: 7.3.4.pre.rc2
4
+ version: 7.3.5.pre.rc1
5
5
  platform: java
6
6
  authors:
7
7
  - Split Software
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2022-02-16 00:00:00.000000000 Z
11
+ date: 2022-04-06 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
@@ -192,6 +192,20 @@ dependencies:
192
192
  - - "~>"
193
193
  - !ruby/object:Gem::Version
194
194
  version: '3.14'
195
+ - !ruby/object:Gem::Dependency
196
+ requirement: !ruby/object:Gem::Requirement
197
+ requirements:
198
+ - - "~>"
199
+ - !ruby/object:Gem::Version
200
+ version: '2.1'
201
+ name: bloomfilter-rb
202
+ prerelease: false
203
+ type: :runtime
204
+ version_requirements: !ruby/object:Gem::Requirement
205
+ requirements:
206
+ - - "~>"
207
+ - !ruby/object:Gem::Version
208
+ version: '2.1'
195
209
  - !ruby/object:Gem::Dependency
196
210
  requirement: !ruby/object:Gem::Requirement
197
211
  requirements:
@@ -363,6 +377,7 @@ extra_rdoc_files: []
363
377
  files:
364
378
  - ".github/pull_request_template.md"
365
379
  - ".github/workflows/ci.yml"
380
+ - ".github/workflows/update-license-year.yml"
366
381
  - ".gitignore"
367
382
  - ".rubocop.yml"
368
383
  - ".simplecov"
@@ -387,8 +402,10 @@ files:
387
402
  - lib/splitclient-rb/cache/adapters/redis_adapter.rb
388
403
  - lib/splitclient-rb/cache/fetchers/segment_fetcher.rb
389
404
  - lib/splitclient-rb/cache/fetchers/split_fetcher.rb
405
+ - lib/splitclient-rb/cache/filter/filter_adapter.rb
390
406
  - lib/splitclient-rb/cache/hashers/impression_hasher.rb
391
407
  - lib/splitclient-rb/cache/observers/impression_observer.rb
408
+ - lib/splitclient-rb/cache/observers/noop_impression_observer.rb
392
409
  - lib/splitclient-rb/cache/repositories/events/memory_repository.rb
393
410
  - lib/splitclient-rb/cache/repositories/events/redis_repository.rb
394
411
  - lib/splitclient-rb/cache/repositories/events_repository.rb
@@ -400,9 +417,12 @@ files:
400
417
  - lib/splitclient-rb/cache/repositories/splits_repository.rb
401
418
  - lib/splitclient-rb/cache/routers/impression_router.rb
402
419
  - lib/splitclient-rb/cache/senders/events_sender.rb
420
+ - lib/splitclient-rb/cache/senders/impressions_adapter/memory_sender.rb
421
+ - lib/splitclient-rb/cache/senders/impressions_adapter/redis_sender.rb
403
422
  - lib/splitclient-rb/cache/senders/impressions_count_sender.rb
404
423
  - lib/splitclient-rb/cache/senders/impressions_formatter.rb
405
424
  - lib/splitclient-rb/cache/senders/impressions_sender.rb
425
+ - lib/splitclient-rb/cache/senders/impressions_sender_adapter.rb
406
426
  - lib/splitclient-rb/cache/senders/localhost_repo_cleaner.rb
407
427
  - lib/splitclient-rb/cache/stores/localhost_split_builder.rb
408
428
  - lib/splitclient-rb/cache/stores/localhost_split_store.rb
@@ -421,7 +441,10 @@ files:
421
441
  - lib/splitclient-rb/engine/back_off.rb
422
442
  - lib/splitclient-rb/engine/common/impressions_counter.rb
423
443
  - lib/splitclient-rb/engine/common/impressions_manager.rb
444
+ - lib/splitclient-rb/engine/common/noop_impressions_counter.rb
424
445
  - lib/splitclient-rb/engine/evaluator/splitter.rb
446
+ - lib/splitclient-rb/engine/impressions/noop_unique_keys_tracker.rb
447
+ - lib/splitclient-rb/engine/impressions/unique_keys_tracker.rb
425
448
  - lib/splitclient-rb/engine/matchers/all_keys_matcher.rb
426
449
  - lib/splitclient-rb/engine/matchers/between_matcher.rb
427
450
  - lib/splitclient-rb/engine/matchers/combiners.rb