prefab-cloud-ruby 0.24.3 → 0.24.5

Sign up to get free protection for your applications and to get access to all the features.
Files changed (45) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/ruby.yml +1 -1
  3. data/.rubocop.yml +13 -0
  4. data/CHANGELOG.md +78 -0
  5. data/Gemfile.lock +4 -4
  6. data/VERSION +1 -1
  7. data/bin/console +21 -0
  8. data/compile_protos.sh +6 -0
  9. data/lib/prefab/client.rb +25 -4
  10. data/lib/prefab/config_client.rb +17 -6
  11. data/lib/prefab/config_loader.rb +1 -1
  12. data/lib/prefab/config_resolver.rb +2 -4
  13. data/lib/prefab/config_value_wrapper.rb +18 -0
  14. data/lib/prefab/context.rb +22 -2
  15. data/lib/prefab/context_shape.rb +20 -0
  16. data/lib/prefab/context_shape_aggregator.rb +63 -0
  17. data/lib/prefab/criteria_evaluator.rb +61 -41
  18. data/lib/prefab/evaluated_configs_aggregator.rb +60 -0
  19. data/lib/prefab/evaluated_keys_aggregator.rb +41 -0
  20. data/lib/prefab/http_connection.rb +5 -1
  21. data/lib/prefab/local_config_parser.rb +13 -13
  22. data/lib/prefab/log_path_aggregator.rb +64 -0
  23. data/lib/prefab/logger_client.rb +11 -13
  24. data/lib/prefab/options.rb +37 -1
  25. data/lib/prefab/periodic_sync.rb +51 -0
  26. data/lib/prefab/time_helpers.rb +7 -0
  27. data/lib/prefab-cloud-ruby.rb +9 -2
  28. data/lib/prefab_pb.rb +33 -220
  29. data/prefab-cloud-ruby.gemspec +21 -5
  30. data/test/test_config_loader.rb +15 -15
  31. data/test/test_config_resolver.rb +102 -102
  32. data/test/test_config_value_unwrapper.rb +13 -13
  33. data/test/test_context.rb +42 -0
  34. data/test/test_context_shape.rb +51 -0
  35. data/test/test_context_shape_aggregator.rb +137 -0
  36. data/test/test_criteria_evaluator.rb +253 -150
  37. data/test/test_evaluated_configs_aggregator.rb +254 -0
  38. data/test/test_evaluated_keys_aggregator.rb +54 -0
  39. data/test/test_helper.rb +34 -2
  40. data/test/test_log_path_aggregator.rb +57 -0
  41. data/test/test_logger.rb +33 -33
  42. data/test/test_weighted_value_resolver.rb +2 -2
  43. metadata +21 -5
  44. data/lib/prefab/log_path_collector.rb +0 -102
  45. data/test/test_log_path_collector.rb +0 -58
@@ -0,0 +1,41 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative 'periodic_sync'
4
+
5
+ module Prefab
6
+ class EvaluatedKeysAggregator
7
+ include Prefab::PeriodicSync
8
+
9
+ attr_reader :data
10
+
11
+ def initialize(client:, max_keys:, sync_interval:)
12
+ @max_keys = max_keys
13
+ @client = client
14
+ @name = 'evaluated_keys_aggregator'
15
+
16
+ @data = Concurrent::Set.new
17
+
18
+ start_periodic_sync(sync_interval)
19
+ end
20
+
21
+ def push(key)
22
+ return if @data.size >= @max_keys
23
+
24
+ @data.add(key)
25
+ end
26
+
27
+ private
28
+
29
+ def flush(to_ship, _)
30
+ @pool.post do
31
+ log_internal "Uploading evaluated keys for #{to_ship.size}"
32
+
33
+ keys = PrefabProto::EvaluatedKeys.new(keys: to_ship.to_a, namespace: @client.namespace)
34
+
35
+ result = @client.post('/api/v1/evaluated-keys', keys)
36
+
37
+ log_internal "Uploaded #{to_ship.size} keys: #{result.status}"
38
+ end
39
+ end
40
+ end
41
+ end
@@ -3,7 +3,11 @@
3
3
  module Prefab
4
4
  class HttpConnection
5
5
  AUTH_USER = 'authuser'
6
- PROTO_HEADERS = { 'Content-Type' => 'application/x-protobuf', 'Accept' => 'application/x-protobuf' }.freeze
6
+ PROTO_HEADERS = {
7
+ 'Content-Type' => 'application/x-protobuf',
8
+ 'Accept' => 'application/x-protobuf',
9
+ 'X-PrefabCloud-Client-Version' => "prefab-cloud-ruby-#{Prefab::VERSION}"
10
+ }.freeze
7
11
 
8
12
  def initialize(api_root, api_key)
9
13
  @api_root = api_root
@@ -18,13 +18,13 @@ module Prefab
18
18
  config[key] = {
19
19
  source: file,
20
20
  match: 'default',
21
- config: Prefab::Config.new(
21
+ config: PrefabProto::Config.new(
22
22
  config_type: :CONFIG,
23
23
  key: key,
24
24
  rows: [
25
- Prefab::ConfigRow.new(values: [
26
- Prefab::ConditionalValue.new(value: value_from(key, value))
27
- ])
25
+ PrefabProto::ConfigRow.new(values: [
26
+ PrefabProto::ConditionalValue.new(value: value_from(key, value))
27
+ ])
28
28
  ]
29
29
  )
30
30
  }
@@ -37,7 +37,7 @@ module Prefab
37
37
  case raw
38
38
  when String
39
39
  if key.to_s.start_with? Prefab::LoggerClient::BASE_KEY
40
- prefab_log_level_resolve = Prefab::LogLevel.resolve(raw.upcase.to_sym) || Prefab::LogLevel::NOT_SET_LOG_LEVEL
40
+ prefab_log_level_resolve = PrefabProto::LogLevel.resolve(raw.upcase.to_sym) || PrefabProto::LogLevel::NOT_SET_LOG_LEVEL
41
41
  { log_level: prefab_log_level_resolve }
42
42
  else
43
43
  { string: raw }
@@ -54,11 +54,11 @@ module Prefab
54
54
  def feature_flag_config(file, key, value)
55
55
  criterion = (parse_criterion(value['criterion']) if value['criterion'])
56
56
 
57
- variant = Prefab::ConfigValue.new(value_from(key, value['value']))
57
+ variant = PrefabProto::ConfigValue.new(value_from(key, value['value']))
58
58
 
59
- row = Prefab::ConfigRow.new(
59
+ row = PrefabProto::ConfigRow.new(
60
60
  values: [
61
- Prefab::ConditionalValue.new(
61
+ PrefabProto::ConditionalValue.new(
62
62
  criteria: [criterion].compact,
63
63
  value: variant
64
64
  )
@@ -70,7 +70,7 @@ module Prefab
70
70
  {
71
71
  source: file,
72
72
  match: key,
73
- config: Prefab::Config.new(
73
+ config: PrefabProto::Config.new(
74
74
  config_type: :FEATURE_FLAG,
75
75
  key: key,
76
76
  allowable_values: [variant],
@@ -80,15 +80,15 @@ module Prefab
80
80
  end
81
81
 
82
82
  def parse_criterion(criterion)
83
- Prefab::Criterion.new(operator: criterion['operator'],
84
- property_name: criterion['property'],
85
- value_to_match: parse_value_to_match(criterion['values']))
83
+ PrefabProto::Criterion.new(operator: criterion['operator'],
84
+ property_name: criterion['property'],
85
+ value_to_match: parse_value_to_match(criterion['values']))
86
86
  end
87
87
 
88
88
  def parse_value_to_match(values)
89
89
  raise "Can't handle #{values}" unless values.instance_of?(Array)
90
90
 
91
- Prefab::ConfigValue.new(string_list: Prefab::StringList.new(values: values))
91
+ PrefabProto::ConfigValue.new(string_list: PrefabProto::StringList.new(values: values))
92
92
  end
93
93
  end
94
94
  end
@@ -0,0 +1,64 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative 'periodic_sync'
4
+
5
+ module Prefab
6
+ class LogPathAggregator
7
+ include Prefab::PeriodicSync
8
+
9
+ INCREMENT = ->(count) { (count || 0) + 1 }
10
+
11
+ SEVERITY_KEY = {
12
+ ::Logger::DEBUG => 'debugs',
13
+ ::Logger::INFO => 'infos',
14
+ ::Logger::WARN => 'warns',
15
+ ::Logger::ERROR => 'errors',
16
+ ::Logger::FATAL => 'fatals'
17
+ }.freeze
18
+
19
+ attr_reader :data
20
+
21
+ def initialize(client:, max_paths:, sync_interval:)
22
+ @max_paths = max_paths
23
+ @client = client
24
+ @name = 'log_path_aggregator'
25
+
26
+ @data = Concurrent::Map.new
27
+
28
+ start_periodic_sync(sync_interval)
29
+ end
30
+
31
+ def push(path, severity)
32
+ return if @data.size >= @max_paths
33
+
34
+ @data.compute([path, severity], &INCREMENT)
35
+ end
36
+
37
+ private
38
+
39
+ def flush(to_ship, start_at_was)
40
+ @pool.post do
41
+ log_internal "Uploading stats for #{to_ship.size} paths"
42
+
43
+ aggregate = Hash.new { |h, k| h[k] = PrefabProto::Logger.new }
44
+
45
+ to_ship.each do |(path, severity), count|
46
+ aggregate[path][SEVERITY_KEY[severity]] = count
47
+ aggregate[path]['logger_name'] = path
48
+ end
49
+
50
+ loggers = PrefabProto::Loggers.new(
51
+ loggers: aggregate.values,
52
+ start_at: start_at_was,
53
+ end_at: Prefab::TimeHelpers.now_in_ms,
54
+ instance_hash: @client.instance_hash,
55
+ namespace: @client.namespace
56
+ )
57
+
58
+ result = @client.post('/api/v1/known-loggers', loggers)
59
+
60
+ log_internal "Uploaded #{to_ship.size} paths: #{result.status}"
61
+ end
62
+ end
63
+ end
64
+ end
@@ -1,7 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'prefab/log_path_collector'
4
-
5
3
  module Prefab
6
4
  class LoggerClient < ::Logger
7
5
  SEP = '.'
@@ -10,30 +8,30 @@ module Prefab
10
8
  INTERNAL_PREFIX = 'cloud.prefab.client'
11
9
 
12
10
  LOG_LEVEL_LOOKUPS = {
13
- Prefab::LogLevel::NOT_SET_LOG_LEVEL => ::Logger::DEBUG,
14
- Prefab::LogLevel::TRACE => ::Logger::DEBUG,
15
- Prefab::LogLevel::DEBUG => ::Logger::DEBUG,
16
- Prefab::LogLevel::INFO => ::Logger::INFO,
17
- Prefab::LogLevel::WARN => ::Logger::WARN,
18
- Prefab::LogLevel::ERROR => ::Logger::ERROR,
19
- Prefab::LogLevel::FATAL => ::Logger::FATAL
11
+ PrefabProto::LogLevel::NOT_SET_LOG_LEVEL => ::Logger::DEBUG,
12
+ PrefabProto::LogLevel::TRACE => ::Logger::DEBUG,
13
+ PrefabProto::LogLevel::DEBUG => ::Logger::DEBUG,
14
+ PrefabProto::LogLevel::INFO => ::Logger::INFO,
15
+ PrefabProto::LogLevel::WARN => ::Logger::WARN,
16
+ PrefabProto::LogLevel::ERROR => ::Logger::ERROR,
17
+ PrefabProto::LogLevel::FATAL => ::Logger::FATAL
20
18
  }
21
19
 
22
- def initialize(logdev, log_path_collector: nil, formatter: nil, prefix: nil)
20
+ def initialize(logdev, log_path_aggregator: nil, formatter: nil, prefix: nil)
23
21
  super(logdev)
24
22
  self.formatter = formatter
25
23
  @config_client = BootstrappingConfigClient.new
26
24
  @silences = Concurrent::Map.new(initial_capacity: 2)
27
25
  @prefix = "#{prefix}#{prefix && '.'}"
28
26
 
29
- @log_path_collector = log_path_collector
27
+ @log_path_aggregator = log_path_aggregator
30
28
  end
31
29
 
32
30
  def add(severity, message = nil, progname = nil, loc, &block)
33
31
  path_loc = get_loc_path(loc)
34
32
  path = @prefix + path_loc
35
33
 
36
- @log_path_collector&.push(path_loc, severity)
34
+ @log_path_aggregator&.push(path_loc, severity)
37
35
 
38
36
  log(message, path, progname, severity, &block)
39
37
  end
@@ -149,7 +147,7 @@ module Prefab
149
147
  closest_log_level_match = @config_client.get(BASE_KEY, :WARN)
150
148
  end
151
149
 
152
- closest_log_level_match_int = Prefab::LogLevel.resolve(closest_log_level_match)
150
+ closest_log_level_match_int = PrefabProto::LogLevel.resolve(closest_log_level_match)
153
151
  LOG_LEVEL_LOOKUPS[closest_log_level_match_int]
154
152
  end
155
153
 
@@ -17,6 +17,7 @@ module Prefab
17
17
  attr_reader :prefab_config_classpath_dir
18
18
  attr_reader :prefab_envs
19
19
  attr_reader :collect_sync_interval
20
+ attr_reader :shape_sync_interval
20
21
 
21
22
  DEFAULT_LOG_FORMATTER = proc { |severity, datetime, progname, msg|
22
23
  "#{severity.ljust(5)} #{datetime}:#{' ' if progname}#{progname} #{msg}\n"
@@ -47,6 +48,9 @@ module Prefab
47
48
  end
48
49
 
49
50
  DEFAULT_MAX_PATHS = 1_000
51
+ DEFAULT_MAX_CONTEXT_KEYS = 100_000
52
+ DEFAULT_MAX_KEYS = 100_000
53
+ DEFAULT_MAX_EVALS = 100_000
50
54
 
51
55
  private def init(
52
56
  api_key: ENV['PREFAB_API_KEY'],
@@ -68,7 +72,14 @@ module Prefab
68
72
  prefab_envs: ENV['PREFAB_ENVS'].nil? ? [] : ENV['PREFAB_ENVS'].split(','),
69
73
  collect_logs: true,
70
74
  collect_max_paths: DEFAULT_MAX_PATHS,
71
- collect_sync_interval: nil
75
+ collect_sync_interval: nil,
76
+ collect_shapes: true,
77
+ collect_max_shapes: DEFAULT_MAX_CONTEXT_KEYS,
78
+ collect_keys: false,
79
+ collect_max_keys: DEFAULT_MAX_KEYS,
80
+ collect_evaluations: false,
81
+ collect_max_evaluations: DEFAULT_MAX_EVALS,
82
+ shape_sync_interval: nil
72
83
  )
73
84
  @api_key = api_key
74
85
  @logdev = logdev
@@ -88,6 +99,13 @@ module Prefab
88
99
  @collect_logs = collect_logs
89
100
  @collect_max_paths = collect_max_paths
90
101
  @collect_sync_interval = collect_sync_interval
102
+ @collect_shapes = collect_shapes
103
+ @collect_max_shapes = collect_max_shapes
104
+ @collect_keys = collect_keys
105
+ @collect_max_keys = collect_max_keys
106
+ @shape_sync_interval = shape_sync_interval
107
+ @collect_evaluations = collect_evaluations
108
+ @collect_max_evaluations = collect_max_evaluations
91
109
  end
92
110
 
93
111
  def initialize(options = {})
@@ -104,6 +122,24 @@ module Prefab
104
122
  @collect_max_paths
105
123
  end
106
124
 
125
+ def collect_max_shapes
126
+ return 0 if !@collect_shapes || local_only?
127
+
128
+ @collect_max_shapes
129
+ end
130
+
131
+ def collect_max_keys
132
+ return 0 if !@collect_keys || local_only?
133
+
134
+ @collect_max_keys
135
+ end
136
+
137
+ def collect_max_evaluations
138
+ return 0 if !@collect_evaluations || local_only?
139
+
140
+ @collect_max_evaluations
141
+ end
142
+
107
143
  # https://api.prefab.cloud -> https://api-prefab-cloud.global.ssl.fastly.net
108
144
  def url_for_api_cdn
109
145
  ENV['PREFAB_CDN_URL'] || "#{@prefab_api_url.gsub(/\./, '-')}.global.ssl.fastly.net"
@@ -0,0 +1,51 @@
1
+ module Prefab
2
+ module PeriodicSync
3
+ def sync
4
+ return if @data.size.zero?
5
+
6
+ log_internal "Syncing #{@data.size} items"
7
+
8
+ start_at_was = @start_at
9
+ @start_at = Prefab::TimeHelpers.now_in_ms
10
+
11
+ flush(prepare_data, start_at_was)
12
+ end
13
+
14
+ def prepare_data
15
+ to_ship = @data.dup
16
+ @data.clear
17
+ to_ship
18
+ end
19
+
20
+ def start_periodic_sync(sync_interval)
21
+ @start_at = Prefab::TimeHelpers.now_in_ms
22
+
23
+ @sync_interval = if sync_interval.is_a?(Numeric)
24
+ proc { sync_interval }
25
+ else
26
+ sync_interval || ExponentialBackoff.new(initial_delay: 8, max_delay: 60 * 10)
27
+ end
28
+
29
+ @pool = Concurrent::ThreadPoolExecutor.new(
30
+ fallback_policy: :discard,
31
+ max_queue: 5,
32
+ max_threads: 4,
33
+ min_threads: 1,
34
+ name: @name
35
+ )
36
+
37
+ Thread.new do
38
+ log_internal "Initialized #{@name} instance_hash=#{@client.instance_hash}"
39
+
40
+ loop do
41
+ sleep @sync_interval.call
42
+ sync
43
+ end
44
+ end
45
+ end
46
+
47
+ def log_internal(message)
48
+ @client.log.log_internal message, @name, nil, ::Logger::DEBUG
49
+ end
50
+ end
51
+ end
@@ -0,0 +1,7 @@
1
+ module Prefab
2
+ module TimeHelpers
3
+ def self.now_in_ms
4
+ ::Time.now.utc.to_i * 1000
5
+ end
6
+ end
7
+ end
@@ -2,15 +2,16 @@
2
2
 
3
3
  module Prefab
4
4
  NO_DEFAULT_PROVIDED = :no_default_provided
5
+ VERSION = File.read(File.dirname(__FILE__) + '/../VERSION').strip
5
6
  end
6
7
 
7
8
  require 'concurrent/atomics'
8
9
  require 'concurrent'
9
10
  require 'faraday'
10
11
  require 'openssl'
11
- require 'openssl'
12
12
  require 'ld-eventsource'
13
13
  require 'prefab_pb'
14
+ require 'prefab/time_helpers'
14
15
  require 'prefab/error'
15
16
  require 'prefab/exponential_backoff'
16
17
  require 'prefab/errors/initialization_timeout_error'
@@ -18,21 +19,27 @@ require 'prefab/errors/invalid_api_key_error'
18
19
  require 'prefab/errors/missing_default_error'
19
20
  require 'prefab/options'
20
21
  require 'prefab/internal_logger'
22
+ require 'prefab/log_path_aggregator'
23
+ require 'prefab/context_shape_aggregator'
24
+ require 'prefab/evaluated_configs_aggregator'
25
+ require 'prefab/evaluated_keys_aggregator'
21
26
  require 'prefab/sse_logger'
22
27
  require 'prefab/weighted_value_resolver'
28
+ require 'prefab/config_value_wrapper'
23
29
  require 'prefab/config_value_unwrapper'
24
30
  require 'prefab/criteria_evaluator'
25
31
  require 'prefab/config_loader'
32
+ require 'prefab/context_shape'
26
33
  require 'prefab/local_config_parser'
27
34
  require 'prefab/yaml_config_parser'
28
35
  require 'prefab/resolved_config_presenter'
29
36
  require 'prefab/config_resolver'
30
37
  require 'prefab/http_connection'
31
38
  require 'prefab/context'
39
+ require 'prefab/logger_client'
32
40
  require 'prefab/client'
33
41
  require 'prefab/config_client'
34
42
  require 'prefab/feature_flag_client'
35
- require 'prefab/logger_client'
36
43
  require 'prefab/noop_cache'
37
44
  require 'prefab/noop_stats'
38
45
  require 'prefab/murmer3'