prefab-cloud-ruby 0.24.3 → 0.24.4

data/lib/prefab/evaluated_keys_aggregator.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require_relative 'periodic_sync'
+
+module Prefab
+  class EvaluatedKeysAggregator
+    include Prefab::PeriodicSync
+
+    attr_reader :data
+
+    def initialize(client:, max_keys:, sync_interval:)
+      @max_keys = max_keys
+      @client = client
+      @name = 'evaluated_keys_aggregator'
+
+      @data = Concurrent::Set.new
+
+      start_periodic_sync(sync_interval)
+    end
+
+    def push(key)
+      return if @data.size >= @max_keys
+
+      @data.add(key)
+    end
+
+    private
+
+    def flush(to_ship, _)
+      @pool.post do
+        log_internal "Uploading evaluated keys for #{to_ship.size}"
+
+        keys = PrefabProto::EvaluatedKeys.new(keys: to_ship.to_a, namespace: @client.namespace)
+
+        result = @client.post('/api/v1/evaluated-keys', keys)
+
+        log_internal "Uploaded #{to_ship.size} keys: #{result.status}"
+      end
+    end
+  end
+end

data/lib/prefab/local_config_parser.rb

@@ -18,13 +18,13 @@ module Prefab
           config[key] = {
             source: file,
             match: 'default',
-            config: Prefab::Config.new(
+            config: PrefabProto::Config.new(
               config_type: :CONFIG,
               key: key,
               rows: [
-                Prefab::ConfigRow.new(values: [
-                                        Prefab::ConditionalValue.new(value: value_from(key, value))
-                                      ])
+                PrefabProto::ConfigRow.new(values: [
+                                             PrefabProto::ConditionalValue.new(value: value_from(key, value))
+                                           ])
               ]
             )
           }
@@ -37,7 +37,7 @@ module Prefab
         case raw
         when String
           if key.to_s.start_with? Prefab::LoggerClient::BASE_KEY
-            prefab_log_level_resolve = Prefab::LogLevel.resolve(raw.upcase.to_sym) || Prefab::LogLevel::NOT_SET_LOG_LEVEL
+            prefab_log_level_resolve = PrefabProto::LogLevel.resolve(raw.upcase.to_sym) || PrefabProto::LogLevel::NOT_SET_LOG_LEVEL
             { log_level: prefab_log_level_resolve }
           else
             { string: raw }
@@ -54,11 +54,11 @@ module Prefab
       def feature_flag_config(file, key, value)
         criterion = (parse_criterion(value['criterion']) if value['criterion'])
 
-        variant = Prefab::ConfigValue.new(value_from(key, value['value']))
+        variant = PrefabProto::ConfigValue.new(value_from(key, value['value']))
 
-        row = Prefab::ConfigRow.new(
+        row = PrefabProto::ConfigRow.new(
           values: [
-            Prefab::ConditionalValue.new(
+            PrefabProto::ConditionalValue.new(
               criteria: [criterion].compact,
               value: variant
             )
@@ -70,7 +70,7 @@ module Prefab
         {
           source: file,
           match: key,
-          config: Prefab::Config.new(
+          config: PrefabProto::Config.new(
             config_type: :FEATURE_FLAG,
             key: key,
             allowable_values: [variant],
@@ -80,15 +80,15 @@ module Prefab
       end
 
       def parse_criterion(criterion)
-        Prefab::Criterion.new(operator: criterion['operator'],
-                              property_name: criterion['property'],
-                              value_to_match: parse_value_to_match(criterion['values']))
+        PrefabProto::Criterion.new(operator: criterion['operator'],
+                                   property_name: criterion['property'],
+                                   value_to_match: parse_value_to_match(criterion['values']))
       end
 
       def parse_value_to_match(values)
         raise "Can't handle #{values}" unless values.instance_of?(Array)
 
-        Prefab::ConfigValue.new(string_list: Prefab::StringList.new(values: values))
+        PrefabProto::ConfigValue.new(string_list: PrefabProto::StringList.new(values: values))
       end
     end
   end

data/lib/prefab/log_path_aggregator.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+require_relative 'periodic_sync'
+
+module Prefab
+  class LogPathAggregator
+    include Prefab::PeriodicSync
+
+    INCREMENT = ->(count) { (count || 0) + 1 }
+
+    SEVERITY_KEY = {
+      ::Logger::DEBUG => 'debugs',
+      ::Logger::INFO => 'infos',
+      ::Logger::WARN => 'warns',
+      ::Logger::ERROR => 'errors',
+      ::Logger::FATAL => 'fatals'
+    }.freeze
+
+    attr_reader :data
+
+    def initialize(client:, max_paths:, sync_interval:)
+      @max_paths = max_paths
+      @client = client
+      @name = 'log_path_aggregator'
+
+      @data = Concurrent::Map.new
+
+      start_periodic_sync(sync_interval)
+    end
+
+    def push(path, severity)
+      return if @data.size >= @max_paths
+
+      @data.compute([path, severity], &INCREMENT)
+    end
+
+    private
+
+    def flush(to_ship, start_at_was)
+      @pool.post do
+        log_internal "Uploading stats for #{to_ship.size} paths"
+
+        aggregate = Hash.new { |h, k| h[k] = PrefabProto::Logger.new }
+
+        to_ship.each do |(path, severity), count|
+          aggregate[path][SEVERITY_KEY[severity]] = count
+          aggregate[path]['logger_name'] = path
+        end
+
+        loggers = PrefabProto::Loggers.new(
+          loggers: aggregate.values,
+          start_at: start_at_was,
+          end_at: Prefab::TimeHelpers.now_in_ms,
+          instance_hash: @client.instance_hash,
+          namespace: @client.namespace
+        )
+
+        result = @client.post('/api/v1/known-loggers', loggers)
+
+        log_internal "Uploaded #{to_ship.size} paths: #{result.status}"
+      end
+    end
+  end
+end

data/lib/prefab/logger_client.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require 'prefab/log_path_collector'
-
 module Prefab
   class LoggerClient < ::Logger
     SEP = '.'
@@ -10,30 +8,30 @@ module Prefab
     INTERNAL_PREFIX = 'cloud.prefab.client'
 
     LOG_LEVEL_LOOKUPS = {
-      Prefab::LogLevel::NOT_SET_LOG_LEVEL => ::Logger::DEBUG,
-      Prefab::LogLevel::TRACE => ::Logger::DEBUG,
-      Prefab::LogLevel::DEBUG => ::Logger::DEBUG,
-      Prefab::LogLevel::INFO => ::Logger::INFO,
-      Prefab::LogLevel::WARN => ::Logger::WARN,
-      Prefab::LogLevel::ERROR => ::Logger::ERROR,
-      Prefab::LogLevel::FATAL => ::Logger::FATAL
+      PrefabProto::LogLevel::NOT_SET_LOG_LEVEL => ::Logger::DEBUG,
+      PrefabProto::LogLevel::TRACE => ::Logger::DEBUG,
+      PrefabProto::LogLevel::DEBUG => ::Logger::DEBUG,
+      PrefabProto::LogLevel::INFO => ::Logger::INFO,
+      PrefabProto::LogLevel::WARN => ::Logger::WARN,
+      PrefabProto::LogLevel::ERROR => ::Logger::ERROR,
+      PrefabProto::LogLevel::FATAL => ::Logger::FATAL
     }
 
-    def initialize(logdev, log_path_collector: nil, formatter: nil, prefix: nil)
+    def initialize(logdev, log_path_aggregator: nil, formatter: nil, prefix: nil)
       super(logdev)
       self.formatter = formatter
       @config_client = BootstrappingConfigClient.new
       @silences = Concurrent::Map.new(initial_capacity: 2)
       @prefix = "#{prefix}#{prefix && '.'}"
 
-      @log_path_collector = log_path_collector
+      @log_path_aggregator = log_path_aggregator
     end
 
     def add(severity, message = nil, progname = nil, loc, &block)
       path_loc = get_loc_path(loc)
       path = @prefix + path_loc
 
-      @log_path_collector&.push(path_loc, severity)
+      @log_path_aggregator&.push(path_loc, severity)
 
       log(message, path, progname, severity, &block)
     end
@@ -149,7 +147,7 @@ module Prefab
         closest_log_level_match = @config_client.get(BASE_KEY, :WARN)
       end
 
-      closest_log_level_match_int = Prefab::LogLevel.resolve(closest_log_level_match)
+      closest_log_level_match_int = PrefabProto::LogLevel.resolve(closest_log_level_match)
       LOG_LEVEL_LOOKUPS[closest_log_level_match_int]
     end
 

data/lib/prefab/options.rb

@@ -17,6 +17,7 @@ module Prefab
     attr_reader :prefab_config_classpath_dir
     attr_reader :prefab_envs
     attr_reader :collect_sync_interval
+    attr_reader :shape_sync_interval
 
     DEFAULT_LOG_FORMATTER = proc { |severity, datetime, progname, msg|
       "#{severity.ljust(5)} #{datetime}:#{' ' if progname}#{progname} #{msg}\n"
@@ -47,6 +48,9 @@ module Prefab
     end
 
     DEFAULT_MAX_PATHS = 1_000
+    DEFAULT_MAX_CONTEXT_KEYS = 100_000
+    DEFAULT_MAX_KEYS = 100_000
+    DEFAULT_MAX_EVALS = 100_000
 
     private def init(
       api_key: ENV['PREFAB_API_KEY'],
@@ -68,7 +72,14 @@ module Prefab
       prefab_envs: ENV['PREFAB_ENVS'].nil? ? [] : ENV['PREFAB_ENVS'].split(','),
       collect_logs: true,
       collect_max_paths: DEFAULT_MAX_PATHS,
-      collect_sync_interval: nil
+      collect_sync_interval: nil,
+      collect_shapes: true,
+      collect_max_shapes: DEFAULT_MAX_CONTEXT_KEYS,
+      collect_keys: false,
+      collect_max_keys: DEFAULT_MAX_KEYS,
+      collect_evaluations: false,
+      collect_max_evaluations: DEFAULT_MAX_EVALS,
+      shape_sync_interval: nil
     )
       @api_key = api_key
       @logdev = logdev
@@ -88,6 +99,13 @@ module Prefab
       @collect_logs = collect_logs
       @collect_max_paths = collect_max_paths
       @collect_sync_interval = collect_sync_interval
+      @collect_shapes = collect_shapes
+      @collect_max_shapes = collect_max_shapes
+      @collect_keys = collect_keys
+      @collect_max_keys = collect_max_keys
+      @shape_sync_interval = shape_sync_interval
+      @collect_evaluations = collect_evaluations
+      @collect_max_evaluations = collect_max_evaluations
     end
 
     def initialize(options = {})
@@ -104,6 +122,24 @@ module Prefab
       @collect_max_paths
     end
 
+    def collect_max_shapes
+      return 0 if !@collect_shapes || local_only?
+
+      @collect_max_shapes
+    end
+
+    def collect_max_keys
+      return 0 if !@collect_keys || local_only?
+
+      @collect_max_keys
+    end
+
+    def collect_max_evaluations
+      return 0 if !@collect_evaluations || local_only?
+
+      @collect_max_evaluations
+    end
+
     # https://api.prefab.cloud -> https://api-prefab-cloud.global.ssl.fastly.net
     def url_for_api_cdn
       ENV['PREFAB_CDN_URL'] || "#{@prefab_api_url.gsub(/\./, '-')}.global.ssl.fastly.net"

data/lib/prefab/periodic_sync.rb

@@ -0,0 +1,51 @@
+module Prefab
+  module PeriodicSync
+    def sync
+      return if @data.size.zero?
+
+      log_internal "Syncing #{@data.size} items"
+
+      start_at_was = @start_at
+      @start_at = Prefab::TimeHelpers.now_in_ms
+
+      flush(prepare_data, start_at_was)
+    end
+
+    def prepare_data
+      to_ship = @data.dup
+      @data.clear
+      to_ship
+    end
+
+    def start_periodic_sync(sync_interval)
+      @start_at = Prefab::TimeHelpers.now_in_ms
+
+      @sync_interval = if sync_interval.is_a?(Numeric)
+                         proc { sync_interval }
+                       else
+                         sync_interval || ExponentialBackoff.new(initial_delay: 8, max_delay: 60 * 10)
+                       end
+
+      @pool = Concurrent::ThreadPoolExecutor.new(
+        fallback_policy: :discard,
+        max_queue: 5,
+        max_threads: 4,
+        min_threads: 1,
+        name: @name
+      )
+
+      Thread.new do
+        log_internal "Initialized #{@name} instance_hash=#{@client.instance_hash}"
+
+        loop do
+          sleep @sync_interval.call
+          sync
+        end
+      end
+    end
+
+    def log_internal(message)
+      @client.log.log_internal message, @name, nil, ::Logger::DEBUG
+    end
+  end
+end

data/lib/prefab/time_helpers.rb

@@ -0,0 +1,7 @@
+module Prefab
+  module TimeHelpers
+    def self.now_in_ms
+      ::Time.now.utc.to_i * 1000
+    end
+  end
+end

data/lib/prefab-cloud-ruby.rb

@@ -11,6 +11,7 @@ require 'openssl'
 require 'openssl'
 require 'ld-eventsource'
 require 'prefab_pb'
+require 'prefab/time_helpers'
 require 'prefab/error'
 require 'prefab/exponential_backoff'
 require 'prefab/errors/initialization_timeout_error'
@@ -18,21 +19,27 @@ require 'prefab/errors/invalid_api_key_error'
 require 'prefab/errors/missing_default_error'
 require 'prefab/options'
 require 'prefab/internal_logger'
+require 'prefab/log_path_aggregator'
+require 'prefab/context_shape_aggregator'
+require 'prefab/evaluated_configs_aggregator'
+require 'prefab/evaluated_keys_aggregator'
 require 'prefab/sse_logger'
 require 'prefab/weighted_value_resolver'
+require 'prefab/config_value_wrapper'
 require 'prefab/config_value_unwrapper'
 require 'prefab/criteria_evaluator'
 require 'prefab/config_loader'
+require 'prefab/context_shape'
 require 'prefab/local_config_parser'
 require 'prefab/yaml_config_parser'
 require 'prefab/resolved_config_presenter'
 require 'prefab/config_resolver'
 require 'prefab/http_connection'
 require 'prefab/context'
+require 'prefab/logger_client'
 require 'prefab/client'
 require 'prefab/config_client'
 require 'prefab/feature_flag_client'
-require 'prefab/logger_client'
 require 'prefab/noop_cache'
 require 'prefab/noop_stats'
 require 'prefab/murmer3'