sdk-reforge 1.9.0

data/lib/reforge/config_client.rb

@@ -0,0 +1,275 @@
+# frozen_string_literal: true
+
+module Reforge
+  class ConfigClient
+    LOG = Reforge::InternalLogger.new(self)
+    DEFAULT_CHECKPOINT_FREQ_SEC = 60
+    STALE_CACHE_WARN_HOURS = 5
+
+    def initialize(base_client, timeout)
+      @base_client = base_client
+      @options = base_client.options
+      LOG.debug 'Initialize ConfigClient'
+      @timeout = timeout
+
+      @checkpoint_freq_secs = DEFAULT_CHECKPOINT_FREQ_SEC
+
+      @config_loader = Reforge::ConfigLoader.new(@base_client)
+      @config_resolver = Reforge::ConfigResolver.new(@base_client, @config_loader)
+
+      @initialization_lock = Concurrent::CountDownLatch.new(1)
+
+      if @options.local_only?
+        finish_init!(:local_only, nil)
+      elsif @options.datafile?
+        load_json_file(@options.datafile)
+      else
+        load_checkpoint
+        start_checkpointing_thread
+        start_streaming
+      end
+    end
+
+    def stop
+      @sse_config_client&.close
+    end
+
+    def start_streaming
+      Thread.new do
+        # wait for the config loader to have a highwater mark before we connect the SSE
+        loop do
+          break if @config_loader.highwater_mark > 0
+
+          sleep 0.1
+        end
+
+        stream_lock = Concurrent::ReadWriteLock.new
+        @sse_config_client = Reforge::SSEConfigClient.new(@options, @config_loader)
+
+        @sse_config_client.start do |configs, _event, source|
+          stream_lock.with_write_lock do
+            load_configs(configs, source)
+          end
+        end
+      end
+    end
+
+    def to_s
+      @config_resolver.to_s
+    end
+
+    def resolver
+      @config_resolver
+    end
+
+    def self.value_to_delta(key, config_value, namespace = nil)
+      PrefabProto::Config.new(key: [namespace, key].compact.join(':'),
+                              rows: [PrefabProto::ConfigRow.new(value: config_value)])
+    end
+
+    def get(key, default = NO_DEFAULT_PROVIDED, properties = NO_DEFAULT_PROVIDED)
+      context = @config_resolver.make_context(properties)
+
+      if !context.blank? && @base_client.example_contexts_aggregator
+        @base_client.example_contexts_aggregator.record(context)
+      end
+
+      evaluation = _get(key, properties)
+
+      @base_client.context_shape_aggregator&.push(context)
+
+      if evaluation
+        evaluation.report_and_return(@base_client.evaluation_summary_aggregator)
+      else
+        handle_default(key, default)
+      end
+    end
+
+    def initialized?
+      @initialization_lock.count <= 0
+    end
+
+    private
+
+    def raw(key)
+      @config_resolver.raw(key)
+    end
+
+    def handle_default(key, default)
+      return default if default != NO_DEFAULT_PROVIDED
+
+      raise Reforge::Errors::MissingDefaultError, key if @options.on_no_default == Reforge::Options::ON_NO_DEFAULT::RAISE
+
+      nil
+    end
+
+    def _get(key, properties)
+      # wait timeout sec for the initialization to be complete
+      success = @initialization_lock.wait(@options.initialization_timeout_sec)
+      if !success
+        unless @options.on_init_failure == Reforge::Options::ON_INITIALIZATION_FAILURE::RETURN
+          raise Reforge::Errors::InitializationTimeoutError.new(@options.initialization_timeout_sec, key)
+        end
+
+        LOG.warn("Couldn't Initialize In #{@options.initialization_timeout_sec}. Key #{key}. Returning what we have")
+      end
+
+      @config_resolver.get key, properties
+    end
+
+    def load_checkpoint
+      success = load_source_checkpoint(:start_at_id => @config_loader.highwater_mark)
+      return if success
+
+      success = load_cache
+      return if success
+
+      LOG.warn 'No success loading checkpoints'
+    end
+
+    def load_source_checkpoint(start_at_id: 0)
+      @options.config_sources.each do |source|
+        conn = Reforge::CachingHttpConnection.new("#{source}/api/v2/configs/#{start_at_id}", @base_client.sdk_key)
+        result = load_url(conn, :remote_api)
+        return true if result
+      end
+
+      return false
+    end
+
+    def load_url(conn, source)
+      resp = conn.get('')
+      if resp.status == 200
+        configs = PrefabProto::Configs.decode(resp.body)
+        load_configs(configs, source)
+        cache_configs(configs)
+        true
+      else
+        LOG.info "Checkpoint #{source} [#{conn.uri}] failed to load. Response #{resp.status}"
+        false
+      end
+    rescue Faraday::ConnectionFailed => e
+      if !initialized?
+        LOG.warn "Connection Fail loading #{source} [#{conn.uri}] checkpoint."
+      else
+        LOG.debug "Connection Fail loading #{source} [#{conn.uri}] checkpoint."
+      end
+      false
+    rescue StandardError => e
+      LOG.warn "Unexpected #{source} [#{conn.uri}] problem loading checkpoint #{e} #{conn}"
+      LOG.debug e.backtrace
+      false
+    end
+
+    def load_configs(configs, source)
+      project_id = configs.config_service_pointer.project_id
+      project_env_id = configs.config_service_pointer.project_env_id
+      @config_resolver.project_env_id = project_env_id
+      starting_highwater_mark = @config_loader.highwater_mark
+
+      default_contexts = configs.default_context&.contexts&.map do |context|
+        [
+          context.type,
+          context.values.keys.map do |k|
+            [k, Reforge::ConfigValueUnwrapper.new(context.values[k], @config_resolver).unwrap]
+          end.to_h
+        ]
+      end.to_h
+
+      Reforge::Context.default_context = default_contexts || {}
+
+      configs.configs.each do |config|
+        @config_loader.set(config, source)
+      end
+      if @config_loader.highwater_mark > starting_highwater_mark
+        LOG.debug("Found new checkpoint with highwater id #{@config_loader.highwater_mark} from #{source} in project #{project_id} environment: #{project_env_id} and namespace: '#{@namespace}'")
+      else
+        LOG.debug("Checkpoint with highwater id #{@config_loader.highwater_mark} from #{source}. No changes.")
+      end
+      @config_resolver.update
+      finish_init!(source, project_id)
+    end
+
+    def cache_path
+      return @cache_path unless @cache_path.nil?
+      @cache_path ||= calc_cache_path
+      FileUtils.mkdir_p(File.dirname(@cache_path))
+      @cache_path
+    end
+
+    def calc_cache_path
+      file_name = "prefab.cache.#{@base_client.options.sdk_key_id}.json"
+      dir = ENV.fetch('XDG_CACHE_HOME', File.join(Dir.home, '.cache'))
+      File.join(dir, file_name)
+    end
+
+    def cache_configs(configs)
+      return unless @options.use_local_cache && !@options.is_fork
+      File.open(cache_path, "w") do |f|
+        f.flock(File::LOCK_EX)
+        f.write(PrefabProto::Configs.encode_json(configs))
+      end
+      LOG.debug "Cached configs to #{cache_path}"
+    rescue => e
+      LOG.debug "Failed to cache configs to #{cache_path} #{e}"
+    end
+
+    def load_cache
+      return false unless @options.use_local_cache
+      File.open(cache_path) do |f|
+        f.flock(File::LOCK_SH)
+        configs = PrefabProto::Configs.decode_json(f.read)
+        load_configs(configs, :cache)
+
+        hours_old = ((Time.now - File.mtime(f)) / 60 / 60).round(2)
+        if hours_old > STALE_CACHE_WARN_HOURS
+          LOG.info "Stale Cache Load: #{hours_old} hours old"
+        end
+        true
+      end
+    rescue => e
+      LOG.debug "Failed to read cached configs at #{cache_path}. #{e}"
+      false
+    end
+
+    def load_json_file(file)
+      File.open(file) do |f|
+        f.flock(File::LOCK_SH)
+        configs = PrefabProto::Configs.decode_json(f.read)
+        load_configs(configs, :datafile)
+      end
+    end
+
+    # A thread that checks for a checkpoint
+    def start_checkpointing_thread
+      Thread.new do
+        loop do
+          started_at = Time.now
+          delta = @checkpoint_freq_secs - (Time.now - started_at)
+          sleep(delta) if delta > 0
+
+          load_checkpoint
+        rescue StandardError => e
+          LOG.debug "Issue Checkpointing #{e.message}"
+        end
+      end
+    end
+
+    def finish_init!(source, project_id)
+      return if initialized?
+
+      LOG.debug "Unlocked Config via #{source}"
+      @initialization_lock.count_down
+
+      presenter = Reforge::ConfigClientPresenter.new(
+        size: @config_resolver.local_store.size,
+        source: source,
+        project_id: project_id,
+        project_env_id: @config_resolver.project_env_id,
+        sdk_key_id: @base_client.options.sdk_key_id
+      )
+      LOG.info presenter.to_s
+      LOG.debug to_s
+    end
+  end
+end

data/lib/reforge/config_client_presenter.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Reforge
+  class ConfigClientPresenter
+    def initialize(size:, source:, project_id:, project_env_id:, sdk_key_id:)
+      @size = size
+      @source = source
+      @project_id = project_id
+      @project_env_id = project_env_id
+      @sdk_key_id = sdk_key_id
+    end
+
+    def to_s
+      "Configuration Loaded count=#{@size} source=#{@source} project=#{@project_id} project-env=#{@project_env_id} prefab.sdk-key-id=#{@sdk_key_id}"
+    end
+  end
+end
+

data/lib/reforge/config_loader.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+module Reforge
+  class ConfigLoader
+    LOG = Reforge::InternalLogger.new(self)
+
+    attr_reader :highwater_mark
+
+    def initialize(base_client)
+      @base_client = base_client
+      @prefab_options = base_client.options
+      @highwater_mark = 0
+      @classpath_config = load_classpath_config
+      @local_overrides = load_local_overrides
+      @api_config = Concurrent::Map.new
+    end
+
+    def calc_config
+      rtn = @classpath_config.clone
+      @api_config.each_key do |k|
+        rtn[k] = @api_config[k]
+      end
+      rtn.merge(@local_overrides)
+    end
+
+    def set(config, source)
+      # don't overwrite newer values
+      return if @api_config[config.key] && @api_config[config.key][:config].id >= config.id
+
+      if config.rows.empty?
+        @api_config.delete(config.key)
+      else
+        if @api_config[config.key]
+          LOG.debug(
+                                    "Replace #{config.key} with value from #{source} #{@api_config[config.key][:config].id} -> #{config.id}")
+        end
+        @api_config[config.key] = { source: source, config: config }
+      end
+      @highwater_mark = [config.id, @highwater_mark].max
+    end
+
+    def rm(key)
+      @api_config.delete key
+    end
+
+    def get_api_deltas
+      configs = PrefabProto::Configs.new
+      @api_config.each_value do |config_value|
+        configs.configs << config_value[:config]
+      end
+      configs
+    end
+
+    private
+
+    def load_classpath_config
+      return {} if @prefab_options.datafile?
+      {}
+    end
+
+    def load_local_overrides
+      return {} if @prefab_options.datafile?
+      {}
+    end
+
+  end
+end

data/lib/reforge/config_resolver.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+module Reforge
+  class ConfigResolver
+    attr_accessor :project_env_id # this will be set by the config_client when it gets an API response
+    attr_reader :local_store
+
+    def initialize(base_client, config_loader)
+      @lock = Concurrent::ReadWriteLock.new
+      @local_store = {}
+      @config_loader = config_loader
+      @project_env_id = 0 # we don't know this yet, it is set from the API results
+      @base_client = base_client
+      @on_update = nil
+      make_local
+    end
+
+    def to_s
+      presenter.to_s
+    end
+
+    def presenter
+      Reforge::ResolvedConfigPresenter.new(self, @lock, @local_store)
+    end
+
+    def keys
+      @local_store.keys
+    end
+
+    def raw(key)
+      @local_store.dig(key, :config)
+    end
+
+    def get(key, properties = NO_DEFAULT_PROVIDED)
+      @lock.with_read_lock do
+        raw_config = raw(key)
+
+        return nil unless raw_config
+
+        evaluate(raw_config, properties)
+      end
+    end
+
+    def evaluate(config, properties = NO_DEFAULT_PROVIDED)
+      Reforge::CriteriaEvaluator.new(config,
+                                    project_env_id: @project_env_id,
+                                    resolver: self,
+                                    namespace: @base_client.options.namespace,
+                                    base_client: @base_client).evaluate(make_context(properties))
+    end
+
+    def update
+      make_local
+
+      @on_update ? @on_update.call : nil
+    end
+
+    def on_update(&block)
+      @on_update = block
+    end
+
+    def make_context(properties)
+      if properties.is_a?(Context)
+        properties
+      elsif properties == NO_DEFAULT_PROVIDED || properties.nil?
+        Context.current
+      else
+        Context.join(parent: Context.current, hash: properties, id: :jit)
+      end
+    end
+
+    def symbolize_json_names?
+      @base_client.options.symbolize_json_names
+    end
+
+    private
+
+    def make_local
+      @lock.with_write_lock do
+        @local_store = @config_loader.calc_config
+      end
+    end
+  end
+end

data/lib/reforge/config_value_unwrapper.rb

@@ -0,0 +1,123 @@
+# frozen_string_literal: true
+
+module Reforge
+  class ConfigValueUnwrapper
+    LOG = Reforge::InternalLogger.new(self)
+    CONFIDENTIAL_PREFIX = "*****"
+    attr_reader :weighted_value_index
+
+    def initialize(config_value, resolver, weighted_value_index = nil)
+      @config_value = config_value
+      @resolver = resolver
+      @weighted_value_index = weighted_value_index
+    end
+
+    def reportable_wrapped_value
+      if @config_value.confidential || @config_value.decrypt_with&.length&.positive?
+        # Unique hash for differentiation
+        Reforge::ConfigValueWrapper.wrap("#{CONFIDENTIAL_PREFIX}#{Digest::MD5.hexdigest(unwrap)[0, 5]}")
+      else
+        @config_value
+      end
+    end
+
+    def reportable_value
+      Reforge::ConfigValueUnwrapper.new(reportable_wrapped_value, @resolver, @weighted_value_index).unwrap
+    end
+
+    def raw_config_value
+      @config_value
+    end
+
+    # this will return the actual value of confidential, use reportable_value unless you need it
+    def unwrap(raw_json: false)
+      raw = case @config_value.type
+            when :int, :string, :double, :bool, :log_level
+              @config_value.public_send(@config_value.type)
+            when :string_list
+              @config_value.string_list.values
+            when :duration
+              Reforge::Duration.new(@config_value.duration.definition)
+            when :json
+              if raw_json
+                @config_value.json.json
+              else
+                JSON.parse(@config_value.json.json, symbolize_names: @resolver.symbolize_json_names?)
+              end
+            else
+              LOG.error "Unknown type: #{@config_value.type}"
+              raise "Unknown type: #{@config_value.type}"
+            end
+      if @config_value.has_decrypt_with?
+        decryption_key = @resolver.get(@config_value.decrypt_with)&.unwrapped_value
+        if decryption_key.nil?
+          LOG.warn "No value for decryption key #{@config_value.decrypt_with} found."
+          return ""
+        else
+          unencrypted = Reforge::Encryption.new(decryption_key).decrypt(raw)
+          return unencrypted
+        end
+      end
+
+      raw
+    end
+
+    def self.deepest_value(config_value, config, context, resolver)
+      if config_value&.type == :weighted_values
+        value, index = Reforge::WeightedValueResolver.new(
+          config_value.weighted_values.weighted_values,
+          config.key,
+          context.get(config_value.weighted_values.hash_by_property_name)
+        ).resolve
+
+        new(deepest_value(value.value, config, context, resolver).raw_config_value, resolver, index)
+
+      elsif config_value&.type == :provided
+        if :ENV_VAR == config_value.provided.source
+          raw = ENV[config_value.provided.lookup]
+          if raw.nil?
+            raise Reforge::Errors::MissingEnvVarError.new("Missing environment variable #{config_value.provided.lookup}")
+          else
+            coerced = coerce_into_type(raw, config, config_value.provided.lookup)
+            new(Reforge::ConfigValueWrapper.wrap(coerced, confidential: config_value.confidential), resolver)
+          end
+        else
+          raise "Unknown Provided Source #{config_value.provided.source}"
+        end
+      else
+        new(config_value, resolver)
+      end
+    end
+
+    # Don't allow env vars to resolve to a value_type other than the config's value_type
+    def self.coerce_into_type(value_string, config, env_var_name)
+      case config.value_type
+      when :INT then Integer(value_string)
+      when :DOUBLE then Float(value_string)
+      when :STRING then String(value_string)
+      when :STRING_LIST then
+        maybe_string_list = YAML.load(value_string)
+        case maybe_string_list
+        when Array
+          maybe_string_list
+        else
+          raise raise Reforge::Errors::EnvVarParseError.new(value_string, config, env_var_name)
+        end
+      when :BOOL then
+        maybe_bool = YAML.load(value_string)
+        case maybe_bool
+        when TrueClass, FalseClass
+          maybe_bool
+        else
+          raise Reforge::Errors::EnvVarParseError.new(value_string, config, env_var_name)
+        end
+      when :NOT_SET_VALUE_TYPE
+        YAML.load(value_string)
+      else
+        raise Reforge::Errors::EnvVarParseError.new(value_string, config, env_var_name)
+      end
+    rescue ArgumentError
+      raise Reforge::Errors::EnvVarParseError.new(value_string, config, env_var_name)
+    end
+  end
+end

data/lib/reforge/config_value_wrapper.rb

@@ -0,0 +1,18 @@
+module Reforge
+  class ConfigValueWrapper
+    def self.wrap(value, confidential: nil)
+      case value
+      when Integer
+        PrefabProto::ConfigValue.new(int: value, confidential: confidential)
+      when Float
+        PrefabProto::ConfigValue.new(double: value, confidential: confidential)
+      when TrueClass, FalseClass
+        PrefabProto::ConfigValue.new(bool: value, confidential: confidential)
+      when Array
+        PrefabProto::ConfigValue.new(string_list: PrefabProto::StringList.new(values: value.map(&:to_s)), confidential: confidential)
+      else
+        PrefabProto::ConfigValue.new(string: value.to_s, confidential: confidential)
+      end
+    end
+  end
+end