prefab-cloud-ruby 0.13.0 → 0.13.1

data/lib/prefab/config_client.rb

@@ -1,48 +1,36 @@
-# frozen_string_literal: true
 module Prefab
   class ConfigClient
-    include Prefab::ConfigHelper
-
     RECONNECT_WAIT = 5
     DEFAULT_CHECKPOINT_FREQ_SEC = 60
     DEFAULT_S3CF_BUCKET = 'http://d2j4ed6ti5snnd.cloudfront.net'
-    SSE_READ_TIMEOUT = 300
 
     def initialize(base_client, timeout)
       @base_client = base_client
-      @options = base_client.options
-      @base_client.log_internal Logger::DEBUG, "Initialize ConfigClient"
       @timeout = timeout
-
-      @stream_lock = Concurrent::ReadWriteLock.new
+      @initialization_lock = Concurrent::ReadWriteLock.new
 
       @checkpoint_freq_secs = DEFAULT_CHECKPOINT_FREQ_SEC
 
       @config_loader = Prefab::ConfigLoader.new(@base_client)
       @config_resolver = Prefab::ConfigResolver.new(@base_client, @config_loader)
 
-      @initialization_lock = Concurrent::ReadWriteLock.new
-      @base_client.log_internal Logger::DEBUG, "Initialize ConfigClient: AcquireWriteLock"
       @initialization_lock.acquire_write_lock
-      @base_client.log_internal Logger::DEBUG, "Initialize ConfigClient: AcquiredWriteLock"
-      @initialized_future = Concurrent::Future.execute { @initialization_lock.acquire_read_lock }
 
       @cancellable_interceptor = Prefab::CancellableInterceptor.new(@base_client)
 
       @s3_cloud_front = ENV["PREFAB_S3CF_BUCKET"] || DEFAULT_S3CF_BUCKET
-
-      if @options.local_only?
-        finish_init!(:local_only)
-      else
-        load_checkpoint
-        start_checkpointing_thread
-        start_streaming
-      end
+      load_checkpoint
+      start_checkpointing_thread
     end
 
     def start_streaming
-      @stream_lock.with_write_lock do
-        start_sse_streaming_connection_thread(@config_loader.highwater_mark) if @streaming_thread.nil?
+      @streaming = true
+      start_api_connection_thread(@config_loader.highwater_mark)
+    end
+
+    def get(prop)
+      @initialization_lock.with_read_lock do
+        @config_resolver.get(prop)
       end
     end
 
@@ -55,7 +43,7 @@ module Prefab
 
       @base_client.request Prefab::ConfigService, :upsert, req_options: { timeout: @timeout }, params: upsert_req
       @base_client.stats.increment("prefab.config.upsert")
-      @config_loader.set(config_delta, :upsert)
+      @config_loader.set(config_delta)
       @config_loader.rm(previous_key) if previous_key&.present?
       @config_resolver.update
     end
@@ -70,48 +58,12 @@ module Prefab
     end
 
     def self.value_to_delta(key, config_value, namespace = nil)
-      Prefab::Config.new(key: [namespace, key].compact.join(":"),
-                         rows: [Prefab::ConfigRow.new(value: config_value)])
-    end
-
-    def get(key, default=Prefab::Client::NO_DEFAULT_PROVIDED)
-      config = _get(key)
-      config ? value_of(config[:value]) : handle_default(key, default)
-    end
-
-    def get_config_obj(key)
-      config = _get(key)
-      config ? config[:config] : nil
+      Prefab::ConfigDelta.new(key: [namespace, key].compact.join(":"),
+                              value: config_value)
     end
 
     private
 
-    def handle_default(key, default)
-      if default != Prefab::Client::NO_DEFAULT_PROVIDED
-        return default
-      end
-
-      if @options.on_no_default == Prefab::Options::ON_NO_DEFAULT::RAISE
-        raise Prefab::Errors::MissingDefaultError.new(key)
-      end
-
-      nil
-    end
-
-    def _get(key)
-      # wait timeout sec for the initalization to be complete
-      @initialized_future.value(@options.initialization_timeout_sec)
-      if @initialized_future.incomplete?
-        if @options.on_init_failure == Prefab::Options::ON_INITIALIZATION_FAILURE::RETURN
-          @base_client.log_internal Logger::WARN, "Couldn't Initialize In #{@options.initialization_timeout_sec}. Key #{key}. Returning what we have"
-          @initialization_lock.release_write_lock
-        else
-          raise Prefab::Errors::InitializationTimeoutError.new(@options.initialization_timeout_sec, key)
-        end
-      end
-      @config_resolver._get(key)
-    end
-
     def stub
       @_stub = Prefab::ConfigService::Stub.new(nil,
                                                nil,
@@ -119,92 +71,53 @@ module Prefab
                                                interceptors: [@base_client.interceptor, @cancellable_interceptor])
     end
 
-    # try API first, if not, fallback to s3
+    # Bootstrap out of the cache
+    # returns the high-watermark of what was in the cache
     def load_checkpoint
-      success = load_checkpoint_api_cdn
-
-      if success
-        return
-      else
-        @base_client.log_internal Logger::INFO, "LoadCheckpoint: Fallback to GRPC API"
-      end
-
-      success = load_checkpoint_from_grpc_api
-
-      if success
-        return
-      else
-        @base_client.log_internal Logger::INFO, "LoadCheckpoint: Fallback to S3"
-      end
-
-      success = load_checkpoint_from_s3
+      success = load_checkpoint_from_config
 
       if !success
-        @base_client.log_internal Logger::WARN, "No success loading checkpoints"
+        @base_client.log_internal Logger::INFO, "Fallback to S3"
+        load_checkpoint_from_s3
       end
-    end
 
-    def load_checkpoint_from_grpc_api
-      config_req = Prefab::ConfigServicePointer.new(start_at_id: @config_loader.highwater_mark)
+    rescue => e
+      @base_client.log_internal Logger::WARN, "Unexpected problem loading checkpoint #{e}"
+    end
 
+    def load_checkpoint_from_config
+      config_req = Prefab::ConfigServicePointer.new(account_id: @base_client.account_id,
+                                                    start_at_id: @config_loader.highwater_mark)
       resp = stub.get_all_config(config_req)
-      load_configs(resp, :remote_api_grpc)
+      load_deltas(resp, :api)
+      resp.deltas.each do |delta|
+        @config_loader.set(delta)
+      end
+      @config_resolver.update
+      finish_init!(:api)
       true
-    rescue GRPC::Unauthenticated
-      @base_client.log_internal Logger::WARN, "Unauthenticated"
     rescue => e
-      @base_client.log_internal Logger::WARN, "Unexpected grpc_api problem loading checkpoint #{e}"
+      @base_client.log_internal Logger::WARN, "Unexpected problem loading checkpoint #{e}"
       false
     end
 
-    def load_checkpoint_api_cdn
-      key_hash = Murmur3.murmur3_32(@base_client.api_key)
-      url = "#{@options.url_for_api_cdn}/api/v1/config/#{@base_client.project_id}/#{key_hash}/0"
-      conn = if Faraday::VERSION[0].to_i >= 2
-               Faraday.new(url) do |conn|
-                 conn.request :authorization, :basic, @base_client.project_id, @base_client.api_key
-               end
-             else
-               Faraday.new(url) do |conn|
-                 conn.request :basic_auth, @base_client.project_id, @base_client.api_key
-               end
-             end
-      load_url(conn, :remote_cdn_api)
-    end
-
     def load_checkpoint_from_s3
       url = "#{@s3_cloud_front}/#{@base_client.api_key.gsub("|", "/")}"
-      load_url(Faraday.new(url), :remote_s3)
-    end
-
-    def load_url(conn, source)
-      resp = conn.get('')
+      resp = Faraday.get url
       if resp.status == 200
-        configs = Prefab::Configs.decode(resp.body)
-        load_configs(configs, source)
-        true
+        deltas = Prefab::ConfigDeltas.decode(resp.body)
+        load_deltas(deltas, :s3)
       else
-        @base_client.log_internal Logger::INFO, "Checkpoint #{source} failed to load. Response #{resp.status}"
-        false
+        @base_client.log_internal Logger::INFO, "No S3 checkpoint. Response #{resp.status} Plan may not support this."
       end
-    rescue => e
-      @base_client.log_internal Logger::WARN, "Unexpected #{source} problem loading checkpoint #{e} #{conn}"
-      false
     end
 
-    def load_configs(configs, source)
-      project_env_id = configs.config_service_pointer.project_env_id
-      @config_resolver.project_env_id = project_env_id
-      starting_highwater_mark = @config_loader.highwater_mark
 
-      configs.configs.each do |config|
-        @config_loader.set(config, source)
-      end
-      if @config_loader.highwater_mark > starting_highwater_mark
-        @base_client.log_internal Logger::INFO, "Found new checkpoint with highwater id #{@config_loader.highwater_mark} from #{source} in project #{@base_client.project_id} environment: #{project_env_id} and namespace: '#{@namespace}'"
-      else
-        @base_client.log_internal Logger::DEBUG, "Checkpoint with highwater id #{@config_loader.highwater_mark} from #{source}. No changes.", "prefab.config_client.load_configs"
+    def load_deltas(deltas, source)
+      deltas.deltas.each do |delta|
+        @config_loader.set(delta)
       end
+      @base_client.log_internal Logger::INFO, "Found checkpoint with highwater id #{@config_loader.highwater_mark} from #{source}"
       @base_client.stats.increment("prefab.config.checkpoint.load")
       @config_resolver.update
       finish_init!(source)
@@ -212,7 +125,6 @@ module Prefab
 
     # A thread that checks for a checkpoint
     def start_checkpointing_thread
-
       Thread.new do
         loop do
           begin
@@ -232,31 +144,47 @@ module Prefab
 
     def finish_init!(source)
       if @initialization_lock.write_locked?
-        @base_client.log_internal Logger::INFO, "Unlocked Config via #{source}"
+        @base_client.log_internal Logger::DEBUG, "Unlocked Config via #{source}"
         @initialization_lock.release_write_lock
         @base_client.log.set_config_client(self)
-        @base_client.log_internal Logger::INFO, to_s
       end
     end
 
-    def start_sse_streaming_connection_thread(start_at_id)
-      auth = "#{@base_client.project_id}:#{@base_client.api_key}"
-      auth_string = Base64.strict_encode64(auth)
-      headers = {
-        "x-prefab-start-at-id": start_at_id,
-        "Authorization": "Basic #{auth_string}",
-      }
-      url = "#{@base_client.prefab_api_url}/api/v1/sse/config"
-      @base_client.log_internal Logger::INFO, "SSE Streaming Connect to #{url} start_at #{start_at_id}"
-      @streaming_thread = SSE::Client.new(url,
-                                          headers: headers,
-                                          read_timeout: SSE_READ_TIMEOUT,
-                                          logger: Prefab::InternalLogger.new("prefab.config.sse", @base_client.log)) do |client|
-        client.on_event do |event|
-          configs = Prefab::Configs.decode(Base64.decode64(event.data))
-          load_configs(configs, :sse)
+    # Setup a streaming connection to the API
+    # Save new config values into the loader
+    def start_api_connection_thread(start_at_id)
+      config_req = Prefab::ConfigServicePointer.new(account_id: @base_client.account_id,
+                                                    start_at_id: start_at_id)
+      @base_client.log_internal Logger::DEBUG, "start api connection thread #{start_at_id}"
+      @base_client.stats.increment("prefab.config.api.start")
+
+      @api_connection_thread = Thread.new do
+        at_exit do
+          @streaming = false
+          @cancellable_interceptor.cancel
+        end
+
+        while @streaming do
+          begin
+            resp = stub.get_config(config_req)
+            resp.each do |r|
+              r.deltas.each do |delta|
+                @config_loader.set(delta)
+              end
+              @config_resolver.update
+              finish_init!(:streaming)
+            end
+          rescue => e
+            if @streaming
+              level = e.code == 1 ? Logger::DEBUG : Logger::INFO
+              @base_client.log_internal level, ("config client encountered #{e.message} pausing #{RECONNECT_WAIT}")
+              reset
+              sleep(RECONNECT_WAIT)
+            end
+          end
         end
       end
+
     end
   end
 end

data/lib/prefab/config_loader.rb

@@ -1,5 +1,3 @@
-# frozen_string_literal: true
-
 require 'yaml'
 module Prefab
   class ConfigLoader
@@ -7,7 +5,6 @@ module Prefab
 
     def initialize(base_client)
       @base_client = base_client
-      @prefab_options = base_client.options
       @highwater_mark = 0
       @classpath_config = load_classpath_config
       @local_overrides = load_local_overrides
@@ -17,27 +14,24 @@ module Prefab
     def calc_config
       rtn = @classpath_config.clone
       @api_config.each_key do |k|
-        rtn[k] = @api_config[k]
+        rtn[k] = @api_config[k].value
       end
       rtn = rtn.merge(@local_overrides)
       rtn
     end
 
-    def set(config, source)
+    def set(delta)
       # don't overwrite newer values
-      if @api_config[config.key] && @api_config[config.key][:config].id >= config.id
+      if @api_config[delta.key] && @api_config[delta.key].id > delta.id
         return
       end
 
-      if config.rows.empty?
-        @api_config.delete(config.key)
+      if delta.value.nil?
+        @api_config.delete(delta.key)
       else
-        if @api_config[config.key]
-          @base_client.log_internal Logger::DEBUG, "Replace #{config.key} with value from #{source} #{ @api_config[config.key][:config].id} -> #{config.id}"
-        end
-        @api_config[config.key] = { source: source, config: config }
+        @api_config[delta.key] = delta
       end
-      @highwater_mark = [config.id, @highwater_mark].max
+      @highwater_mark = [delta.id, @highwater_mark].max
     end
 
     def rm(key)
@@ -45,22 +39,22 @@ module Prefab
     end
 
     def get_api_deltas
-      configs = Prefab::Configs.new
+      deltas = Prefab::ConfigDeltas.new
       @api_config.each_value do |config_value|
-        configs.configs << config_value[:config]
+        deltas.deltas << config_value
       end
-      configs
+      deltas
     end
 
     private
 
     def load_classpath_config
-      classpath_dir = @prefab_options.prefab_config_classpath_dir
+      classpath_dir = ENV['PREFAB_CONFIG_CLASSPATH_DIR'] || "."
       load_glob(File.join(classpath_dir, ".prefab*config.yaml"))
     end
 
     def load_local_overrides
-      override_dir = @prefab_options.prefab_config_override_dir
+      override_dir = ENV['PREFAB_CONFIG_OVERRIDE_DIR'] || Dir.home
       load_glob(File.join(override_dir, ".prefab*config.yaml"))
     end
 
@@ -69,39 +63,7 @@ module Prefab
       Dir.glob(glob).each do |file|
         yaml = load(file)
         yaml.each do |k, v|
-          if v.class == Hash
-            v.each do |env_k, env_v|
-              if k == @prefab_options.defaults_env
-                if env_v.class == Hash && env_v['feature_flag']
-                  rtn[env_k] = feature_flag_config(file, k, env_k, env_v)
-                else
-                  rtn[env_k] = {
-                    source: file,
-                    match: k,
-                    config: Prefab::Config.new(
-                      key: env_k,
-                      rows: [
-                        Prefab::ConfigRow.new(value: Prefab::ConfigValue.new(value_from(env_v)))
-                      ]
-                    )
-                  }
-                end
-              else
-                next
-              end
-            end
-          else
-            rtn[k] = {
-              source: file,
-              match: "default",
-              config: Prefab::Config.new(
-                key: k,
-                rows: [
-                  Prefab::ConfigRow.new(value: Prefab::ConfigValue.new(value_from(v)))
-                ]
-              )
-            }
-          end
+          rtn[k] = Prefab::ConfigValue.new(value_from(v))
         end
       end
       rtn
@@ -129,52 +91,5 @@ module Prefab
         { double: raw }
       end
     end
-
-    def feature_flag_config(file, k, env_k, env_v)
-      criteria = Prefab::Criteria.new(operator: 'ALWAYS_TRUE')
-
-      if env_v['criteria']
-        criteria = Prefab::Criteria.new(criteria_values(env_v['criteria']))
-      end
-
-      row = Prefab::ConfigRow.new(
-        value: Prefab::ConfigValue.new(
-          feature_flag: Prefab::FeatureFlag.new(
-            active: true,
-            inactive_variant_idx: -1, # not supported
-            rules: [
-              Prefab::Rule.new(
-                variant_weights: [
-                  Prefab::VariantWeight.new(variant_idx: 0, weight: 1000)
-                ],
-                criteria: criteria
-              )
-            ]
-          )
-        )
-      )
-
-      unless env_v.has_key?('value')
-        raise Prefab::Error, "Feature flag config `#{env_k}` #{file} must have a `value`"
-      end
-
-      {
-        source: file,
-        match: k,
-        config: Prefab::Config.new(
-          key: env_k,
-          variants: [Prefab::FeatureFlagVariant.new(value_from(env_v['value']))],
-          rows: [row]
-        )
-      }
-    end
-
-    def criteria_values(criteria_hash)
-      if RUBY_VERSION < '2.7'
-        criteria_hash.transform_keys(&:to_sym)
-      else
-        criteria_hash
-      end
-    end
   end
 end

data/lib/prefab/config_resolver.rb

@@ -1,62 +1,59 @@
-# frozen_string_literal: true
 module Prefab
   class ConfigResolver
-    include Prefab::ConfigHelper
-    NAMESPACE_DELIMITER = "."
-
-    attr_accessor :project_env_id # this will be set by the config_client when it gets an API response
+    NAMESPACE_DELIMITER = ".".freeze
+    NAME_KEY_DELIMITER = ":".freeze
 
     def initialize(base_client, config_loader)
       @lock = Concurrent::ReadWriteLock.new
       @local_store = {}
-      @namespace = base_client.options.namespace
+      @namespace = base_client.namespace
       @config_loader = config_loader
-      @project_env_id = 0
       make_local
     end
 
     def to_s
-      str = "\n"
+      str = ""
       @lock.with_read_lock do
         @local_store.each do |k, v|
-          elements = [k.slice(0..49).ljust(50)]
-          if v.nil?
-            elements << "tombstone"
-          else
-            value = v[:value]
-            elements << value_of(value).to_s.slice(0..34).ljust(35)
-            elements << value_of(value).class.to_s.slice(0..6).ljust(7)
-            elements << "Match: #{v[:match]}".slice(0..29).ljust(30)
-            elements << "Source: #{v[:source]}"
-          end
-          str += elements.join(" | ") << "\n"
+          value = v[:value]
+          str << "|#{k}| in #{v[:namespace]} |#{value_of(value)}|#{value_of(value).class}\n"
         end
       end
       str
     end
 
     def get(property)
-      config = _get(property)
-      config ? value_of(config[:value]) : nil
-    end
-
-    def get_config(property)
-      config = _get(property)
-      config ? config[:config] : nil
-    end
-
-    def _get(key)
-      @lock.with_read_lock do
-        @local_store[key]
+      config = @lock.with_read_lock do
+        @local_store[property]
       end
+      config ? value_of(config[:value]) : nil
     end
 
     def update
       make_local
     end
 
+    def export_api_deltas
+      @config_loader.get_api_deltas
+    end
+
     private
 
+    def value_of(config_value)
+      case config_value.type
+      when :string
+        config_value.string
+      when :int
+        config_value.int
+      when :double
+        config_value.double
+      when :bool
+        config_value.bool
+      when :feature_flag
+        config_value.feature_flag
+      end
+    end
+
     # Should client a.b.c see key in namespace a.b? yes
     # Should client a.b.c see key in namespace a.b.c? yes
     # Should client a.b.c see key in namespace a.b.d? no
@@ -64,37 +61,33 @@ module Prefab
     #
     def starts_with_ns?(key_namespace, client_namespace)
       zipped = key_namespace.split(NAMESPACE_DELIMITER).zip(client_namespace.split(NAMESPACE_DELIMITER))
-      mapped = zipped.map do |k, c|
-        (k.nil? || k.empty?) || k == c
-      end
-      [mapped.all?, mapped.size]
+      zipped.map do |k, c|
+        (k.nil? || k.empty?) || c == k
+      end.all?
     end
 
     def make_local
       store = {}
-      @config_loader.calc_config.each do |key, config_resolver_obj|
-        config = config_resolver_obj[:config]
-        sortable = config.rows.map do |row|
-          if row.project_env_id != 0
-            if row.project_env_id == @project_env_id
-              if !row.namespace.empty?
-                (starts_with, count) = starts_with_ns?(row.namespace, @namespace)
-                # rubocop:disable BlockNesting
-                { sortable: 2 + count, match: "nm:#{row.namespace}", value: row.value, config: config} if starts_with
-              else
-                { sortable: 1, match: "env:#{row.project_env_id}", value: row.value, config: config}
-              end
-            end
-          else
-            match = config_resolver_obj[:match] || "default"
-            { sortable: 0, match: match, value: row.value, config: config}
+      @config_loader.calc_config.each do |prop, value|
+        property = prop
+        key_namespace = ""
+
+        split = prop.split(NAME_KEY_DELIMITER)
+
+        if split.size > 1
+          property = split[1..-1].join(NAME_KEY_DELIMITER)
+          key_namespace = split[0]
+        end
+
+        if starts_with_ns?(key_namespace, @namespace)
+          existing = store[property]
+          if existing.nil?
+            store[property] = { namespace: key_namespace, value: value }
+          elsif existing[:namespace].split(NAMESPACE_DELIMITER).size < key_namespace.split(NAMESPACE_DELIMITER).size
+            store[property] = { namespace: key_namespace, value: value }
           end
-        end.compact
-        to_store = sortable.sort_by { |h| h[:sortable] }.last
-        to_store[:source] = config_resolver_obj[:source]
-        store[key] = to_store
+        end
       end
-
       @lock.with_write_lock do
         @local_store = store
       end