prefab-cloud-ruby 0.13.1 → 0.13.2

data/lib/prefab/config_client.rb

@@ -1,36 +1,48 @@
+# frozen_string_literal: true
 module Prefab
   class ConfigClient
+    include Prefab::ConfigHelper
+
     RECONNECT_WAIT = 5
     DEFAULT_CHECKPOINT_FREQ_SEC = 60
     DEFAULT_S3CF_BUCKET = 'http://d2j4ed6ti5snnd.cloudfront.net'
+    SSE_READ_TIMEOUT = 300
 
     def initialize(base_client, timeout)
       @base_client = base_client
+      @options = base_client.options
+      @base_client.log_internal Logger::DEBUG, "Initialize ConfigClient"
       @timeout = timeout
-      @initialization_lock = Concurrent::ReadWriteLock.new
+
+      @stream_lock = Concurrent::ReadWriteLock.new
 
       @checkpoint_freq_secs = DEFAULT_CHECKPOINT_FREQ_SEC
 
       @config_loader = Prefab::ConfigLoader.new(@base_client)
       @config_resolver = Prefab::ConfigResolver.new(@base_client, @config_loader)
 
+      @initialization_lock = Concurrent::ReadWriteLock.new
+      @base_client.log_internal Logger::DEBUG, "Initialize ConfigClient: AcquireWriteLock"
       @initialization_lock.acquire_write_lock
+      @base_client.log_internal Logger::DEBUG, "Initialize ConfigClient: AcquiredWriteLock"
+      @initialized_future = Concurrent::Future.execute { @initialization_lock.acquire_read_lock }
 
       @cancellable_interceptor = Prefab::CancellableInterceptor.new(@base_client)
 
       @s3_cloud_front = ENV["PREFAB_S3CF_BUCKET"] || DEFAULT_S3CF_BUCKET
-      load_checkpoint
-      start_checkpointing_thread
-    end
 
-    def start_streaming
-      @streaming = true
-      start_api_connection_thread(@config_loader.highwater_mark)
+      if @options.local_only?
+        finish_init!(:local_only)
+      else
+        load_checkpoint
+        start_checkpointing_thread
+        start_streaming
+      end
     end
 
-    def get(prop)
-      @initialization_lock.with_read_lock do
-        @config_resolver.get(prop)
+    def start_streaming
+      @stream_lock.with_write_lock do
+        start_sse_streaming_connection_thread(@config_loader.highwater_mark) if @streaming_thread.nil?
       end
     end
 
@@ -43,7 +55,7 @@ module Prefab
 
       @base_client.request Prefab::ConfigService, :upsert, req_options: { timeout: @timeout }, params: upsert_req
       @base_client.stats.increment("prefab.config.upsert")
-      @config_loader.set(config_delta)
+      @config_loader.set(config_delta, :upsert)
       @config_loader.rm(previous_key) if previous_key&.present?
       @config_resolver.update
     end
@@ -58,12 +70,48 @@ module Prefab
     end
 
     def self.value_to_delta(key, config_value, namespace = nil)
-      Prefab::ConfigDelta.new(key: [namespace, key].compact.join(":"),
-                              value: config_value)
+      Prefab::Config.new(key: [namespace, key].compact.join(":"),
+                         rows: [Prefab::ConfigRow.new(value: config_value)])
+    end
+
+    def get(key, default=Prefab::Client::NO_DEFAULT_PROVIDED)
+      config = _get(key)
+      config ? value_of(config[:value]) : handle_default(key, default)
+    end
+
+    def get_config_obj(key)
+      config = _get(key)
+      config ? config[:config] : nil
     end
 
     private
 
+    def handle_default(key, default)
+      if default != Prefab::Client::NO_DEFAULT_PROVIDED
+        return default
+      end
+
+      if @options.on_no_default == Prefab::Options::ON_NO_DEFAULT::RAISE
+        raise Prefab::Errors::MissingDefaultError.new(key)
+      end
+
+      nil
+    end
+
+    def _get(key)
+      # wait timeout sec for the initalization to be complete
+      @initialized_future.value(@options.initialization_timeout_sec)
+      if @initialized_future.incomplete?
+        if @options.on_init_failure == Prefab::Options::ON_INITIALIZATION_FAILURE::RETURN
+          @base_client.log_internal Logger::WARN, "Couldn't Initialize In #{@options.initialization_timeout_sec}. Key #{key}. Returning what we have"
+          @initialization_lock.release_write_lock
+        else
+          raise Prefab::Errors::InitializationTimeoutError.new(@options.initialization_timeout_sec, key)
+        end
+      end
+      @config_resolver._get(key)
+    end
+
     def stub
       @_stub = Prefab::ConfigService::Stub.new(nil,
                                                nil,
@@ -71,53 +119,92 @@ module Prefab
                                                interceptors: [@base_client.interceptor, @cancellable_interceptor])
     end
 
-    # Bootstrap out of the cache
-    # returns the high-watermark of what was in the cache
+    # try API first, if not, fallback to s3
     def load_checkpoint
-      success = load_checkpoint_from_config
+      success = load_checkpoint_api_cdn
 
-      if !success
-        @base_client.log_internal Logger::INFO, "Fallback to S3"
-        load_checkpoint_from_s3
+      if success
+        return
+      else
+        @base_client.log_internal Logger::INFO, "LoadCheckpoint: Fallback to GRPC API"
       end
 
-    rescue => e
-      @base_client.log_internal Logger::WARN, "Unexpected problem loading checkpoint #{e}"
+      success = load_checkpoint_from_grpc_api
+
+      if success
+        return
+      else
+        @base_client.log_internal Logger::INFO, "LoadCheckpoint: Fallback to S3"
+      end
+
+      success = load_checkpoint_from_s3
+
+      if !success
+        @base_client.log_internal Logger::WARN, "No success loading checkpoints"
+      end
     end
 
-    def load_checkpoint_from_config
-      config_req = Prefab::ConfigServicePointer.new(account_id: @base_client.account_id,
-                                                    start_at_id: @config_loader.highwater_mark)
+    def load_checkpoint_from_grpc_api
+      config_req = Prefab::ConfigServicePointer.new(start_at_id: @config_loader.highwater_mark)
+
       resp = stub.get_all_config(config_req)
-      load_deltas(resp, :api)
-      resp.deltas.each do |delta|
-        @config_loader.set(delta)
-      end
-      @config_resolver.update
-      finish_init!(:api)
+      load_configs(resp, :remote_api_grpc)
       true
+    rescue GRPC::Unauthenticated
+      @base_client.log_internal Logger::WARN, "Unauthenticated"
     rescue => e
-      @base_client.log_internal Logger::WARN, "Unexpected problem loading checkpoint #{e}"
+      @base_client.log_internal Logger::WARN, "Unexpected grpc_api problem loading checkpoint #{e}"
       false
     end
 
+    def load_checkpoint_api_cdn
+      key_hash = Murmur3.murmur3_32(@base_client.api_key)
+      url = "#{@options.url_for_api_cdn}/api/v1/config/#{@base_client.project_id}/#{key_hash}/0"
+      conn = if Faraday::VERSION[0].to_i >= 2
+               Faraday.new(url) do |conn|
+                 conn.request :authorization, :basic, @base_client.project_id, @base_client.api_key
+               end
+             else
+               Faraday.new(url) do |conn|
+                 conn.request :basic_auth, @base_client.project_id, @base_client.api_key
+               end
+             end
+      load_url(conn, :remote_cdn_api)
+    end
+
     def load_checkpoint_from_s3
       url = "#{@s3_cloud_front}/#{@base_client.api_key.gsub("|", "/")}"
-      resp = Faraday.get url
+      load_url(Faraday.new(url), :remote_s3)
+    end
+
+    def load_url(conn, source)
+      resp = conn.get('')
       if resp.status == 200
-        deltas = Prefab::ConfigDeltas.decode(resp.body)
-        load_deltas(deltas, :s3)
+        configs = Prefab::Configs.decode(resp.body)
+        load_configs(configs, source)
+        true
       else
-        @base_client.log_internal Logger::INFO, "No S3 checkpoint. Response #{resp.status} Plan may not support this."
+        @base_client.log_internal Logger::INFO, "Checkpoint #{source} failed to load. Response #{resp.status}"
+        false
       end
+    rescue => e
+      @base_client.log_internal Logger::WARN, "Unexpected #{source} problem loading checkpoint #{e} #{conn}"
+      false
     end
 
+    def load_configs(configs, source)
+      project_env_id = configs.config_service_pointer.project_env_id
+      @config_resolver.project_env_id = project_env_id
+      starting_highwater_mark = @config_loader.highwater_mark
 
-    def load_deltas(deltas, source)
-      deltas.deltas.each do |delta|
-        @config_loader.set(delta)
+      configs.configs.each do |config|
+        @config_loader.set(config, source)
+      end
+      if @config_loader.highwater_mark > starting_highwater_mark
+        @base_client.log_internal Logger::INFO, "Found new checkpoint with highwater id #{@config_loader.highwater_mark} from #{source} in project #{@base_client.project_id} environment: #{project_env_id} and namespace: '#{@namespace}'"
+      else
+        @base_client.log_internal Logger::DEBUG, "Checkpoint with highwater id #{@config_loader.highwater_mark} from #{source}. No changes.", "prefab.config_client.load_configs"
       end
-      @base_client.log_internal Logger::INFO, "Found checkpoint with highwater id #{@config_loader.highwater_mark} from #{source}"
       @base_client.stats.increment("prefab.config.checkpoint.load")
       @config_resolver.update
       finish_init!(source)
@@ -125,6 +212,7 @@ module Prefab
 
     # A thread that checks for a checkpoint
     def start_checkpointing_thread
+
       Thread.new do
         loop do
           begin
@@ -144,47 +232,31 @@ module Prefab
 
     def finish_init!(source)
       if @initialization_lock.write_locked?
-        @base_client.log_internal Logger::DEBUG, "Unlocked Config via #{source}"
+        @base_client.log_internal Logger::INFO, "Unlocked Config via #{source}"
         @initialization_lock.release_write_lock
         @base_client.log.set_config_client(self)
+        @base_client.log_internal Logger::INFO, to_s
       end
     end
 
-    # Setup a streaming connection to the API
-    # Save new config values into the loader
-    def start_api_connection_thread(start_at_id)
-      config_req = Prefab::ConfigServicePointer.new(account_id: @base_client.account_id,
-                                                    start_at_id: start_at_id)
-      @base_client.log_internal Logger::DEBUG, "start api connection thread #{start_at_id}"
-      @base_client.stats.increment("prefab.config.api.start")
-
-      @api_connection_thread = Thread.new do
-        at_exit do
-          @streaming = false
-          @cancellable_interceptor.cancel
-        end
-
-        while @streaming do
-          begin
-            resp = stub.get_config(config_req)
-            resp.each do |r|
-              r.deltas.each do |delta|
-                @config_loader.set(delta)
-              end
-              @config_resolver.update
-              finish_init!(:streaming)
-            end
-          rescue => e
-            if @streaming
-              level = e.code == 1 ? Logger::DEBUG : Logger::INFO
-              @base_client.log_internal level, ("config client encountered #{e.message} pausing #{RECONNECT_WAIT}")
-              reset
-              sleep(RECONNECT_WAIT)
-            end
-          end
+    def start_sse_streaming_connection_thread(start_at_id)
+      auth = "#{@base_client.project_id}:#{@base_client.api_key}"
+      auth_string = Base64.strict_encode64(auth)
+      headers = {
+        "x-prefab-start-at-id": start_at_id,
+        "Authorization": "Basic #{auth_string}",
+      }
+      url = "#{@base_client.prefab_api_url}/api/v1/sse/config"
+      @base_client.log_internal Logger::INFO, "SSE Streaming Connect to #{url} start_at #{start_at_id}"
+      @streaming_thread = SSE::Client.new(url,
+                                          headers: headers,
+                                          read_timeout: SSE_READ_TIMEOUT,
+                                          logger: Prefab::InternalLogger.new("prefab.config.sse", @base_client.log)) do |client|
+        client.on_event do |event|
+          configs = Prefab::Configs.decode(Base64.decode64(event.data))
+          load_configs(configs, :sse)
         end
       end
-
     end
   end
 end

data/lib/prefab/config_helper.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+module Prefab
+  module ConfigHelper
+    def value_of(config_value)
+      case config_value.type
+      when :string
+        config_value.string
+      when :int
+        config_value.int
+      when :double
+        config_value.double
+      when :bool
+        config_value.bool
+      when :feature_flag
+        config_value.feature_flag
+      when :segment
+        config_value.segment
+      end
+    end
+
+    def value_of_variant(feature_flag_variant)
+      return feature_flag_variant.string if feature_flag_variant.has_string?
+      return feature_flag_variant.int if feature_flag_variant.has_int?
+      return feature_flag_variant.double if feature_flag_variant.has_double?
+      return feature_flag_variant.bool if feature_flag_variant.has_bool?
+      return nil
+    end
+  end
+end

data/lib/prefab/config_loader.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'yaml'
 module Prefab
   class ConfigLoader
@@ -5,6 +7,7 @@ module Prefab
 
     def initialize(base_client)
       @base_client = base_client
+      @prefab_options = base_client.options
       @highwater_mark = 0
       @classpath_config = load_classpath_config
       @local_overrides = load_local_overrides
@@ -14,24 +17,27 @@ module Prefab
     def calc_config
       rtn = @classpath_config.clone
       @api_config.each_key do |k|
-        rtn[k] = @api_config[k].value
+        rtn[k] = @api_config[k]
       end
       rtn = rtn.merge(@local_overrides)
       rtn
     end
 
-    def set(delta)
+    def set(config, source)
       # don't overwrite newer values
-      if @api_config[delta.key] && @api_config[delta.key].id > delta.id
+      if @api_config[config.key] && @api_config[config.key][:config].id >= config.id
         return
       end
 
-      if delta.value.nil?
-        @api_config.delete(delta.key)
+      if config.rows.empty?
+        @api_config.delete(config.key)
       else
-        @api_config[delta.key] = delta
+        if @api_config[config.key]
+          @base_client.log_internal Logger::DEBUG, "Replace #{config.key} with value from #{source} #{ @api_config[config.key][:config].id} -> #{config.id}"
+        end
+        @api_config[config.key] = { source: source, config: config }
       end
-      @highwater_mark = [delta.id, @highwater_mark].max
+      @highwater_mark = [config.id, @highwater_mark].max
     end
 
     def rm(key)
@@ -39,22 +45,22 @@ module Prefab
     end
 
     def get_api_deltas
-      deltas = Prefab::ConfigDeltas.new
+      configs = Prefab::Configs.new
       @api_config.each_value do |config_value|
-        deltas.deltas << config_value
+        configs.configs << config_value[:config]
       end
-      deltas
+      configs
     end
 
     private
 
     def load_classpath_config
-      classpath_dir = ENV['PREFAB_CONFIG_CLASSPATH_DIR'] || "."
+      classpath_dir = @prefab_options.prefab_config_classpath_dir
       load_glob(File.join(classpath_dir, ".prefab*config.yaml"))
     end
 
     def load_local_overrides
-      override_dir = ENV['PREFAB_CONFIG_OVERRIDE_DIR'] || Dir.home
+      override_dir = @prefab_options.prefab_config_override_dir
       load_glob(File.join(override_dir, ".prefab*config.yaml"))
     end
 
@@ -63,7 +69,39 @@ module Prefab
       Dir.glob(glob).each do |file|
         yaml = load(file)
         yaml.each do |k, v|
-          rtn[k] = Prefab::ConfigValue.new(value_from(v))
+          if v.class == Hash
+            v.each do |env_k, env_v|
+              if k == @prefab_options.defaults_env
+                if env_v.class == Hash && env_v['feature_flag']
+                  rtn[env_k] = feature_flag_config(file, k, env_k, env_v)
+                else
+                  rtn[env_k] = {
+                    source: file,
+                    match: k,
+                    config: Prefab::Config.new(
+                      key: env_k,
+                      rows: [
+                        Prefab::ConfigRow.new(value: Prefab::ConfigValue.new(value_from(env_v)))
+                      ]
+                    )
+                  }
+                end
+              else
+                next
+              end
+            end
+          else
+            rtn[k] = {
+              source: file,
+              match: "default",
+              config: Prefab::Config.new(
+                key: k,
+                rows: [
+                  Prefab::ConfigRow.new(value: Prefab::ConfigValue.new(value_from(v)))
+                ]
+              )
+            }
+          end
         end
       end
       rtn
@@ -91,5 +129,52 @@ module Prefab
         { double: raw }
       end
     end
+
+    def feature_flag_config(file, k, env_k, env_v)
+      criteria = Prefab::Criteria.new(operator: 'ALWAYS_TRUE')
+
+      if env_v['criteria']
+        criteria = Prefab::Criteria.new(criteria_values(env_v['criteria']))
+      end
+
+      row = Prefab::ConfigRow.new(
+        value: Prefab::ConfigValue.new(
+          feature_flag: Prefab::FeatureFlag.new(
+            active: true,
+            inactive_variant_idx: -1, # not supported
+            rules: [
+              Prefab::Rule.new(
+                variant_weights: [
+                  Prefab::VariantWeight.new(variant_idx: 0, weight: 1000)
+                ],
+                criteria: criteria
+              )
+            ]
+          )
+        )
+      )
+
+      unless env_v.has_key?('value')
+        raise Prefab::Error, "Feature flag config `#{env_k}` #{file} must have a `value`"
+      end
+
+      {
+        source: file,
+        match: k,
+        config: Prefab::Config.new(
+          key: env_k,
+          variants: [Prefab::FeatureFlagVariant.new(value_from(env_v['value']))],
+          rows: [row]
+        )
+      }
+    end
+
+    def criteria_values(criteria_hash)
+      if RUBY_VERSION < '2.7'
+        criteria_hash.transform_keys(&:to_sym)
+      else
+        criteria_hash
+      end
+    end
   end
 end

data/lib/prefab/config_resolver.rb

@@ -1,59 +1,62 @@
+# frozen_string_literal: true
 module Prefab
   class ConfigResolver
-    NAMESPACE_DELIMITER = ".".freeze
-    NAME_KEY_DELIMITER = ":".freeze
+    include Prefab::ConfigHelper
+    NAMESPACE_DELIMITER = "."
+
+    attr_accessor :project_env_id # this will be set by the config_client when it gets an API response
 
     def initialize(base_client, config_loader)
       @lock = Concurrent::ReadWriteLock.new
       @local_store = {}
-      @namespace = base_client.namespace
+      @namespace = base_client.options.namespace
       @config_loader = config_loader
+      @project_env_id = 0
       make_local
     end
 
     def to_s
-      str = ""
+      str = "\n"
       @lock.with_read_lock do
         @local_store.each do |k, v|
-          value = v[:value]
-          str << "|#{k}| in #{v[:namespace]} |#{value_of(value)}|#{value_of(value).class}\n"
+          elements = [k.slice(0..49).ljust(50)]
+          if v.nil?
+            elements << "tombstone"
+          else
+            value = v[:value]
+            elements << value_of(value).to_s.slice(0..34).ljust(35)
+            elements << value_of(value).class.to_s.slice(0..6).ljust(7)
+            elements << "Match: #{v[:match]}".slice(0..29).ljust(30)
+            elements << "Source: #{v[:source]}"
+          end
+          str += elements.join(" | ") << "\n"
         end
       end
       str
     end
 
     def get(property)
-      config = @lock.with_read_lock do
-        @local_store[property]
-      end
+      config = _get(property)
       config ? value_of(config[:value]) : nil
     end
 
-    def update
-      make_local
+    def get_config(property)
+      config = _get(property)
+      config ? config[:config] : nil
     end
 
-    def export_api_deltas
-      @config_loader.get_api_deltas
+    def _get(key)
+      @lock.with_read_lock do
+        @local_store[key]
+      end
     end
 
-    private
-
-    def value_of(config_value)
-      case config_value.type
-      when :string
-        config_value.string
-      when :int
-        config_value.int
-      when :double
-        config_value.double
-      when :bool
-        config_value.bool
-      when :feature_flag
-        config_value.feature_flag
-      end
+    def update
+      make_local
     end
 
+    private
+
     # Should client a.b.c see key in namespace a.b? yes
     # Should client a.b.c see key in namespace a.b.c? yes
     # Should client a.b.c see key in namespace a.b.d? no
@@ -61,33 +64,37 @@ module Prefab
     #
     def starts_with_ns?(key_namespace, client_namespace)
       zipped = key_namespace.split(NAMESPACE_DELIMITER).zip(client_namespace.split(NAMESPACE_DELIMITER))
-      zipped.map do |k, c|
-        (k.nil? || k.empty?) || c == k
-      end.all?
+      mapped = zipped.map do |k, c|
+        (k.nil? || k.empty?) || k == c
+      end
+      [mapped.all?, mapped.size]
     end
 
     def make_local
       store = {}
-      @config_loader.calc_config.each do |prop, value|
-        property = prop
-        key_namespace = ""
-
-        split = prop.split(NAME_KEY_DELIMITER)
-
-        if split.size > 1
-          property = split[1..-1].join(NAME_KEY_DELIMITER)
-          key_namespace = split[0]
-        end
-
-        if starts_with_ns?(key_namespace, @namespace)
-          existing = store[property]
-          if existing.nil?
-            store[property] = { namespace: key_namespace, value: value }
-          elsif existing[:namespace].split(NAMESPACE_DELIMITER).size < key_namespace.split(NAMESPACE_DELIMITER).size
-            store[property] = { namespace: key_namespace, value: value }
+      @config_loader.calc_config.each do |key, config_resolver_obj|
+        config = config_resolver_obj[:config]
+        sortable = config.rows.map do |row|
+          if row.project_env_id != 0
+            if row.project_env_id == @project_env_id
+              if !row.namespace.empty?
+                (starts_with, count) = starts_with_ns?(row.namespace, @namespace)
+                # rubocop:disable BlockNesting
+                { sortable: 2 + count, match: "nm:#{row.namespace}", value: row.value, config: config} if starts_with
+              else
+                { sortable: 1, match: "env:#{row.project_env_id}", value: row.value, config: config}
+              end
+            end
+          else
+            match = config_resolver_obj[:match] || "default"
+            { sortable: 0, match: match, value: row.value, config: config}
           end
-        end
+        end.compact
+        to_store = sortable.sort_by { |h| h[:sortable] }.last
+        to_store[:source] = config_resolver_obj[:source]
+        store[key] = to_store
       end
+
       @lock.with_write_lock do
         @local_store = store
       end

data/lib/prefab/error.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+module Prefab
+  class Error < StandardError
+  end
+end

data/lib/prefab/errors/initialization_timeout_error.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Prefab
+  module Errors
+    class InitializationTimeoutError < Prefab::Error
+      def initialize(timeout_sec, key)
+        message = "Prefab couldn't initialize in #{timeout_sec} second timeout. Trying to fetch key `#{key}`."
+
+        super(message)
+      end
+    end
+  end
+end