launchdarkly-server-sdk 5.5.7

data/lib/ldclient-rb/impl.rb

@@ -0,0 +1,13 @@
+
+module LaunchDarkly
+  #
+  # Internal implementation classes. Everything in this module should be considered unsupported
+  # and subject to change.
+  #
+  # @since 5.5.0
+  # @private
+  #
+  module Impl
+    # code is in ldclient-rb/impl/
+  end
+end

data/lib/ldclient-rb/impl/integrations/consul_impl.rb

@@ -0,0 +1,158 @@
+require "json"
+
+module LaunchDarkly
+  module Impl
+    module Integrations
+      module Consul
+        #
+        # Internal implementation of the Consul feature store, intended to be used with CachingStoreWrapper.
+        #
+        class ConsulFeatureStoreCore
+          begin
+            require "diplomat"
+            CONSUL_ENABLED = true
+          rescue ScriptError, StandardError
+            CONSUL_ENABLED = false
+          end
+
+          def initialize(opts)
+            if !CONSUL_ENABLED
+              raise RuntimeError.new("can't use Consul feature store without the 'diplomat' gem")
+            end
+
+            @prefix = (opts[:prefix] || LaunchDarkly::Integrations::Consul.default_prefix) + '/'
+            @logger = opts[:logger] || Config.default_logger
+            Diplomat.configuration = opts[:consul_config] if !opts[:consul_config].nil?
+            Diplomat.configuration.url = opts[:url] if !opts[:url].nil?
+            @logger.info("ConsulFeatureStore: using Consul host at #{Diplomat.configuration.url}")
+          end
+
+          def init_internal(all_data)
+            # Start by reading the existing keys; we will later delete any of these that weren't in all_data.
+            unused_old_keys = Set.new
+            keys = Diplomat::Kv.get(@prefix, { keys: true, recurse: true }, :return)
+            unused_old_keys.merge(keys) if keys != ""
+
+            ops = []
+            num_items = 0
+
+            # Insert or update every provided item
+            all_data.each do |kind, items|
+              items.values.each do |item|
+                value = item.to_json
+                key = item_key(kind, item[:key])
+                ops.push({ 'KV' => { 'Verb' => 'set', 'Key' => key, 'Value' => value } })
+                unused_old_keys.delete(key)
+                num_items = num_items + 1
+              end
+            end
+
+            # Now delete any previously existing items whose keys were not in the current data
+            unused_old_keys.each do |key|
+              ops.push({ 'KV' => { 'Verb' => 'delete', 'Key' => key } })
+            end
+    
+            # Now set the special key that we check in initialized_internal?
+            ops.push({ 'KV' => { 'Verb' => 'set', 'Key' => inited_key, 'Value' => '' } })
+            
+            ConsulUtil.batch_operations(ops)
+
+            @logger.info { "Initialized database with #{num_items} items" }
+          end
+
+          def get_internal(kind, key)
+            value = Diplomat::Kv.get(item_key(kind, key), {}, :return)  # :return means "don't throw an error if not found"
+            (value.nil? || value == "") ? nil : JSON.parse(value, symbolize_names: true)
+          end
+
+          def get_all_internal(kind)
+            items_out = {}
+            results = Diplomat::Kv.get(kind_key(kind), { recurse: true }, :return)
+            (results == "" ? [] : results).each do |result|
+              value = result[:value]
+              if !value.nil?
+                item = JSON.parse(value, symbolize_names: true)
+                items_out[item[:key].to_sym] = item
+              end
+            end
+            items_out
+          end
+
+          def upsert_internal(kind, new_item)
+            key = item_key(kind, new_item[:key])
+            json = new_item.to_json
+
+            # We will potentially keep retrying indefinitely until someone's write succeeds
+            while true
+              old_value = Diplomat::Kv.get(key, { decode_values: true }, :return)
+              if old_value.nil? || old_value == ""
+                mod_index = 0
+              else
+                old_item = JSON.parse(old_value[0]["Value"], symbolize_names: true)
+                # Check whether the item is stale. If so, don't do the update (and return the existing item to
+                # FeatureStoreWrapper so it can be cached)
+                if old_item[:version] >= new_item[:version]
+                  return old_item
+                end
+                mod_index = old_value[0]["ModifyIndex"]
+              end
+
+              # Otherwise, try to write. We will do a compare-and-set operation, so the write will only succeed if
+              # the key's ModifyIndex is still equal to the previous value. If the previous ModifyIndex was zero,
+              # it means the key did not previously exist and the write will only succeed if it still doesn't exist.
+              success = Diplomat::Kv.put(key, json, cas: mod_index)
+              return new_item if success
+
+              # If we failed, retry the whole shebang
+              @logger.debug { "Concurrent modification detected, retrying" }
+            end
+          end
+
+          def initialized_internal?
+            # Unfortunately we need to use exceptions here, instead of the :return parameter, because with
+            # :return there's no way to distinguish between a missing value and an empty string.
+            begin
+              Diplomat::Kv.get(inited_key, {})
+              true
+            rescue Diplomat::KeyNotFound
+              false
+            end
+          end
+
+          def stop
+            # There's no Consul client instance to dispose of
+          end
+
+          private
+
+          def item_key(kind, key)
+            kind_key(kind) + key.to_s
+          end
+
+          def kind_key(kind)
+            @prefix + kind[:namespace] + '/'
+          end
+          
+          def inited_key
+            @prefix + '$inited'
+          end
+        end
+
+        class ConsulUtil
+          #
+          # Submits as many transactions as necessary to submit all of the given operations.
+          # The ops array is consumed.
+          #
+          def self.batch_operations(ops)
+            batch_size = 64  # Consul can only do this many at a time
+            while true
+              chunk = ops.shift(batch_size)
+              break if chunk.empty?
+              Diplomat::Kv.txn(chunk)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ldclient-rb/impl/integrations/dynamodb_impl.rb

@@ -0,0 +1,228 @@
+require "json"
+
+module LaunchDarkly
+  module Impl
+    module Integrations
+      module DynamoDB
+        #
+        # Internal implementation of the DynamoDB feature store, intended to be used with CachingStoreWrapper.
+        #
+        class DynamoDBFeatureStoreCore
+          begin
+            require "aws-sdk-dynamodb"
+            AWS_SDK_ENABLED = true
+          rescue ScriptError, StandardError
+            begin
+              require "aws-sdk"
+              AWS_SDK_ENABLED = true
+            rescue ScriptError, StandardError
+              AWS_SDK_ENABLED = false
+            end
+          end
+
+          PARTITION_KEY = "namespace"
+          SORT_KEY = "key"
+
+          VERSION_ATTRIBUTE = "version"
+          ITEM_JSON_ATTRIBUTE = "item"
+
+          def initialize(table_name, opts)
+            if !AWS_SDK_ENABLED
+              raise RuntimeError.new("can't use DynamoDB feature store without the aws-sdk or aws-sdk-dynamodb gem")
+            end
+
+            @table_name = table_name
+            @prefix = opts[:prefix]
+            @logger = opts[:logger] || Config.default_logger
+
+            if !opts[:existing_client].nil?
+              @client = opts[:existing_client]
+            else
+              @client = Aws::DynamoDB::Client.new(opts[:dynamodb_opts] || {})
+            end
+
+            @logger.info("DynamoDBFeatureStore: using DynamoDB table \"#{table_name}\"")
+          end
+
+          def init_internal(all_data)
+            # Start by reading the existing keys; we will later delete any of these that weren't in all_data.
+            unused_old_keys = read_existing_keys(all_data.keys)
+
+            requests = []
+            num_items = 0
+
+            # Insert or update every provided item
+            all_data.each do |kind, items|
+              items.values.each do |item|
+                requests.push({ put_request: { item: marshal_item(kind, item) } })
+                unused_old_keys.delete([ namespace_for_kind(kind), item[:key] ])
+                num_items = num_items + 1
+              end
+            end
+
+            # Now delete any previously existing items whose keys were not in the current data
+            unused_old_keys.each do |tuple|
+              del_item = make_keys_hash(tuple[0], tuple[1])
+              requests.push({ delete_request: { key: del_item } })
+            end
+    
+            # Now set the special key that we check in initialized_internal?
+            inited_item = make_keys_hash(inited_key, inited_key)
+            requests.push({ put_request: { item: inited_item } })
+
+            DynamoDBUtil.batch_write_requests(@client, @table_name, requests)
+
+            @logger.info { "Initialized table #{@table_name} with #{num_items} items" }
+          end
+
+          def get_internal(kind, key)
+            resp = get_item_by_keys(namespace_for_kind(kind), key)
+            unmarshal_item(resp.item)
+          end
+
+          def get_all_internal(kind)
+            items_out = {}
+            req = make_query_for_kind(kind)
+            while true
+              resp = @client.query(req)
+              resp.items.each do |item|
+                item_out = unmarshal_item(item)
+                items_out[item_out[:key].to_sym] = item_out
+              end
+              break if resp.last_evaluated_key.nil? || resp.last_evaluated_key.length == 0
+              req.exclusive_start_key = resp.last_evaluated_key
+            end
+            items_out
+          end
+
+          def upsert_internal(kind, new_item)
+            encoded_item = marshal_item(kind, new_item)
+            begin
+              @client.put_item({
+                table_name: @table_name,
+                item: encoded_item,
+                condition_expression: "attribute_not_exists(#namespace) or attribute_not_exists(#key) or :version > #version",
+                expression_attribute_names: {
+                  "#namespace" => PARTITION_KEY,
+                  "#key" => SORT_KEY,
+                  "#version" => VERSION_ATTRIBUTE
+                },
+                expression_attribute_values: {
+                  ":version" => new_item[:version]
+                }
+              })
+              new_item
+            rescue Aws::DynamoDB::Errors::ConditionalCheckFailedException
+              # The item was not updated because there's a newer item in the database.
+              # We must now read the item that's in the database and return it, so CachingStoreWrapper can cache it.
+              get_internal(kind, new_item[:key])
+            end
+          end
+
+          def initialized_internal?
+            resp = get_item_by_keys(inited_key, inited_key)
+            !resp.item.nil? && resp.item.length > 0
+          end
+
+          def stop
+            # AWS client doesn't seem to have a close method
+          end
+
+          private
+
+          def prefixed_namespace(base_str)
+            (@prefix.nil? || @prefix == "") ? base_str : "#{@prefix}:#{base_str}"
+          end
+
+          def namespace_for_kind(kind)
+            prefixed_namespace(kind[:namespace])
+          end
+
+          def inited_key
+            prefixed_namespace("$inited")
+          end
+
+          def make_keys_hash(namespace, key)
+            {
+              PARTITION_KEY => namespace,
+              SORT_KEY => key
+            }
+          end
+
+          def make_query_for_kind(kind)
+            {
+              table_name: @table_name,
+              consistent_read: true,
+              key_conditions: {
+                PARTITION_KEY => {
+                  comparison_operator: "EQ",
+                  attribute_value_list: [ namespace_for_kind(kind) ]
+                }
+              }
+            }
+          end
+
+          def get_item_by_keys(namespace, key)
+            @client.get_item({
+              table_name: @table_name,
+              key: make_keys_hash(namespace, key)
+            })
+          end
+
+          def read_existing_keys(kinds)
+            keys = Set.new
+            kinds.each do |kind|
+              req = make_query_for_kind(kind).merge({
+                projection_expression: "#namespace, #key",
+                expression_attribute_names: {
+                  "#namespace" => PARTITION_KEY,
+                  "#key" => SORT_KEY
+                }
+              })
+              while true
+                resp = @client.query(req)
+                resp.items.each do |item|
+                  namespace = item[PARTITION_KEY]
+                  key = item[SORT_KEY]
+                  keys.add([ namespace, key ])
+                end
+                break if resp.last_evaluated_key.nil? || resp.last_evaluated_key.length == 0
+                req.exclusive_start_key = resp.last_evaluated_key
+              end
+            end
+            keys
+          end
+
+          def marshal_item(kind, item)
+            make_keys_hash(namespace_for_kind(kind), item[:key]).merge({
+              VERSION_ATTRIBUTE => item[:version],
+              ITEM_JSON_ATTRIBUTE => item.to_json
+            })
+          end
+
+          def unmarshal_item(item)
+            return nil if item.nil? || item.length == 0
+            json_attr = item[ITEM_JSON_ATTRIBUTE]
+            raise RuntimeError.new("DynamoDB map did not contain expected item string") if json_attr.nil?
+            JSON.parse(json_attr, symbolize_names: true)
+          end
+        end
+
+        class DynamoDBUtil
+          #
+          # Calls client.batch_write_item as many times as necessary to submit all of the given requests.
+          # The requests array is consumed.
+          #
+          def self.batch_write_requests(client, table, requests)
+            batch_size = 25
+            while true
+              chunk = requests.shift(batch_size)
+              break if chunk.empty?
+              client.batch_write_item({ request_items: { table => chunk } })
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ldclient-rb/impl/integrations/redis_impl.rb

@@ -0,0 +1,155 @@
+require "concurrent/atomics"
+require "json"
+
+module LaunchDarkly
+  module Impl
+    module Integrations
+      module Redis
+        #
+        # Internal implementation of the Redis feature store, intended to be used with CachingStoreWrapper.
+        #
+        class RedisFeatureStoreCore
+          begin
+            require "redis"
+            require "connection_pool"
+            REDIS_ENABLED = true
+          rescue ScriptError, StandardError
+            REDIS_ENABLED = false
+          end
+
+          def initialize(opts)
+            if !REDIS_ENABLED
+              raise RuntimeError.new("can't use Redis feature store because one of these gems is missing: redis, connection_pool")
+            end
+
+            @redis_opts = opts[:redis_opts] || Hash.new
+            if opts[:redis_url]
+              @redis_opts[:url] = opts[:redis_url]
+            end
+            if !@redis_opts.include?(:url)
+              @redis_opts[:url] = LaunchDarkly::Integrations::Redis::default_redis_url
+            end
+            max_connections = opts[:max_connections] || 16
+            @pool = opts[:pool] || ConnectionPool.new(size: max_connections) do
+              ::Redis.new(@redis_opts)
+            end
+            @prefix = opts[:prefix] || LaunchDarkly::Integrations::Redis::default_prefix
+            @logger = opts[:logger] || Config.default_logger
+            @test_hook = opts[:test_hook]  # used for unit tests, deliberately undocumented
+
+            @stopped = Concurrent::AtomicBoolean.new(false)
+
+            with_connection do |redis|
+              @logger.info("RedisFeatureStore: using Redis instance at #{redis.connection[:host]}:#{redis.connection[:port]} \
+      and prefix: #{@prefix}")
+            end
+          end
+
+          def init_internal(all_data)
+            count = 0
+            with_connection do |redis|
+              redis.multi do |multi|
+                all_data.each do |kind, items|
+                  multi.del(items_key(kind))
+                  count = count + items.count
+                  items.each do |key, item|
+                    multi.hset(items_key(kind), key, item.to_json)
+                  end
+                end
+                multi.set(inited_key, inited_key)
+              end
+            end
+            @logger.info { "RedisFeatureStore: initialized with #{count} items" }
+          end
+
+          def get_internal(kind, key)
+            with_connection do |redis|
+              get_redis(redis, kind, key)
+            end
+          end
+
+          def get_all_internal(kind)
+            fs = {}
+            with_connection do |redis|
+              hashfs = redis.hgetall(items_key(kind))
+              hashfs.each do |k, json_item|
+                f = JSON.parse(json_item, symbolize_names: true)
+                fs[k.to_sym] = f
+              end
+            end
+            fs
+          end
+
+          def upsert_internal(kind, new_item)
+            base_key = items_key(kind)
+            key = new_item[:key]
+            try_again = true
+            final_item = new_item
+            while try_again
+              try_again = false
+              with_connection do |redis|
+                redis.watch(base_key) do
+                  old_item = get_redis(redis, kind, key)
+                  before_update_transaction(base_key, key)
+                  if old_item.nil? || old_item[:version] < new_item[:version]
+                    result = redis.multi do |multi|
+                      multi.hset(base_key, key, new_item.to_json)
+                    end
+                    if result.nil?
+                      @logger.debug { "RedisFeatureStore: concurrent modification detected, retrying" }
+                      try_again = true
+                    end
+                  else
+                    final_item = old_item
+                    action = new_item[:deleted] ? "delete" : "update"
+                    @logger.warn { "RedisFeatureStore: attempted to #{action} #{key} version: #{old_item[:version]} \
+    in '#{kind[:namespace]}' with a version that is the same or older: #{new_item[:version]}" }
+                  end
+                  redis.unwatch
+                end
+              end
+            end
+            final_item
+          end
+
+          def initialized_internal?
+            with_connection { |redis| redis.exists(inited_key) }
+          end
+
+          def stop
+            if @stopped.make_true
+              @pool.shutdown { |redis| redis.close }
+            end
+          end
+
+          private
+
+          def before_update_transaction(base_key, key)
+            @test_hook.before_update_transaction(base_key, key) if !@test_hook.nil?
+          end
+
+          def items_key(kind)
+            @prefix + ":" + kind[:namespace]
+          end
+
+          def cache_key(kind, key)
+            kind[:namespace] + ":" + key.to_s
+          end
+
+          def inited_key
+            @prefix + ":$inited"
+          end
+
+          def with_connection
+            @pool.with { |redis| yield(redis) }
+          end
+
+          def get_redis(redis, kind, key)
+            json_item = redis.hget(items_key(kind), key)
+            json_item.nil? ? nil : JSON.parse(json_item, symbolize_names: true)
+          end
+        end
+      end
+    end
+  end
+end