ldclient-rb 5.4.3 → 5.5.0

data/lib/ldclient-rb/impl/integrations/dynamodb_impl.rb

@@ -0,0 +1,228 @@
+require "json"
+
+module LaunchDarkly
+  module Impl
+    module Integrations
+      module DynamoDB
+        #
+        # Internal implementation of the DynamoDB feature store, intended to be used with CachingStoreWrapper.
+        #
+        class DynamoDBFeatureStoreCore
+          begin
+            require "aws-sdk-dynamodb"
+            AWS_SDK_ENABLED = true
+          rescue ScriptError, StandardError
+            begin
+              require "aws-sdk"
+              AWS_SDK_ENABLED = true
+            rescue ScriptError, StandardError
+              AWS_SDK_ENABLED = false
+            end
+          end
+
+          PARTITION_KEY = "namespace"
+          SORT_KEY = "key"
+
+          VERSION_ATTRIBUTE = "version"
+          ITEM_JSON_ATTRIBUTE = "item"
+
+          def initialize(table_name, opts)
+            if !AWS_SDK_ENABLED
+              raise RuntimeError.new("can't use DynamoDB feature store without the aws-sdk or aws-sdk-dynamodb gem")
+            end
+
+            @table_name = table_name
+            @prefix = opts[:prefix]
+            @logger = opts[:logger] || Config.default_logger
+
+            if !opts[:existing_client].nil?
+              @client = opts[:existing_client]
+            else
+              @client = Aws::DynamoDB::Client.new(opts[:dynamodb_opts] || {})
+            end
+
+            @logger.info("DynamoDBFeatureStore: using DynamoDB table \"#{table_name}\"")
+          end
+
+          def init_internal(all_data)
+            # Start by reading the existing keys; we will later delete any of these that weren't in all_data.
+            unused_old_keys = read_existing_keys(all_data.keys)
+
+            requests = []
+            num_items = 0
+
+            # Insert or update every provided item
+            all_data.each do |kind, items|
+              items.values.each do |item|
+                requests.push({ put_request: { item: marshal_item(kind, item) } })
+                unused_old_keys.delete([ namespace_for_kind(kind), item[:key] ])
+                num_items = num_items + 1
+              end
+            end
+
+            # Now delete any previously existing items whose keys were not in the current data
+            unused_old_keys.each do |tuple|
+              del_item = make_keys_hash(tuple[0], tuple[1])
+              requests.push({ delete_request: { key: del_item } })
+            end
+    
+            # Now set the special key that we check in initialized_internal?
+            inited_item = make_keys_hash(inited_key, inited_key)
+            requests.push({ put_request: { item: inited_item } })
+
+            DynamoDBUtil.batch_write_requests(@client, @table_name, requests)
+
+            @logger.info { "Initialized table #{@table_name} with #{num_items} items" }
+          end
+
+          def get_internal(kind, key)
+            resp = get_item_by_keys(namespace_for_kind(kind), key)
+            unmarshal_item(resp.item)
+          end
+
+          def get_all_internal(kind)
+            items_out = {}
+            req = make_query_for_kind(kind)
+            while true
+              resp = @client.query(req)
+              resp.items.each do |item|
+                item_out = unmarshal_item(item)
+                items_out[item_out[:key].to_sym] = item_out
+              end
+              break if resp.last_evaluated_key.nil? || resp.last_evaluated_key.length == 0
+              req.exclusive_start_key = resp.last_evaluated_key
+            end
+            items_out
+          end
+
+          def upsert_internal(kind, new_item)
+            encoded_item = marshal_item(kind, new_item)
+            begin
+              @client.put_item({
+                table_name: @table_name,
+                item: encoded_item,
+                condition_expression: "attribute_not_exists(#namespace) or attribute_not_exists(#key) or :version > #version",
+                expression_attribute_names: {
+                  "#namespace" => PARTITION_KEY,
+                  "#key" => SORT_KEY,
+                  "#version" => VERSION_ATTRIBUTE
+                },
+                expression_attribute_values: {
+                  ":version" => new_item[:version]
+                }
+              })
+              new_item
+            rescue Aws::DynamoDB::Errors::ConditionalCheckFailedException
+              # The item was not updated because there's a newer item in the database.
+              # We must now read the item that's in the database and return it, so CachingStoreWrapper can cache it.
+              get_internal(kind, new_item[:key])
+            end
+          end
+
+          def initialized_internal?
+            resp = get_item_by_keys(inited_key, inited_key)
+            !resp.item.nil? && resp.item.length > 0
+          end
+
+          def stop
+            # AWS client doesn't seem to have a close method
+          end
+
+          private
+
+          def prefixed_namespace(base_str)
+            (@prefix.nil? || @prefix == "") ? base_str : "#{@prefix}:#{base_str}"
+          end
+
+          def namespace_for_kind(kind)
+            prefixed_namespace(kind[:namespace])
+          end
+
+          def inited_key
+            prefixed_namespace("$inited")
+          end
+
+          def make_keys_hash(namespace, key)
+            {
+              PARTITION_KEY => namespace,
+              SORT_KEY => key
+            }
+          end
+
+          def make_query_for_kind(kind)
+            {
+              table_name: @table_name,
+              consistent_read: true,
+              key_conditions: {
+                PARTITION_KEY => {
+                  comparison_operator: "EQ",
+                  attribute_value_list: [ namespace_for_kind(kind) ]
+                }
+              }
+            }
+          end
+
+          def get_item_by_keys(namespace, key)
+            @client.get_item({
+              table_name: @table_name,
+              key: make_keys_hash(namespace, key)
+            })
+          end
+
+          def read_existing_keys(kinds)
+            keys = Set.new
+            kinds.each do |kind|
+              req = make_query_for_kind(kind).merge({
+                projection_expression: "#namespace, #key",
+                expression_attribute_names: {
+                  "#namespace" => PARTITION_KEY,
+                  "#key" => SORT_KEY
+                }
+              })
+              while true
+                resp = @client.query(req)
+                resp.items.each do |item|
+                  namespace = item[PARTITION_KEY]
+                  key = item[SORT_KEY]
+                  keys.add([ namespace, key ])
+                end
+                break if resp.last_evaluated_key.nil? || resp.last_evaluated_key.length == 0
+                req.exclusive_start_key = resp.last_evaluated_key
+              end
+            end
+            keys
+          end
+
+          def marshal_item(kind, item)
+            make_keys_hash(namespace_for_kind(kind), item[:key]).merge({
+              VERSION_ATTRIBUTE => item[:version],
+              ITEM_JSON_ATTRIBUTE => item.to_json
+            })
+          end
+
+          def unmarshal_item(item)
+            return nil if item.nil? || item.length == 0
+            json_attr = item[ITEM_JSON_ATTRIBUTE]
+            raise RuntimeError.new("DynamoDB map did not contain expected item string") if json_attr.nil?
+            JSON.parse(json_attr, symbolize_names: true)
+          end
+        end
+
+        class DynamoDBUtil
+          #
+          # Calls client.batch_write_item as many times as necessary to submit all of the given requests.
+          # The requests array is consumed.
+          #
+          def self.batch_write_requests(client, table, requests)
+            batch_size = 25
+            while true
+              chunk = requests.shift(batch_size)
+              break if chunk.empty?
+              client.batch_write_item({ request_items: { table => chunk } })
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ldclient-rb/impl/integrations/redis_impl.rb

@@ -0,0 +1,155 @@
+require "concurrent/atomics"
+require "json"
+
+module LaunchDarkly
+  module Impl
+    module Integrations
+      module Redis
+        #
+        # Internal implementation of the Redis feature store, intended to be used with CachingStoreWrapper.
+        #
+        class RedisFeatureStoreCore
+          begin
+            require "redis"
+            require "connection_pool"
+            REDIS_ENABLED = true
+          rescue ScriptError, StandardError
+            REDIS_ENABLED = false
+          end
+
+          def initialize(opts)
+            if !REDIS_ENABLED
+              raise RuntimeError.new("can't use Redis feature store because one of these gems is missing: redis, connection_pool")
+            end
+
+            @redis_opts = opts[:redis_opts] || Hash.new
+            if opts[:redis_url]
+              @redis_opts[:url] = opts[:redis_url]
+            end
+            if !@redis_opts.include?(:url)
+              @redis_opts[:url] = LaunchDarkly::Integrations::Redis::default_redis_url
+            end
+            max_connections = opts[:max_connections] || 16
+            @pool = opts[:pool] || ConnectionPool.new(size: max_connections) do
+              ::Redis.new(@redis_opts)
+            end
+            @prefix = opts[:prefix] || LaunchDarkly::Integrations::Redis::default_prefix
+            @logger = opts[:logger] || Config.default_logger
+            @test_hook = opts[:test_hook]  # used for unit tests, deliberately undocumented
+
+            @stopped = Concurrent::AtomicBoolean.new(false)
+
+            with_connection do |redis|
+              @logger.info("RedisFeatureStore: using Redis instance at #{redis.connection[:host]}:#{redis.connection[:port]} \
+      and prefix: #{@prefix}")
+            end
+          end
+
+          def init_internal(all_data)
+            count = 0
+            with_connection do |redis|
+              redis.multi do |multi|
+                all_data.each do |kind, items|
+                  multi.del(items_key(kind))
+                  count = count + items.count
+                  items.each do |key, item|
+                    multi.hset(items_key(kind), key, item.to_json)
+                  end
+                end
+                multi.set(inited_key, inited_key)
+              end
+            end
+            @logger.info { "RedisFeatureStore: initialized with #{count} items" }
+          end
+
+          def get_internal(kind, key)
+            with_connection do |redis|
+              get_redis(redis, kind, key)
+            end
+          end
+
+          def get_all_internal(kind)
+            fs = {}
+            with_connection do |redis|
+              hashfs = redis.hgetall(items_key(kind))
+              hashfs.each do |k, json_item|
+                f = JSON.parse(json_item, symbolize_names: true)
+                fs[k.to_sym] = f
+              end
+            end
+            fs
+          end
+
+          def upsert_internal(kind, new_item)
+            base_key = items_key(kind)
+            key = new_item[:key]
+            try_again = true
+            final_item = new_item
+            while try_again
+              try_again = false
+              with_connection do |redis|
+                redis.watch(base_key) do
+                  old_item = get_redis(redis, kind, key)
+                  before_update_transaction(base_key, key)
+                  if old_item.nil? || old_item[:version] < new_item[:version]
+                    result = redis.multi do |multi|
+                      multi.hset(base_key, key, new_item.to_json)
+                    end
+                    if result.nil?
+                      @logger.debug { "RedisFeatureStore: concurrent modification detected, retrying" }
+                      try_again = true
+                    end
+                  else
+                    final_item = old_item
+                    action = new_item[:deleted] ? "delete" : "update"
+                    @logger.warn { "RedisFeatureStore: attempted to #{action} #{key} version: #{old_item[:version]} \
+    in '#{kind[:namespace]}' with a version that is the same or older: #{new_item[:version]}" }
+                  end
+                  redis.unwatch
+                end
+              end
+            end
+            final_item
+          end
+
+          def initialized_internal?
+            with_connection { |redis| redis.exists(inited_key) }
+          end
+
+          def stop
+            if @stopped.make_true
+              @pool.shutdown { |redis| redis.close }
+            end
+          end
+
+          private
+
+          def before_update_transaction(base_key, key)
+            @test_hook.before_update_transaction(base_key, key) if !@test_hook.nil?
+          end
+
+          def items_key(kind)
+            @prefix + ":" + kind[:namespace]
+          end
+
+          def cache_key(kind, key)
+            kind[:namespace] + ":" + key.to_s
+          end
+
+          def inited_key
+            @prefix + ":$inited"
+          end
+
+          def with_connection
+            @pool.with { |redis| yield(redis) }
+          end
+
+          def get_redis(redis, kind, key)
+            json_item = redis.hget(items_key(kind), key)
+            json_item.nil? ? nil : JSON.parse(json_item, symbolize_names: true)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ldclient-rb/impl/store_client_wrapper.rb

@@ -0,0 +1,47 @@
+require "ldclient-rb/interfaces"
+require "ldclient-rb/impl/store_data_set_sorter"
+
+module LaunchDarkly
+  module Impl
+    #
+    # Provides additional behavior that the client requires before or after feature store operations.
+    # Currently this just means sorting the data set for init(). In the future we may also use this
+    # to provide an update listener capability.
+    #
+    class FeatureStoreClientWrapper
+      include Interfaces::FeatureStore
+
+      def initialize(store)
+        @store = store
+      end
+
+      def init(all_data)
+        @store.init(FeatureStoreDataSetSorter.sort_all_collections(all_data))
+      end
+
+      def get(kind, key)
+        @store.get(kind, key)
+      end
+
+      def all(kind)
+        @store.all(kind)
+      end
+
+      def upsert(kind, item)
+        @store.upsert(kind, item)
+      end
+
+      def delete(kind, key, version)
+        @store.delete(kind, key, version)
+      end
+
+      def initialized?
+        @store.initialized?
+      end
+
+      def stop
+        @store.stop
+      end
+    end
+  end
+end

data/lib/ldclient-rb/impl/store_data_set_sorter.rb

@@ -0,0 +1,55 @@
+
+module LaunchDarkly
+  module Impl
+    #
+    # Implements a dependency graph ordering for data to be stored in a feature store. We must use this
+    # on every data set that will be passed to the feature store's init() method.
+    #
+    class FeatureStoreDataSetSorter
+      #
+      # Returns a copy of the input hash that has the following guarantees: the iteration order of the outer
+      # hash will be in ascending order by the VersionDataKind's :priority property (if any), and for each
+      # data kind that has a :get_dependency_keys function, the inner hash will have an iteration order
+      # where B is before A if A has a dependency on B.
+      #
+      # This implementation relies on the fact that hashes in Ruby have an iteration order that is the same
+      # as the insertion order. Also, due to the way we deserialize JSON received from LaunchDarkly, the
+      # keys in the inner hash will always be symbols.
+      #
+      def self.sort_all_collections(all_data)
+        outer_hash = {}
+        kinds = all_data.keys.sort_by { |k|
+          k[:priority].nil? ? k[:namespace].length : k[:priority]  # arbitrary order if priority is unknown
+        }
+        kinds.each do |kind|
+          items = all_data[kind]
+          outer_hash[kind] = self.sort_collection(kind, items)
+        end
+        outer_hash
+      end
+
+      def self.sort_collection(kind, input)
+        dependency_fn = kind[:get_dependency_keys]
+        return input if dependency_fn.nil? || input.empty?
+        remaining_items = input.clone
+        items_out = {}
+        while !remaining_items.empty?
+          # pick a random item that hasn't been updated yet
+          key, item = remaining_items.first
+          self.add_with_dependencies_first(item, dependency_fn, remaining_items, items_out)
+        end
+        items_out
+      end
+
+      def self.add_with_dependencies_first(item, dependency_fn, remaining_items, items_out)
+        item_key = item[:key].to_sym
+        remaining_items.delete(item_key)  # we won't need to visit this item again
+        dependency_fn.call(item).each do |dep_key|
+          dep_item = remaining_items[dep_key.to_sym]
+          self.add_with_dependencies_first(dep_item, dependency_fn, remaining_items, items_out) if !dep_item.nil?
+        end
+        items_out[item_key] = item
+      end
+    end
+  end
+end