elasticgraph-support 0.18.0.0

data/lib/elastic_graph/support/hash_util.rb

@@ -0,0 +1,191 @@
+# Copyright 2024 Block, Inc.
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+#
+# frozen_string_literal: true
+
+module ElasticGraph
+  module Support
+    class HashUtil
+      # Recursively transforms any hash keys in the given object to string keys, without
+      # mutating the provided argument.
+      def self.stringify_keys(object)
+        recursively_transform(object) do |key, value, hash|
+          hash[key.to_s] = value
+        end
+      end
+
+      # Recursively transforms any hash keys in the given object to symbol keys, without
+      # mutating the provided argument.
+      #
+      # Important note: this should never be used on untrusted input. Symbols are not GCd in
+      # Ruby in the same way as strings.
+      def self.symbolize_keys(object)
+        recursively_transform(object) do |key, value, hash|
+          hash[key.to_sym] = value
+        end
+      end
+
+      # Recursively prunes nil values from the hash, at any level of its structure, without
+      # mutating the provided argument. Key paths that are pruned are yielded to the caller
+      # to allow the caller to have awareness of what was pruned.
+      def self.recursively_prune_nils_from(object, &block)
+        recursively_prune_if(object, block, &:nil?)
+      end
+
+      # Recursively prunes nil values or empty hash/array values from the hash, at any level
+      # of its structure, without mutating the provided argument. Key paths that are pruned
+      # are yielded to the caller to allow the caller to have awareness of what was pruned.
+      def self.recursively_prune_nils_and_empties_from(object, &block)
+        recursively_prune_if(object, block) do |value|
+          if value.is_a?(::Hash) || value.is_a?(::Array)
+            value.empty?
+          else
+            value.nil?
+          end
+        end
+      end
+
+      # Recursively flattens the provided source hash, converting keys to strings along the way
+      # with dots used to separate nested parts. For example:
+      #
+      # flatten_and_stringify_keys({ a: { b: 3 }, c: 5 }, prefix: "foo") returns:
+      # { "foo.a.b" => 3, "foo.c" => 5 }
+      def self.flatten_and_stringify_keys(source_hash, prefix: nil)
+        # @type var flat_hash: ::Hash[::String, untyped]
+        flat_hash = {}
+        prefix = prefix ? "#{prefix}." : ""
+        # `_ =` is needed by steep because it thinks `prefix` could be `nil` in spite of the above line.
+        populate_flat_hash(source_hash, _ = prefix, flat_hash)
+        flat_hash
+      end
+
+      # Recursively merges the values from `hash2` into `hash1`, without mutating either `hash1` or `hash2`.
+      # When a key is in both `hash2` and `hash1`, takes the value from `hash2` just like `Hash#merge` does.
+      def self.deep_merge(hash1, hash2)
+        # `_ =` needed to satisfy steep--the types here are quite complicated.
+        _ = hash1.merge(hash2) do |key, hash1_value, hash2_value|
+          if ::Hash === hash1_value && ::Hash === hash2_value
+            deep_merge(hash1_value, hash2_value)
+          else
+            hash2_value
+          end
+        end
+      end
+
+      # Fetches a list of (potentially) nested value from a hash. The `key_path` is expected
+      # to be a string with dots between the nesting levels (e.g. `foo.bar`). Returns `[]` if
+      # the value at any parent key is `nil`. Returns a flat array of values if the structure
+      # at any level is an array.
+      #
+      # Raises an error if the key is not found unless a default block is provided.
+      # Raises an error if any parent value is not a hash as expected.
+      # Raises an error if the provided path is not a full path to a leaf in the nested structure.
+      def self.fetch_leaf_values_at_path(hash, key_path, &default)
+        do_fetch_leaf_values_at_path(hash, key_path.split("."), 0, &default)
+      end
+
+      # Fetches a single value from the hash at the given path. The `key_path` is expected
+      # to be a string with dots between the nesting levels (e.g. `foo.bar`).
+      #
+      # If any parent value is not a hash as expected, raises an error.
+      # If the key at any level is not found, yields to the provided block (which can provide a default value)
+      # or raises an error if no block is provided.
+      def self.fetch_value_at_path(hash, key_path)
+        path_parts = key_path.split(".")
+
+        path_parts.each.with_index(1).reduce(hash) do |inner_hash, (key, num_parts)|
+          if inner_hash.is_a?(::Hash)
+            inner_hash.fetch(key) do
+              missing_path = path_parts.first(num_parts).join(".")
+              return yield missing_path if block_given?
+              raise KeyError, "Key not found: #{missing_path.inspect}"
+            end
+          else
+            raise KeyError, "Value at key #{path_parts.first(num_parts - 1).join(".").inspect} is not a `Hash` as expected; " \
+              "instead, was a `#{(_ = inner_hash).class}`"
+          end
+        end
+      end
+
+      private_class_method def self.recursively_prune_if(object, notify_pruned_path)
+        recursively_transform(object) do |key, value, hash, key_path|
+          if yield(value)
+            notify_pruned_path&.call(key_path)
+          else
+            hash[key] = value
+          end
+        end
+      end
+
+      private_class_method def self.recursively_transform(object, key_path = nil, &hash_entry_handler)
+        case object
+        when ::Hash
+          # @type var initial: ::Hash[key, value]
+          initial = {}
+          object.each_with_object(initial) do |(key, value), hash|
+            updated_path = key_path ? "#{key_path}.#{key}" : key.to_s
+            value = recursively_transform(value, updated_path, &hash_entry_handler)
+            hash_entry_handler.call(key, value, hash, updated_path)
+          end
+        when ::Array
+          object.map.with_index do |item, index|
+            recursively_transform(item, "#{key_path}[#{index}]", &hash_entry_handler)
+          end
+        else
+          object
+        end
+      end
+
+      private_class_method def self.populate_flat_hash(source_hash, prefix, flat_hash)
+        source_hash.each do |key, value|
+          if value.is_a?(::Hash)
+            populate_flat_hash(value, "#{prefix}#{key}.", flat_hash)
+          elsif value.is_a?(::Array) && value.grep(::Hash).any?
+            raise ArgumentError, "`flatten_and_stringify_keys` cannot handle nested arrays of hashes, but got: #{value.inspect}"
+          else
+            flat_hash["#{prefix}#{key}"] = value
+          end
+        end
+      end
+
+      private_class_method def self.do_fetch_leaf_values_at_path(object, path_parts, level_index, &default)
+        if level_index == path_parts.size
+          if object.is_a?(::Hash)
+            raise KeyError, "Key was not a path to a leaf field: #{path_parts.join(".").inspect}"
+          else
+            return Array(object)
+          end
+        end
+
+        case object
+        when nil
+          []
+        when ::Hash
+          key = path_parts[level_index]
+          if object.key?(key)
+            do_fetch_leaf_values_at_path(object.fetch(key), path_parts, level_index + 1, &default)
+          else
+            missing_path = path_parts.first(level_index + 1).join(".")
+            if default
+              Array(default.call(missing_path))
+            else
+              raise KeyError, "Key not found: #{missing_path.inspect}"
+            end
+          end
+        when ::Array
+          object.flat_map do |element|
+            do_fetch_leaf_values_at_path(element, path_parts, level_index, &default)
+          end
+        else
+          # Note: we intentionally do not put the value (`current_level_hash`) in the
+          # error message, as that would risk leaking PII. But the class of the value should be OK.
+          raise KeyError, "Value at key #{path_parts.first(level_index).join(".").inspect} is not a `Hash` as expected; " \
+            "instead, was a `#{object.class}`"
+        end
+      end
+    end
+  end
+end

data/lib/elastic_graph/support/logger.rb

@@ -0,0 +1,82 @@
+# Copyright 2024 Block, Inc.
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+#
+# frozen_string_literal: true
+
+require "elastic_graph/error"
+require "json"
+require "logger"
+require "pathname"
+
+module ElasticGraph
+  module Support
+    module Logger
+      # Builds a logger instance from the given parsed YAML config.
+      def self.from_parsed_yaml(parsed_yaml)
+        Factory.build(config: Config.from_parsed_yaml(parsed_yaml))
+      end
+
+      module Factory
+        def self.build(config:, device: nil)
+          ::Logger.new(
+            device || config.prepared_device,
+            level: config.level,
+            formatter: config.formatter
+          )
+        end
+      end
+
+      class JSONAwareFormatter
+        def initialize
+          @original_formatter = ::Logger::Formatter.new
+        end
+
+        def call(severity, datetime, progname, msg)
+          msg = msg.is_a?(::Hash) ? ::JSON.generate(msg, space: " ") : msg
+          @original_formatter.call(severity, datetime, progname, msg)
+        end
+      end
+
+      class Config < ::Data.define(
+        # Determines what severity level we log. Valid values are `DEBUG`, `INFO`, `WARN`,
+        # `ERROR`, `FATAL` and `UNKNOWN`.
+        :level,
+        # Determines where we log to. Must be a string. "stdout" or "stderr" are interpreted
+        # as being those output streams; any other value is assumed to be a file path.
+        :device,
+        # Object used to format log messages. Defaults to an instance of `JSONAwareFormatter`.
+        :formatter
+      )
+        def prepared_device
+          case device
+          when "stdout" then $stdout
+          when "stderr" then $stderr
+          else
+            ::Pathname.new(device).parent.mkpath
+            device
+          end
+        end
+
+        def self.from_parsed_yaml(hash)
+          hash = hash.fetch("logger")
+          extra_keys = hash.keys - EXPECTED_KEYS
+
+          unless extra_keys.empty?
+            raise ConfigError, "Unknown `logger` config settings: #{extra_keys.join(", ")}"
+          end
+
+          new(
+            level: hash["level"] || "INFO",
+            device: hash.fetch("device"),
+            formatter: ::Object.const_get(hash.fetch("formatter", JSONAwareFormatter.name)).new
+          )
+        end
+
+        EXPECTED_KEYS = members.map(&:to_s)
+      end
+    end
+  end
+end

data/lib/elastic_graph/support/memoizable_data.rb

@@ -0,0 +1,147 @@
+# Copyright 2024 Block, Inc.
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+#
+# frozen_string_literal: true
+
+require "delegate"
+require "stringio"
+
+module ElasticGraph
+  module Support
+    # `::Data.define` in Ruby 3.2+ is *very* handy for defining immutable value objects. However, one annoying
+    # aspect: instances are frozen, which gets in the way of defining a memoized method (e.g. a method that
+    # caches the result of an expensive computation). While memoization is not always safe (e.g. if you rely
+    # on an impure side-effect...) it's safe if what you're memoizing is a pure function of the immutable state
+    # of the value object. We rely on that very heavily in ElasticGraph (and used it with a prior "value objects"
+    # library we use before upgrading to Ruby 3.2).
+    #
+    # This abstraction aims to behave just like `::Data.define`, but with the added ability to define memoized methods.
+    # It makes this possible by combining `::Data.define` with `SimpleDelegator`: that defines a data class, but then
+    # wraps instances of it in a `SimpleDelegator` instance which is _not_ frozen. The memoized methods can then be
+    # defined on the wrapper.
+    #
+    # Note: getting this code to typecheck with steep is quite difficult, so we're just skipping it.
+    __skip__ =
+      module MemoizableData
+        # Defines a data class using the provided attributes.
+        #
+        # A block can be provided in order to define custom methods (including memoized methods!) or to override
+        # `initialize` in order to provide field defaults.
+        def self.define(*attributes, &block)
+          data_class = ::Data.define(*attributes)
+
+          DelegateClass(data_class) do
+            # Store a reference to our wrapped data class so we can use it in `ClassMethods` below.
+            const_set(:DATA_CLASS, data_class)
+
+            # Define default version of` after_initialize`. This is a hook that a user may override.
+            # standard:disable Lint/NestedMethodDefinition
+            private def after_initialize
+            end
+            # standard:enable Lint/NestedMethodDefinition
+
+            # If a block is provided, we evaluate it so that it can define memoized methods.
+            if block
+              original_initialize = instance_method(:initialize)
+              module_eval(&block)
+
+              # It's useful for the caller to be define `initialize` in order to provide field defaults, as
+              # shown in the `Data` docs:
+              #
+              # https://rubyapi.org/3.2/o/data
+              #
+              # However, to make that work, we need the `initialize` definition to be included in the data class,
+              # rather than in our `DelegateClass` wrapper.
+              #
+              # Here we detect when the block defines an `initialize` method.
+              if instance_method(:initialize) != original_initialize
+                # To mix the `initialize` override into the data class, we re-evaluate the block in a new module here.
+                # The module ignores all method definitions except `initialize`.
+                init_override_module = ::Module.new do
+                  # We want to ignore all methods except the `initialize` method so that this module only contains `initialize`.
+                  def self.method_added(method_name)
+                    remove_method(method_name) unless method_name == :initialize
+                  end
+
+                  module_eval(&block)
+                end
+
+                data_class.include(init_override_module)
+              end
+            end
+
+            # `DelegateClass` objects are mutable via the `__setobj__` method. We don't want to allow mutation, so we undefine it here.
+            undef_method :__setobj__
+
+            prepend MemoizableData::InstanceMethods
+            extend MemoizableData::ClassMethods
+          end
+        end
+
+        module InstanceMethods
+          # SimpleDelegator#== automatically defines `==` so that it unwraps the wrapped type and calls `==` on it.
+          # However, the wrapped type doesn't automatically define `==` when given an equivalent wrapped instance.
+          #
+          # For `==` to work correctly we need to unwrap _both_ sides before delegating, which this takes care of.
+          def ==(other)
+            case other
+            when MemoizableData::InstanceMethods
+              __getobj__ == other.__getobj__
+            else
+              super
+            end
+          end
+
+          # `with` is a standard `Data` API that returns a new instance with the specified fields updated.
+          #
+          # Since `DelegateClass` delegates all methods to the wrapped object, `with` will return an instance of the
+          # data class and not our wrapper. To overcome that, we redefine it here so that the new instance is re-wrapped.
+          def with(**updates)
+            # Note: we intentionally do _not_ `super` to the `Date#with` method here, because in Ruby 3.2 it has a bug that
+            # impacts us: `with` does not call `initialize` as it should. Some of our value classes (based on the old `values` gem)
+            # depend on this behavior, so here we work around it by delegating to `new` after merging the attributes.
+            #
+            # This bug is fixed in Ruby 3.3 so we should be able to revert back to an implementation that delegates with `super`
+            # after we are on Ruby 3.3. For more info, see:
+            # - https://bugs.ruby-lang.org/issues/19259
+            # - https://github.com/ruby/ruby/pull/7031
+            self.class.new(**to_h.merge(updates))
+          end
+        end
+
+        module ClassMethods
+          # `new` on a `SimpleDelegator` class accepts an instance of the wrapped type to wrap. `MemoizableData` is intended to
+          # hide the wrapping we're doing here, so here we want `new` to accept the direct arguments that `new` on the `Data` class
+          # would accept. Here we instantiate the data class and the wrap it.
+          def new(*args, **kwargs)
+            data_instance = self::DATA_CLASS.new(*args, **kwargs)
+
+            # Here we re-implement `new` (rather than using `super`) because `initialize` may be overridden.
+            allocate.instance_eval do
+              # Match `__setobj__` behavior: https://github.com/ruby/ruby/blob/v3_2_2/lib/delegate.rb#L411
+              @delegate_dc_obj = data_instance
+              after_initialize
+              self
+            end
+          end
+
+          # `SimpleDelegator` delegates instance methods but not class methods. This is a standard `Data` class method
+          # that is worth delegating.
+          def members
+            self::DATA_CLASS.members
+          end
+
+          def method_added(method_name)
+            return unless method_name == :initialize
+
+            raise "`#{name}` overrides `initialize` in a subclass of `#{MemoizableData.name}`, but that can break things. Instead:\n\n" \
+              "  1) If you want to coerce field values or provide default field values, define `initialize` in a block passed to `#{MemoizableData.name}.define`.\n" \
+              "  2) If you want to perform some post-initialization setup, define an `after_initialize` method."
+          end
+        end
+      end
+  end
+end

data/lib/elastic_graph/support/monotonic_clock.rb

@@ -0,0 +1,20 @@
+# Copyright 2024 Block, Inc.
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+#
+# frozen_string_literal: true
+
+module ElasticGraph
+  module Support
+    # A simple abstraction that provides a monotonic clock.
+    class MonotonicClock
+      # Returns an abstract "now" value in integer milliseconds, suitable for calculating
+      # a duration or deadline, without being impacted by leap seconds, etc.
+      def now_in_ms
+        ::Process.clock_gettime(::Process::CLOCK_MONOTONIC, :millisecond)
+      end
+    end
+  end
+end

data/lib/elastic_graph/support/threading.rb

@@ -0,0 +1,42 @@
+# Copyright 2024 Block, Inc.
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+#
+# frozen_string_literal: true
+
+module ElasticGraph
+  module Support
+    module Threading
+      # Like Enumerable#map, but performs the map in parallel using one thread per list item.
+      # Exceptions that happen in the threads will propagate to the caller at the end.
+      # Due to Ruby's GVL, this will never be helpful for pure computation, but can be
+      # quite helpful when dealing with blocking I/O. However, the cost of threads is
+      # such that this method should not be used when you have a large list of items to
+      # map over (say, hundreds or thousands of items or more).
+      def self.parallel_map(items)
+        threads = _ = items.map do |item|
+          ::Thread.new do
+            # Disable reporting of exceptions. We use `value` at the end of this method, which
+            # propagates any exception that happened in the thread to the calling thread. If
+            # this is true (the default), then the exception is also printed to $stderr which
+            # is quite noisy.
+            ::Thread.current.report_on_exception = false
+
+            yield item
+          end
+        end
+
+        # `value` here either returns the value of the final expression in the thread, or raises
+        # whatever exception happened in the thread. `join` doesn't propagate the exception in
+        # the same way, so we always want to use `Thread#value` even if we are just using threads
+        # for side effects.
+        threads.map(&:value)
+      rescue => e
+        e.set_backtrace(e.backtrace + caller)
+        raise e
+      end
+    end
+  end
+end