julewire-core 1.0.0 → 1.1.0

data/lib/julewire/core/serialization/value_copy.rb

@@ -3,129 +3,123 @@
 module Julewire
   module Core
     module Serialization
+      module ValueCopyTruncation
+        private
+
+        def validate_optional_limit(value, name:)
+          return unless value
+
+          Validation.validate_integer_limit!(value, name: name)
+        end
+
+        def finish_hash(result, fields)
+          add_truncation_metadata!(result, fields)
+          finish_container(result, fields)
+        end
+
+        def finish_array(result, fields)
+          if @track_truncation && fields
+            result << freeze_container({ Serializer::TRUNCATION_METADATA_KEY.to_sym => truncation_metadata(fields) })
+          end
+          finish_container(result, fields)
+        end
+
+        def add_truncation_metadata!(result, fields)
+          return unless @track_truncation && fields
+
+          result[Serializer::TRUNCATION_METADATA_KEY.to_sym] = truncation_metadata(fields)
+        end
+
+        def finish_container(result, fields)
+          value = freeze_container(result)
+          fields ? mark_truncated(value) : value
+        end
+
+        def truncation_metadata(fields)
+          TruncationMetadata.build(
+            fields,
+            key_style: :symbol,
+            compact_limits: true,
+            freeze_values: @freeze_values,
+            max_array_items: @max_array_items,
+            max_depth: @max_depth,
+            max_hash_keys: @max_hash_keys,
+            max_string_bytes: @max_string_bytes
+          )
+        end
+
+        def mark_truncated(value)
+          @last_truncated = true
+          value
+        end
+
+        def consume_truncated
+          truncated = @last_truncated
+          @last_truncated = false
+          truncated
+        end
+
+        def append_truncation_field(fields, field)
+          TruncationMetadata.append_field(fields, field)
+        end
+      end
+      private_constant :ValueCopyTruncation
+
       class ValueCopy
         include ValueTraversal
+        include ValueCopyTruncation
 
         CIRCULAR_REFERENCE = Core::CIRCULAR_REFERENCE
         EMPTY_ARRAY = [].freeze
         EMPTY_HASH = {}.freeze
-        POOL_KEY = :julewire_core_value_copy_pool
-        private_constant :EMPTY_ARRAY, :EMPTY_HASH, :POOL_KEY
+        private_constant :EMPTY_ARRAY, :EMPTY_HASH
 
         class << self
-          def call(
+          def call( # rubocop:disable Metrics/ParameterLists
             value,
             compact_empty: false,
             freeze_values: false,
-            max_depth: Core::NORMALIZATION_MAX_DEPTH,
+            max_array_items: nil,
+            max_depth: NORMALIZATION_MAX_DEPTH,
+            max_hash_keys: nil,
+            max_string_bytes: nil,
+            preserve_truncation_metadata: false,
             symbolize_keys: false
           )
-            return copy_leaf(value, freeze_values: freeze_values) unless container?(value)
-
-            copy_with(
-              cached_copier(
-                compact_empty: compact_empty,
-                freeze_values: freeze_values,
-                max_depth: max_depth,
-                symbolize_keys: symbolize_keys
-              ),
-              value
-            )
-          end
-
-          def omitted_empty?(value)
-            value.nil? || (value.is_a?(Hash) && value.empty?) || (value.is_a?(Array) && value.empty?)
-          end
-
-          private
-
-          def container?(value) = value.is_a?(Hash) || value.is_a?(Array)
-
-          def cached_copier(compact_empty:, freeze_values:, max_depth:, symbolize_keys:)
-            # One copier per thread/options avoids per-record walker allocation.
-            pool = Thread.current.thread_variable_get(POOL_KEY)
-            unless pool
-              pool = {}
-              Thread.current.thread_variable_set(POOL_KEY, pool)
-            end
-
-            key = cache_key(
-              compact_empty: compact_empty,
-              freeze_values: freeze_values,
-              max_depth: max_depth,
-              symbolize_keys: symbolize_keys
-            )
-            pool[key] ||= new(
-              compact_empty: compact_empty,
-              freeze_values: freeze_values,
-              max_depth: max_depth,
-              symbolize_keys: symbolize_keys
-            )
-          end
-
-          def cache_key(compact_empty:, freeze_values:, max_depth:, symbolize_keys:)
-            depth_key = max_depth || -1
-            flags = 0
-            flags |= 1 if compact_empty
-            flags |= 2 if freeze_values
-            flags |= 4 if symbolize_keys
-            (depth_key << 3) | flags
-          end
-
-          def copy_with(copier, value)
-            return copier.call_reusable(value) unless copier.in_use?
-
             new(
-              compact_empty: copier.compact_empty,
-              freeze_values: copier.freeze_values,
-              max_depth: copier.max_depth,
-              symbolize_keys: copier.symbolize_keys
+              compact_empty:,
+              freeze_values:,
+              max_array_items:,
+              max_depth:,
+              max_hash_keys:,
+              max_string_bytes:,
+              preserve_truncation_metadata:,
+              symbolize_keys:
             ).call(value)
           end
 
-          def copy_leaf(value, freeze_values:)
-            return copy_string(value, freeze_values: freeze_values) if value.is_a?(String)
-            return copy_time(value, freeze_values: freeze_values) if value.is_a?(Time)
-
-            value
-          end
-
-          def copy_string(value, freeze_values:)
-            copy = value.frozen? ? value : value.dup
-            freeze_values ? copy.freeze : copy
-          end
-
-          def copy_time(value, freeze_values:)
-            return value unless freeze_values
-            return value if value.frozen?
-
-            value.dup.freeze
+          def omitted_empty?(value)
+            value.nil? || (value.is_a?(Hash) && value.empty?) || (value.is_a?(Array) && value.empty?)
           end
         end
 
-        attr_reader :compact_empty, :freeze_values, :max_depth, :symbolize_keys
-
-        def initialize(compact_empty:, freeze_values:, max_depth:, symbolize_keys:)
+        def initialize(compact_empty:, freeze_values:, max_array_items:, max_depth:, max_hash_keys:, max_string_bytes:,
+                       preserve_truncation_metadata:, symbolize_keys:)
           @compact_empty = compact_empty
           @freeze_values = freeze_values
+          @max_array_items = validate_optional_limit(max_array_items, name: :max_array_items)
           @max_depth = max_depth
+          @max_hash_keys = validate_optional_limit(max_hash_keys, name: :max_hash_keys)
+          @max_string_bytes = validate_optional_limit(max_string_bytes, name: :max_string_bytes)
+          @preserve_truncation_metadata = preserve_truncation_metadata
           @symbolize_keys = symbolize_keys
-          @in_use = false
+          @track_truncation = @max_array_items || @max_hash_keys || @max_string_bytes
         end
 
         def call(value)
           traverse(value) { |root, depth| copy_value(root, depth) }
         end
 
-        def call_reusable(value)
-          @in_use = true
-          call(value)
-        ensure
-          @in_use = false
-        end
-
-        def in_use? = @in_use
-
         private
 
         def copy_value(value, depth)
@@ -137,7 +131,7 @@ module Julewire
         end
 
         def copy_container(value, depth)
-          return copy_string(Serializer::MAX_DEPTH_VALUE) if depth_limited?(depth)
+          return mark_truncated(Serializer::MAX_DEPTH_VALUE) if depth == @max_depth
           return frozen_empty_container(value) if @freeze_values && value.empty?
 
           with_traversal_container(value, CIRCULAR_REFERENCE) do
@@ -145,52 +139,132 @@ module Julewire
           end
         end
 
-        def depth_limited?(depth)
-          @max_depth && depth >= @max_depth
-        end
-
         def frozen_empty_container(value)
           value.is_a?(Hash) ? EMPTY_HASH : EMPTY_ARRAY
         end
 
         def copy_hash(value, depth)
+          fields = nil
           result = {}
+          visited = 0
           value.each do |key, item|
+            if visited == @max_hash_keys
+              fields = append_truncation_field(fields, "hash_keys")
+              break
+            end
+
+            visited += 1
             next if @compact_empty && self.class.omitted_empty?(item)
 
-            copied = copy_value(item, depth + 1)
-            next if @compact_empty && self.class.omitted_empty?(copied)
+            fields = copy_hash_entry(result, fields, key, item, depth)
+          end
+          finish_hash(result, fields)
+        end
 
-            result[copy_key(key)] = copied
+        def copy_hash_entry(result, fields, key, item, depth)
+          return copy_truncation_metadata_entry(result, fields, key, item) if reserved_truncation_key?(key)
+
+          validate_symbolized_key_shape!(key)
+          copied = copy_value(item, depth + 1)
+          return fields if @compact_empty && self.class.omitted_empty?(copied)
+
+          copied_key = copied_key_value(key)
+          value_or_key_truncated = consume_truncated
+          result[copied_key] = copied
+          fields = append_truncation_field(fields, copied_key.to_s) if value_or_key_truncated
+          fields
+        end
+
+        def copy_truncation_metadata_entry(result, fields, key, item)
+          unless @preserve_truncation_metadata &&
+                 allowed_truncation_metadata_key?(key) &&
+                 TruncationMetadata.valid?(item, max_fields: truncation_metadata_field_limit)
+            raise_reserved_key!
           end
-          freeze_container(result)
+
+          result[copy_truncation_metadata_key(key)] = TruncationMetadata.copy(
+            item,
+            freeze_values: @freeze_values
+          )
+          fields
+        end
+
+        def truncation_metadata_field_limit
+          @max_array_items || Serializer::DEFAULT_MAX_ARRAY_ITEMS
+        end
+
+        def allowed_truncation_metadata_key?(key)
+          key.instance_of?(Symbol) || @symbolize_keys
+        end
+
+        def reserved_truncation_key?(key)
+          key == Serializer::TRUNCATION_METADATA_KEY || key == Serializer::TRUNCATION_METADATA_KEY.to_sym
+        end
+
+        def copy_truncation_metadata_key(key)
+          key.to_sym
+        end
+
+        def validate_symbolized_key_shape!(key)
+          return unless @symbolize_keys
+          return if key.is_a?(String) || key.instance_of?(Symbol)
+
+          raise TypeError, Fields::Internal::FIELD_KEY_ERROR
         end
 
         def copy_array(value, depth)
           result = []
+          visited = 0
+          array_items_truncated = false
+          array_item_values_truncated = false
           value.each do |item|
+            if visited == @max_array_items
+              array_items_truncated = true
+              break
+            end
+
+            visited += 1
             next if @compact_empty && self.class.omitted_empty?(item)
 
             copied = copy_value(item, depth + 1)
+            child_truncated = consume_truncated
             next if @compact_empty && self.class.omitted_empty?(copied)
 
             result << copied
+            array_item_values_truncated ||= child_truncated
           end
-          freeze_container(result)
+
+          finish_array(result, array_truncation_fields(array_items_truncated, array_item_values_truncated))
+        end
+
+        def array_truncation_fields(array_items_truncated, array_item_values_truncated)
+          return unless array_items_truncated || array_item_values_truncated
+
+          fields = []
+          fields << "array_item_values" if array_item_values_truncated
+          fields << "array_items" if array_items_truncated
+          fields
         end
 
-        def copy_key(key)
-          return key.to_sym if @symbolize_keys && key.is_a?(String)
-          return copy_string(key) if key.is_a?(String)
+        def copied_key_value(key)
+          return key unless key.is_a?(String)
 
-          key
+          copy = copy_string(key)
+          @symbolize_keys ? copy.to_sym : copy
+        end
+
+        def raise_reserved_key!
+          raise ArgumentError, "#{Serializer::TRUNCATION_METADATA_KEY} is reserved for Julewire truncation metadata"
         end
 
         def copy_string(value)
-          return value unless value.is_a?(String)
+          if @max_string_bytes && value.bytesize > @max_string_bytes
+            copy = "#{value.byteslice(0, @max_string_bytes).scrub("?")}#{Serializer::TRUNCATED_SUFFIX}"
+            return mark_truncated(freeze_container(copy))
+          end
 
           copy = value.frozen? ? value : value.dup
-          @freeze_values ? copy.freeze : copy
+          freeze_container(copy)
         end
 
         def copy_time(value)

data/lib/julewire/core/serialization/value_traversal.rb

@@ -5,142 +5,25 @@ module Julewire
     module Serialization
       module ValueTraversal
         def traverse(value)
-          previous_seen = @traversal_seen
-          previous_first_seen = @traversal_first_seen
-          previous_second_seen = @traversal_second_seen
-          previous_third_seen = @traversal_third_seen
-          previous_fourth_seen = @traversal_fourth_seen
-          @traversal_seen = nil
-          @traversal_first_seen = nil
-          @traversal_second_seen = nil
-          @traversal_third_seen = nil
-          @traversal_fourth_seen = nil
           yield(value, 0)
-        ensure
-          @traversal_seen = previous_seen
-          @traversal_first_seen = previous_first_seen
-          @traversal_second_seen = previous_second_seen
-          @traversal_third_seen = previous_third_seen
-          @traversal_fourth_seen = previous_fourth_seen
         end
 
         private
 
-        def with_traversal_container(value, circular_value)
-          added = false
-          seen = @traversal_seen
-          first_seen = @traversal_first_seen
-          second_seen = @traversal_second_seen
-          third_seen = @traversal_third_seen
-          fourth_seen = @traversal_fourth_seen
-          return circular_value if seen&.key?(value) || first_seen.equal?(value) || second_seen.equal?(value) ||
-                                   third_seen.equal?(value) || fourth_seen.equal?(value)
+        def with_traversal_container(value, circular_value, &)
+          seen = traversal_seen
+          return circular_value if seen.include?(value)
 
-          added = mark_traversal_container(value, seen, first_seen, second_seen, third_seen, fourth_seen)
-          yield
-        ensure
-          unmark_traversal_container(value, added)
-        end
-
-        def traversal_seen?(value)
-          @traversal_seen&.key?(value) || @traversal_first_seen.equal?(value) ||
-            @traversal_second_seen.equal?(value) || @traversal_third_seen.equal?(value) ||
-            @traversal_fourth_seen.equal?(value)
-        end
-
-        def with_marked_traversal_container(value)
-          added = mark_traversal_container(
-            value,
-            @traversal_seen,
-            @traversal_first_seen,
-            @traversal_second_seen,
-            @traversal_third_seen,
-            @traversal_fourth_seen
-          )
-          yield
-        ensure
-          unmark_traversal_container(value, added)
-        end
-
-        def mark_traversal_container(value, seen, first_seen, second_seen, third_seen, fourth_seen)
-          # Most records visit only a handful of live containers. Keep those in
-          # slots and allocate the identity hash only for genuinely deep walks.
-          if seen
-            seen[value] = true
-            :hash
-          elsif first_seen.nil?
-            mark_first_seen(value)
-          elsif second_seen.nil?
-            mark_second_seen(value)
-          elsif third_seen.nil?
-            mark_third_seen(value)
-          elsif fourth_seen.nil?
-            mark_fourth_seen(value)
-          else
-            promote_traversal_seen(value, first_seen, second_seen, third_seen, fourth_seen)
-          end
-        end
-
-        def unmark_traversal_container(value, added)
-          case added
-          when :hash
-            @traversal_seen.delete(value)
-          when :first, :second, :third, :fourth
-            unmark_traversal_slot(value, added)
-          end
-        end
-
-        def mark_first_seen(value)
-          @traversal_first_seen = value
-          :first
-        end
-
-        def mark_second_seen(value)
-          @traversal_second_seen = value
-          :second
-        end
-
-        def mark_third_seen(value)
-          @traversal_third_seen = value
-          :third
-        end
-
-        def mark_fourth_seen(value)
-          @traversal_fourth_seen = value
-          :fourth
-        end
-
-        def promote_traversal_seen(value, first_seen, second_seen, third_seen, fourth_seen)
-          @traversal_seen = {}.compare_by_identity
-          @traversal_seen[first_seen] = true
-          @traversal_seen[second_seen] = true
-          @traversal_seen[third_seen] = true
-          @traversal_seen[fourth_seen] = true
-          @traversal_seen[value] = true
-          clear_traversal_slots!
-          :hash
-        end
-
-        def unmark_traversal_slot(value, added)
-          return @traversal_seen.delete(value) if @traversal_seen
-
-          clear_traversal_slot(added)
-        end
-
-        def clear_traversal_slot(slot)
-          case slot
-          when :first then @traversal_first_seen = nil
-          when :second then @traversal_second_seen = nil
-          when :third then @traversal_third_seen = nil
-          when :fourth then @traversal_fourth_seen = nil
+          seen.add(value)
+          begin
+            yield
+          ensure
+            seen.delete(value)
           end
         end
 
-        def clear_traversal_slots!
-          @traversal_first_seen = nil
-          @traversal_second_seen = nil
-          @traversal_third_seen = nil
-          @traversal_fourth_seen = nil
+        def traversal_seen
+          @traversal_seen ||= Set.new.compare_by_identity
         end
       end
 

data/lib/julewire/core/testing.rb

@@ -19,20 +19,15 @@ module Julewire
           nil
         end
 
-        def flush(*)
-          self
-        end
-
-        def close(*)
-          self
-        end
+        def flush(*) = self
+        def close(*) = self
 
         def health
-          { status: :ok, counts: { captured: @records.size } }
+          { status: :ok, counts: { captured: records.size } }
         end
 
         def clear
-          @records.clear
+          records.clear
           self
         end
       end
@@ -46,17 +41,16 @@ module Julewire
         end
 
         def write(value)
-          @writes << value
+          writes << value
           value.bytesize
         end
 
         def flush = self
-
         def close = self
       end
 
       class << self
-        def configure_capture_destination(runtime = Julewire, **)
+        def configure_capture_destination(runtime, **)
           destination = CaptureDestination.new(**)
           runtime.configure do |config|
             config.destinations.clear
@@ -70,53 +64,9 @@ module Julewire
           yield records if block_given?
           records
         end
-
-        def unregister_destination(kind)
-          Core::Destinations.__send__(:unregister, kind)
-        end
-
-        def reset_shared_scheduler
-          Core::Scheduling::SharedScheduler.__send__(:reset_for_test!)
-        end
-
-        def with_overridden_singleton_method(receiver, method_name, replacement)
-          singleton_class = class << receiver; self; end
-          method_exists =
-            singleton_class.method_defined?(method_name) || singleton_class.private_method_defined?(method_name)
-          original = singleton_class.instance_method(method_name) if method_exists
-          verbose = $VERBOSE
-          $VERBOSE = nil
-          singleton_class.define_method(method_name, replacement)
-          yield
-        ensure
-          $VERBOSE = nil
-          if original
-            singleton_class.define_method(method_name, original)
-          elsif singleton_class&.method_defined?(method_name) || singleton_class&.private_method_defined?(method_name)
-            singleton_class.remove_method(method_name)
-          end
-          $VERBOSE = verbose
-        end
-
-        def nonblocking_queue_values(queue)
-          values = []
-          loop do
-            values << queue.pop(true)
-          rescue ThreadError
-            return values
-          end
-        end
       end
     end
   end
 
   Testing = Core::Testing unless const_defined?(:Testing, false)
 end
-
-require_relative "testing/chaos"
-require_relative "testing/chaos/catalog"
-require_relative "testing/chaos/core_runtime"
-require_relative "testing/chaos/destination"
-require_relative "testing/chaos/emitter"
-require_relative "testing/chaos/raising_output"
-require_relative "testing/contracts"

data/lib/julewire/core/validation.rb

@@ -7,9 +7,8 @@ module Julewire
       class << self
         def validate_byte_limit!(value, name:)
           return if value.nil?
+          return value if value.instance_of?(Integer) && value.positive?
 
-          validate_integer_limit!(value, name: name, positive: true)
-        rescue ArgumentError
           raise ArgumentError, "#{name} must be nil or a positive Integer"
         end
 
@@ -18,7 +17,7 @@ module Julewire
         end
 
         def validate_integer_limit!(value, name:, positive: false)
-          return value if value.is_a?(Integer) && valid_integer_limit?(value, positive: positive)
+          return value if value.instance_of?(Integer) && valid_integer_limit?(value, positive: positive)
 
           qualifier = positive ? "positive" : "non-negative"
           raise ArgumentError, "#{name} must be a #{qualifier} Integer"

data/lib/julewire/core/version.rb

@@ -2,6 +2,6 @@
 
 module Julewire
   module Core
-    VERSION = "1.0.0"
+    VERSION = "1.1.0"
   end
 end

data/lib/julewire/core.rb

@@ -6,7 +6,7 @@ module Julewire
   # Each gem extends the shared Julewire namespace.
   loader = Zeitwerk::Loader.for_gem_extension(self)
   loader.inflector.inflect("cli" => "CLI")
-  loader.ignore("#{__dir__}/core/testing.rb", "#{__dir__}/core/testing")
+  loader.ignore("#{__dir__}/core/testing.rb")
   loader.setup
 
   module Core
@@ -25,7 +25,7 @@ module Julewire
 
           value.to_sym
         when Symbol
-          raise ArgumentError, "#{name} must not be empty" if value.name.empty?
+          raise ArgumentError, "#{name} must not be empty" if value.empty?
 
           value
         else
@@ -54,11 +54,6 @@ module Julewire
   extend Core::FacadeMethods
 
   Core::RuntimeLocator.current = Core::Runtime.new
-  Core.singleton_class.class_eval do
-    define_method(:loader) { loader }
-    private :loader
-  end
-
   ConsoleFormatter = Core::Records::ConsoleFormatter
   JsonEncoder = Core::Serialization::JsonEncoder
   Match = Core::Processing::Match