julewire-core 1.0.0 → 1.0.1

data/lib/julewire/core/fields/field_stack.rb

@@ -13,17 +13,20 @@ module Julewire
         class Layer
           attr_reader :fields, :parent
 
-          def initialize(parent, fields, delete_paths: nil, clear_parent_deletes: true)
+          def initialize(parent, fields, delete_paths: nil, clear_parent_deletes: true, owned: false)
             @parent = parent
             @fields = fields
             @delete_paths = delete_paths
             @clear_parent_deletes = clear_parent_deletes
+            @owned = owned
             @active_delete_paths_computed = false
             @active_delete_paths = nil
             @snapshot = nil
             @value_cache = nil
           end
 
+          def owned? = @owned
+
           def snapshot
             @snapshot ||= build_snapshot
           end
@@ -35,7 +38,7 @@ module Julewire
                       FieldSet.value_for(snapshot, key, default: MISSING)
                     else
                       field_value = FieldSet.value_for(@fields, key, default: MISSING)
-                      field_value.equal?(MISSING) ? parent_value_for(key) : Fields::Internal.frozen_copy(field_value)
+                      field_value.equal?(MISSING) ? parent_value_for(key) : frozen_field_value(field_value)
                     end
             (@value_cache ||= {})[key] = value
           end
@@ -56,6 +59,14 @@ module Julewire
             @clear_parent_deletes ? @delete_paths : active_delete_paths
           end
 
+          def merge_into(snapshot)
+            if @owned
+              Fields::Internal.merge_owned!(snapshot, FieldSet.deep_symbolize_owned_keys(@fields))
+            else
+              FieldSet.merge!(snapshot, @fields)
+            end
+          end
+
           private
 
           def build_snapshot
@@ -64,30 +75,35 @@ module Julewire
 
             snapshot = source_snapshot_base
             source_chain.reverse_each do |source|
-              FieldSet.merge!(snapshot, source.fields)
+              source.merge_into(snapshot)
               paths = source.delete_paths_for_snapshot
               Fields::Internal.apply_delete_paths!(snapshot, paths) if paths
             end
-            Fields::Internal.frozen_copy(snapshot)
+            Fields::Internal.frozen_owned_copy(snapshot)
           end
 
           def build_direct_snapshot
-            snapshot = FieldSet.merge!({}, @fields)
+            snapshot = merge_into({})
             paths = delete_paths_for_snapshot
             Fields::Internal.apply_delete_paths!(snapshot, paths) if paths
-            Fields::Internal.frozen_copy(snapshot)
+            Fields::Internal.frozen_owned_copy(snapshot)
           end
 
           def build_parent_snapshot
-            snapshot = FieldSet.merge(@parent.snapshot, @fields)
+            snapshot = FieldSet.deep_dup_owned(@parent.snapshot)
+            merge_into(snapshot)
             paths = delete_paths_for_snapshot
             Fields::Internal.apply_delete_paths!(snapshot, paths) if paths
-            Fields::Internal.frozen_copy(snapshot)
+            Fields::Internal.frozen_owned_copy(snapshot)
           end
 
           def source_snapshot_base
             source = source_chain_base
-            source ? FieldSet.deep_dup(source.snapshot) : {}
+            source ? FieldSet.deep_dup_owned(source.snapshot) : {}
+          end
+
+          def frozen_field_value(value)
+            @owned ? Fields::Internal.frozen_owned_copy(value) : Fields::Internal.frozen_copy(value)
           end
 
           def source_chain
@@ -181,7 +197,7 @@ module Julewire
           return if fields.empty?
 
           fields = normalize_owned_keys(fields) if owned
-          @source = Layer.new(@source, fields, clear_parent_deletes: true)
+          @source = Layer.new(@source, fields, clear_parent_deletes: true, owned: owned)
           invalidate_snapshot!
         end
 
@@ -199,7 +215,7 @@ module Julewire
           return yield if fields.empty?
 
           fields = normalize_owned_keys(fields) if owned
-          with_layer(fields, &)
+          with_layer(fields, owned: owned, &)
         end
 
         def without(path, &)
@@ -226,9 +242,15 @@ module Julewire
           FieldSet.coerce(fields, keyword_fields)
         end
 
-        def with_layer(fields, delete_paths: nil)
+        def with_layer(fields, delete_paths: nil, owned: false)
           previous_source = @source
-          @source = Layer.new(previous_source, fields, delete_paths: delete_paths, clear_parent_deletes: false)
+          @source = Layer.new(
+            previous_source,
+            fields,
+            delete_paths: delete_paths,
+            clear_parent_deletes: false,
+            owned: owned
+          )
           invalidate_snapshot!
           begin
             yield
@@ -250,13 +272,17 @@ module Julewire
           unless @source.parent
             # Single-layer hits avoid Layer's delete-path/cache bookkeeping.
             field_value = FieldSet.value_for(@source.fields, key, default: MISSING)
-            return Fields::Internal.frozen_copy(field_value) unless field_value.equal?(MISSING)
+            return frozen_source_value(field_value, @source.owned?) unless field_value.equal?(MISSING)
           end
 
           value = @source.value_for(key)
           value.equal?(MISSING) ? MISSING : value
         end
 
+        def frozen_source_value(value, owned)
+          owned ? Fields::Internal.frozen_owned_copy(value) : Fields::Internal.frozen_copy(value)
+        end
+
         def invalidate_snapshot!
           @version += 1
           @snapshot = nil

data/lib/julewire/core/fields/internal.rb

@@ -18,12 +18,29 @@ module Julewire
           end
 
           def frozen_copy(value)
-            return EMPTY_HASH if value.is_a?(Hash) && value.empty?
-            return EMPTY_ARRAY if value.is_a?(Array) && value.empty?
+            frozen_copy_with(value, preserve_truncation_metadata: false)
+          end
+
+          def frozen_owned_copy(value)
+            frozen_copy_with(value, preserve_truncation_metadata: true)
+          end
+
+          def frozen_deep_symbolize_keys(value)
+            frozen_deep_symbolize_keys_with(value, preserve_truncation_metadata: false)
+          end
 
-            Serialization::ValueCopy.call(value, freeze_values: true)
+          def frozen_deep_symbolize_owned_keys(value)
+            frozen_deep_symbolize_keys_with(value, preserve_truncation_metadata: true)
           end
 
+          def delete_path!(target, path) = Deletion.delete_path!(target, path)
+
+          def apply_delete_paths!(target, paths) = Deletion.apply_delete_paths!(target, paths)
+
+          def clear_delete_paths!(paths, fields) = Deletion.clear_delete_paths!(paths, fields)
+
+          def normalize_path(path) = Deletion.normalize_path(path)
+
           def deep_merge(left, right)
             deep_merge!(FieldSet.deep_symbolize_keys(left), right)
           end
@@ -52,22 +69,33 @@ module Julewire
             merge_values!(target, fields) { |value, _existing| value }
           end
 
-          def frozen_deep_symbolize_keys(value)
+          private
+
+          def frozen_copy_with(value, preserve_truncation_metadata:)
             return EMPTY_HASH if value.is_a?(Hash) && value.empty?
             return EMPTY_ARRAY if value.is_a?(Array) && value.empty?
 
-            Serialization::ValueCopy.call(value, freeze_values: true, symbolize_keys: true)
+            Serialization::ValueCopy.call(
+              value,
+              freeze_values: true,
+              preserve_truncation_metadata: preserve_truncation_metadata
+            )
           end
 
-          def delete_path!(target, path) = Deletion.delete_path!(target, path)
-
-          def apply_delete_paths!(target, paths) = Deletion.apply_delete_paths!(target, paths)
-
-          def clear_delete_paths!(paths, fields) = Deletion.clear_delete_paths!(paths, fields)
-
-          def normalize_path(path) = Deletion.normalize_path(path)
+          def frozen_deep_symbolize_keys_with(value, preserve_truncation_metadata:)
+            return EMPTY_HASH if value.is_a?(Hash) && value.empty?
+            return EMPTY_ARRAY if value.is_a?(Array) && value.empty?
 
-          private
+            Serialization::ValueCopy.call(
+              value,
+              freeze_values: true,
+              max_array_items: Serialization::Serializer::DEFAULT_MAX_ARRAY_ITEMS,
+              max_hash_keys: Serialization::Serializer::DEFAULT_MAX_HASH_KEYS,
+              max_string_bytes: Serialization::Serializer::DEFAULT_MAX_STRING_BYTES,
+              preserve_truncation_metadata: preserve_truncation_metadata,
+              symbolize_keys: true
+            )
+          end
 
           def merge_values!(target, fields)
             return target unless fields.is_a?(Hash)

data/lib/julewire/core/fields/stack_set.rb

@@ -48,8 +48,8 @@ module Julewire
           stack(section).delete(path)
         end
 
-        def with(section, fields, owned: false, &)
-          stack(section).with(fields, owned: owned, &)
+        def with(section, fields = nil, owned: false, **keyword_fields, &)
+          stack(section).with(fields, owned: owned, **keyword_fields, &)
         end
 
         def without(section, path, &)

data/lib/julewire/core/integration/destination_health.rb

@@ -5,9 +5,10 @@ module Julewire
     module Integration
       # @api integration_spi
       class DestinationHealth
-        def initialize(counter_keys:, failure_counter: :failures)
+        def initialize(counter_keys:, callback_failure_counter: nil, failure_counter: :failures)
           @failure_counter = failure_counter
           @state = Diagnostics::Health.new(
+            callback_failure_counter: callback_failure_counter,
             counter_keys: counter_keys,
             failure_counter: failure_counter,
             track_failures: failure_counter == :failures
@@ -26,16 +27,26 @@ module Julewire
           @state.record_loss(reason: reason, counter: counter, degrade: false, **metadata)
         end
 
+        def record_callback_failure(callback_failure)
+          @state.record_callback_failure(callback_failure)
+        end
+
         def clear_degraded! = @state.clear_failures!
 
         def degraded? = @state.degraded?(status_from: :failure_or_loss)
 
+        def last_callback_failure = @state.last_callback_failure
+
         def last_loss = @state.last_loss
 
         def last_failure = @state.last_failure
 
         def snapshot(status: nil, **fields)
-          @state.snapshot(status: status, status_from: :failure_or_loss, include_loss: true, **fields)
+          snapshot = @state.snapshot(status: status, status_from: :failure_or_loss, include_loss: true, **fields)
+          callback_failure = @state.last_callback_failure
+          return snapshot unless callback_failure
+
+          snapshot.merge(last_callback_failure: callback_failure).freeze
         end
       end
     end

data/lib/julewire/core/propagation/carrier.rb

@@ -8,8 +8,33 @@ module Julewire
       # @api public
       module Carrier
         DEFAULT_KEY = "julewire"
+        DEFAULT_MAX_BYTES = 65_536
         DEFAULT_ENVELOPE = Core.sentinel(:default_envelope)
+        class Extracted
+          attr_reader :envelope, :status, :reason, :error
+
+          def initialize(envelope:, status:, reason: nil, error: nil)
+            @envelope = envelope
+            @status = status
+            @reason = reason
+            @error = error
+          end
+
+          def failure? = !error.nil?
+        end
         private_constant :DEFAULT_ENVELOPE
+        private_constant :Extracted
+
+        # @api integration_spi
+        class ExtractionError < StandardError
+          attr_reader :status, :reason
+
+          def initialize(status, reason)
+            @status = status
+            @reason = reason
+            super(reason)
+          end
+        end
 
         class << self
           def encode(envelope: DEFAULT_ENVELOPE, max_bytes: nil)
@@ -31,18 +56,20 @@ module Julewire
             carrier
           end
 
-          def extract(carrier, key: DEFAULT_KEY)
-            value = carrier_value(carrier, key)
-            return {} unless value
+          def extract(carrier, key: DEFAULT_KEY, max_bytes: DEFAULT_MAX_BYTES)
+            extract_result(carrier, key: key, max_bytes: max_bytes).envelope
+          end
 
-            parsed = JSON.parse(value.to_s)
-            parsed.is_a?(Hash) ? Fields::FieldSet.deep_symbolize_keys(parsed) : {}
-          rescue StandardError
-            {}
+          # @api integration_spi
+          def extract_result(carrier, key: DEFAULT_KEY, max_bytes: DEFAULT_MAX_BYTES)
+            Validation.validate_byte_limit!(max_bytes, name: :max_bytes)
+
+            extract_payload(carrier, key: key, max_bytes: max_bytes)
           end
 
-          def restore(carrier, key: DEFAULT_KEY, link_executions: false, &)
-            Propagation.restore(extract(carrier, key: key), link_executions: link_executions, &)
+          def restore(carrier, key: DEFAULT_KEY, link_executions: false, max_bytes: DEFAULT_MAX_BYTES, &)
+            envelope = extract_result(carrier, key: key, max_bytes: max_bytes).envelope
+            Propagation.restore(envelope, link_executions: link_executions, owned: true, &)
           end
 
           def serialized_envelope(envelope)
@@ -53,6 +80,33 @@ module Julewire
 
           private
 
+          def extract_payload(carrier, key:, max_bytes:)
+            value = carrier_value(carrier, key)
+            return extracted({}, :missing) unless value
+
+            string = value.to_s
+            if max_bytes && string.bytesize > max_bytes
+              return extracted_failure(:oversized, "carrier payload exceeds max_bytes")
+            end
+
+            parsed = JSON.parse(string)
+            return extracted_failure(:non_hash, "carrier payload must be a JSON object") unless parsed.is_a?(Hash)
+
+            extracted(Fields::FieldSet.deep_symbolize_owned_keys(parsed), :ok)
+          rescue StandardError => e
+            extracted_failure(:malformed, "carrier payload is not valid JSON", e)
+          end
+
+          def extracted(envelope, status)
+            Extracted.new(envelope: envelope, status: status)
+          end
+
+          def extracted_failure(status, reason, cause = nil)
+            error = ExtractionError.new(status, reason)
+            error.set_backtrace(cause.backtrace) if cause&.backtrace
+            Extracted.new(envelope: {}, status: status, reason: reason, error: error)
+          end
+
           def carrier_value(carrier, key)
             return unless carrier.respond_to?(:[])
 

data/lib/julewire/core/propagation.rb

@@ -13,10 +13,10 @@ module Julewire
         end
 
         def capture_local
-          capture_with { Fields::FieldSet.deep_dup(it) }
+          capture_with { Fields::FieldSet.deep_dup_owned(it) }
         end
 
-        def restore(envelope, link_executions: false, &)
+        def restore(envelope, link_executions: false, owned: false, &)
           raise ArgumentError, "block required" unless block_given?
 
           sections = FIELD_SECTIONS.to_h { |section| [section, hash_value(envelope, section)] }
@@ -25,6 +25,7 @@ module Julewire
             **sections,
             execution: execution,
             link_executions: link_executions,
+            owned: owned,
             &
           )
         end

data/lib/julewire/core/records/draft.rb

@@ -90,10 +90,11 @@ module Julewire
           public
 
           def from_normalized_hash(data, lineage: nil, freeze_sections: true)
-            normalized = Fields::FieldSet.deep_symbolize_keys(data)
+            Record.validate_normalized_hash!(data)
+            normalized = data.dup
             lineage ||= Execution::Lineage.from_execution_hash(normalized[:execution])
-            normalized[:execution] = Execution::Lineage.clean_lazy_relationship_hash(normalized[:execution])
-            normalized = Fields::Internal.frozen_deep_symbolize_keys(normalized) if freeze_sections
+            normalized[:execution] = Execution::Lineage.clean_normalized_lazy_relationship_hash(normalized[:execution])
+            normalized = Fields::Internal.frozen_owned_copy(normalized) if freeze_sections
             new(
               normalized,
               lineage: lineage,
@@ -182,7 +183,7 @@ module Julewire
 
         def each_key(&) = @data.each_key(&)
 
-        def to_h = Fields::FieldSet.deep_dup(@data)
+        def to_h = Fields::FieldSet.deep_dup_owned(@data)
 
         def transform_field!(key)
           key = Fields::Internal.normalize_key(key)
@@ -236,7 +237,7 @@ module Julewire
         end
 
         def ensure_mutable_data!
-          @data = Fields::FieldSet.deep_dup(@data) if @data.frozen?
+          @data = Fields::FieldSet.deep_dup_owned(@data) if @data.frozen?
         end
 
         def transformed_field_lineage(key, value)
@@ -553,7 +554,12 @@ module Julewire
           def normalized_hash(value)
             return empty_hash if value.is_a?(Hash) && value.empty?
 
-            return value if @input_owned && !@freeze_sections && value.is_a?(Hash)
+            if @input_owned
+              return Fields::Internal.frozen_deep_symbolize_owned_keys(value) if @freeze_sections
+              return value if value.is_a?(Hash)
+
+              return Fields::FieldSet.deep_symbolize_owned_keys(value)
+            end
             return Fields::Internal.frozen_deep_symbolize_keys(value) if @freeze_sections
 
             Fields::FieldSet.deep_symbolize_keys(value)

data/lib/julewire/core/records/record.rb

@@ -19,21 +19,20 @@ module Julewire
 
         class << self
           def from_normalized_hash(record, lineage: nil)
-            if record.is_a?(Hash)
-              lineage ||= Execution::Lineage.from_execution_hash(record[:execution])
-              record = record.merge(execution: Execution::Lineage.clean_lazy_relationship_hash(record[:execution]))
-            end
             validate_normalized_hash!(record)
+            execution = record.fetch(:execution)
+            lineage ||= Execution::Lineage.from_execution_hash(execution)
+            record = record.merge(
+              execution: Execution::Lineage.clean_normalized_lazy_relationship_hash(execution)
+            )
             new(snapshot_hash(record), lineage: lineage)
           end
 
           def from_owned_hash(record, lineage: nil, trust_frozen: false)
-            if record.is_a?(Hash)
-              lineage ||= Execution::Lineage.from_execution_hash(record.fetch(:execution))
-              execution = Execution::Lineage.clean_lazy_relationship_hash(record.fetch(:execution))
-              record = record.frozen? ? record.merge(execution: execution) : replace_execution(record, execution)
-            end
             validate_normalized_hash!(record)
+            lineage ||= Execution::Lineage.from_execution_hash(record.fetch(:execution))
+            execution = Execution::Lineage.clean_normalized_lazy_relationship_hash(record.fetch(:execution))
+            record = record.frozen? ? record.merge(execution: execution) : replace_execution(record, execution)
             new(Serialization::DeepFreeze.call(record, trust_frozen: trust_frozen), lineage: lineage)
           end
 
@@ -70,8 +69,38 @@ module Julewire
           end
 
           def validate_symbol_keys!(record)
-            record.each_key do |key|
-              raise TypeError, "record must not use string keys" if key.is_a?(String)
+            validate_value_symbol_keys!(record)
+          end
+
+          def validate_value_symbol_keys!(root)
+            queue = [[root, 0]]
+            seen = {}.compare_by_identity
+
+            queue.each do |value, depth|
+              break if depth == NORMALIZATION_MAX_DEPTH
+              next unless mark_symbol_key_container(value, seen)
+
+              enqueue_symbol_key_children(queue, value, depth + 1)
+            end
+          end
+
+          def mark_symbol_key_container(value, seen)
+            return unless value.is_a?(Hash) || value.is_a?(Array)
+            return if seen.key?(value)
+
+            seen[value] = nil
+            value
+          end
+
+          def enqueue_symbol_key_children(queue, value, depth)
+            if value.is_a?(Hash)
+              value.each do |key, item|
+                raise TypeError, "record must not use string keys" if key.is_a?(String)
+
+                queue << [item, depth]
+              end
+            else
+              value.each { queue << [it, depth] }
             end
           end
 
@@ -126,7 +155,8 @@ module Julewire
           def snapshot_hash(record)
             Serialization::ValueCopy.call(
               record,
-              freeze_values: true
+              freeze_values: true,
+              preserve_truncation_metadata: true
             )
           end
         end
@@ -149,7 +179,7 @@ module Julewire
 
         def each(&) = @data.each(&)
 
-        def to_h = Fields::FieldSet.deep_dup(@data)
+        def to_h = Fields::FieldSet.deep_dup_owned(@data)
 
         # @api internal
         def serializable_data = @data

data/lib/julewire/core/runtime.rb

@@ -80,22 +80,14 @@ module Julewire
         end
       end
 
-      def emit_envelope(input:, context:, scope:, carry: {}, attributes: {}, neutral: {}, enforce_level: true)
+      def emit_envelope(input:, context:, scope:, carry: {}, attributes: {}, neutral: {}, enforce_level: true,
+                        owned: false)
         reject_runtime_call_during_configure!(:emit_envelope)
         state = runtime_state
         return record_post_close_emit(state) if state.pipeline_closed
 
         begin
-          record = Records::Draft.build(
-            input,
-            context: envelope_hash(context),
-            attributes: envelope_hash(attributes),
-            neutral: envelope_hash(neutral),
-            carry: envelope_hash(carry),
-            scope: scope,
-            error_backtrace_lines: state.configuration.error_backtrace_lines,
-            invalid_severity_reporter: @invalid_severity_reporter
-          ).to_record
+          record = envelope_draft(input, context, attributes, neutral, carry, scope, state, owned: owned).to_record
           state.pipeline.emit_record(record, enforce_level: enforce_level)
         rescue StandardError => e
           notify_failure(e, state, action: :emit_envelope)
@@ -200,6 +192,33 @@ module Julewire
 
       private
 
+      def envelope_draft(input, context, attributes, neutral, carry, scope, state, owned:)
+        if owned
+          Records::Draft.build_pipeline_owned(
+            input,
+            context: envelope_hash(context),
+            attributes: envelope_hash(attributes),
+            neutral: envelope_hash(neutral),
+            carry: envelope_hash(carry),
+            scope: scope,
+            error_backtrace_lines: state.configuration.error_backtrace_lines,
+            input_owned: true,
+            invalid_severity_reporter: @invalid_severity_reporter
+          )
+        else
+          Records::Draft.build(
+            input,
+            context: envelope_hash(context),
+            attributes: envelope_hash(attributes),
+            neutral: envelope_hash(neutral),
+            carry: envelope_hash(carry),
+            scope: scope,
+            error_backtrace_lines: state.configuration.error_backtrace_lines,
+            invalid_severity_reporter: @invalid_severity_reporter
+          )
+        end
+      end
+
       def before_execution_boundary_call!(action)
         reject_runtime_call_during_configure!(action)
       end

data/lib/julewire/core/serialization/bounded_transform.rb

@@ -49,6 +49,17 @@ module Julewire
           transformed = @transform.call(value, key: key, path: path, original: @root, depth: depth)
           transformed.equal?(CONTINUE) ? value : transformed
         end
+
+        def truncation_metadata(fields)
+          TruncationMetadata.build(
+            fields,
+            key_style: :symbol,
+            max_array_items: @max_array_items,
+            max_depth: @max_depth,
+            max_hash_keys: @max_hash_keys,
+            max_string_bytes: @max_string_bytes
+          )
+        end
       end
     end
   end