julewire-core 1.0.0

data/lib/julewire/core/processing/pipeline.rb

@@ -0,0 +1,360 @@
+# frozen_string_literal: true
+
+module Julewire
+  module Core
+    module Processing
+      class Pipeline # rubocop:disable Metrics/ClassLength -- Pipeline owns normalization, processors, and destinations.
+        COUNTER_KEYS = %i[
+          callback_error
+          entered
+          level_dropped
+          no_output_dropped
+          processor_dropped
+          processor_error
+        ].freeze
+        EMPTY_HASH = {}.freeze
+        private_constant :EMPTY_HASH
+
+        # @api integration_spi
+        # Integration-facing raw/normalized record pipeline.
+        def initialize(configuration:, invalid_severity_reporter: Diagnostics::InvalidSeverityReporter.counter)
+          @on_drop = configuration.on_drop
+          @on_failure = configuration.on_failure
+          @invalid_severity_reporter = invalid_severity_reporter
+          @destinations = build_destinations(configuration)
+          @labels = Fields::FieldSet.deep_symbolize_keys(configuration.labels.to_h)
+          @threshold = build_threshold(configuration)
+          @error_backtrace_lines = configuration.error_backtrace_lines
+          Validation.validate_callable!(configuration.on_failure, name: :on_failure, allow_nil: true)
+          @processor_chain = build_processor_chain(configuration)
+          @processors_empty = @processor_chain.empty?
+          @labels_empty = @labels.empty?
+          initialize_tracking
+        end
+
+        def emit(input = Core::UNSET, **fields, &)
+          emit_with_level_check(input, true, fields, &)
+        end
+
+        def emit_without_level(input = Core::UNSET, **fields, &)
+          emit_with_level_check(input, false, fields, &)
+        end
+
+        # Trusted integration path for adapter-owned input hashes.
+        def emit_integration(input, enforce_level: true)
+          emit_input_with_guard(input, enforce_level: enforce_level, lazy: false) do
+            build_draft(input, input_owned: true)
+          end
+        end
+
+        # Runtime summaries already carry their captured scope fields.
+        def emit_isolated_input(input, enforce_level: true)
+          emit_input_with_guard(input, enforce_level: enforce_level, lazy: false) do
+            build_isolated_draft(input)
+          end
+        end
+
+        # Trusted normalized-record path used by runtime summaries and bridge envelopes.
+        def emit_record(record, enforce_level: true)
+          return if no_output_dropped?
+
+          degradation_marker = @health.degradation_marker
+          emit_validated_record(record, enforce_level: enforce_level)
+          finish_emit_attempt(degradation_marker)
+        rescue StandardError => e
+          record_emit_failure(e, record)
+        end
+
+        def after_fork!
+          initialize_tracking
+          @destinations.after_fork!
+          self
+        end
+
+        def flush(timeout: nil)
+          @destinations.flush(timeout: timeout)
+        end
+
+        def close(timeout: nil, skip_resource_identities: nil)
+          @destinations.close(timeout: timeout, skip_resource_identities: skip_resource_identities)
+        end
+
+        def lifecycle_resource_identities
+          @destinations.lifecycle_resource_identities
+        end
+
+        def health
+          {
+            configured: !@destinations.empty?,
+            counts: pipeline_counts_snapshot,
+            destinations: @destinations.health,
+            last_callback_failure: @health.last_callback_failure,
+            last_failure: @health.last_failure,
+            status: pipeline_status
+          }
+        end
+
+        private
+
+        def build_destinations(configuration)
+          Destinations::Collection.build(
+            configuration: configuration,
+            defaults: destination_defaults(configuration),
+            on_drop: method(:record_destination_drop),
+            on_failure: method(:notify_failure)
+          )
+        end
+
+        def destination_defaults(configuration)
+          {
+            encoder: Serialization::JsonEncoder.new(max_backtrace_lines: configuration.error_backtrace_lines),
+            formatter: Records::Formatter.new,
+            error_backtrace_lines: configuration.error_backtrace_lines,
+            on_drop: configuration.on_drop,
+            on_failure: configuration.on_failure
+          }
+        end
+
+        def build_threshold(configuration)
+          LevelThreshold.new(
+            level: configuration.level,
+            invalid_severity_reporter: @invalid_severity_reporter
+          )
+        end
+
+        def build_processor_chain(configuration)
+          ProcessorChain.new(
+            processors: configuration.processors.to_a.freeze,
+            error_backtrace_lines: @error_backtrace_lines,
+            on_error: method(:record_processor_error)
+          )
+        end
+
+        def emit_with_level_check(input, enforce_level, fields, &)
+          input = Core.emit_input(input, fields)
+
+          emit_input_with_guard(input, enforce_level: enforce_level, lazy: block_given?) do
+            input = Records::LazyEmitInput.call(input, &) if block_given?
+            build_draft(input)
+          end
+        end
+
+        def emit_input_with_guard(input, enforce_level:, lazy:)
+          return if no_output_dropped?
+
+          degradation_marker = @health.degradation_marker
+          if raw_input_blocked?(input, enforce_level: enforce_level, lazy: lazy)
+            increment_pipeline_counter(:level_dropped)
+          else
+            emit_prepared_draft(yield, enforce_level: enforce_level, merge_static_labels: false)
+          end
+          finish_emit_attempt(degradation_marker)
+        rescue StandardError => e
+          notify_failure(e, phase: :emit)
+          emit_internal_error_record(e)
+          nil
+        end
+
+        def no_output_dropped?
+          return false unless @destinations.empty?
+
+          record_no_output_drop
+          true
+        end
+
+        def finish_emit_attempt(degradation_marker)
+          clear_degradation_if_unchanged(degradation_marker)
+          nil
+        end
+
+        def emit_validated_record(record, enforce_level:)
+          Records::Record.validate_normalized!(record)
+          return emit_fast_record(record, enforce_level: enforce_level) if fast_record_path?
+
+          emit_prepared_draft(Records::Draft.from_record(record, freeze_sections: false),
+                              enforce_level: enforce_level)
+          nil
+        end
+
+        def fast_record_path?
+          @labels_empty && @processors_empty
+        end
+
+        def emit_fast_record(record, enforce_level:)
+          if enforce_level && !emit_record?(record)
+            increment_pipeline_counter(:level_dropped)
+            return
+          end
+
+          increment_pipeline_counter(:entered)
+          emit_to_destinations(record)
+          nil
+        end
+
+        def raw_input_blocked?(input, enforce_level:, lazy:)
+          enforce_level &&
+            (!lazy || Records::RawInput.explicit_severity?(input)) &&
+            !@threshold.raw_input_allowed?(input)
+        end
+
+        def build_draft(input, input_owned: false)
+          store = ContextStore.current
+          build_draft_from(
+            input,
+            input_owned: input_owned,
+            context: store.context_hash,
+            neutral: store.neutral_hash,
+            attributes: store.attributes_hash,
+            carry: store.carry_hash,
+            scope: store.current_scope_or_snapshot
+          )
+        end
+
+        def build_isolated_draft(input, input_owned: false)
+          build_draft_from(
+            input,
+            input_owned: input_owned,
+            context: EMPTY_HASH,
+            neutral: EMPTY_HASH,
+            attributes: EMPTY_HASH,
+            carry: EMPTY_HASH,
+            scope: nil
+          )
+        end
+
+        def build_draft_from(input, input_owned:, context:, neutral:, attributes:, carry:, scope:)
+          Records::Draft.build_pipeline_owned(
+            input,
+            context: context,
+            neutral: neutral,
+            attributes: attributes,
+            carry: carry,
+            static_labels: @labels,
+            input_owned: input_owned,
+            freeze_sections: @processors_empty,
+            scope: scope,
+            error_backtrace_lines: @error_backtrace_lines,
+            invalid_severity_reporter: @invalid_severity_reporter
+          )
+        end
+
+        def initialize_tracking
+          @health = Diagnostics::Health.new(
+            counter_keys: COUNTER_KEYS,
+            callback_metadata: {},
+            callback_failure_counter: :callback_error
+          )
+        end
+
+        def pipeline_status
+          return :unconfigured if @destinations.empty?
+
+          @health.degraded? ? :degraded : :ok
+        end
+
+        def clear_degradation_if_unchanged(marker)
+          @health.clear_degradation_if_unchanged(marker)
+        end
+
+        def merge_static_labels(draft)
+          return draft if @labels_empty
+
+          draft[:labels] = Fields::FieldSet.merge(@labels, draft.fetch(:labels))
+          draft
+        end
+
+        def emit_record?(record_or_draft)
+          @threshold.allow?(record_or_draft.fetch(:severity))
+        end
+
+        def emit_prepared_draft(draft, enforce_level:, merge_static_labels: true)
+          if enforce_level && !emit_record?(draft)
+            increment_pipeline_counter(:level_dropped)
+            return
+          end
+
+          draft = merge_static_labels(draft) if merge_static_labels
+          increment_pipeline_counter(:entered)
+          emit_processed_draft(draft, enforce_level: enforce_level)
+        rescue StandardError => e
+          record_emit_failure(e, draft)
+        end
+
+        def emit_processed_draft(draft, enforce_level:)
+          return emit_to_destinations(draft.to_record) if @processors_empty
+
+          processed = @processor_chain.call(draft)
+          if processed.equal?(ProcessorChain::DROP)
+            increment_pipeline_counter(:processor_dropped)
+            return
+          end
+
+          processed, enforce_processed_level = processed_draft_and_level(processed, enforce_level)
+
+          if enforce_processed_level && !emit_record?(processed)
+            increment_pipeline_counter(:level_dropped)
+            return
+          end
+
+          emit_to_destinations(processed.to_record)
+        end
+
+        def processed_draft_and_level(processed, default_enforce_level)
+          if processed.is_a?(ProcessorChain::ErrorResult)
+            [processed.draft, false]
+          else
+            [processed, default_enforce_level]
+          end
+        end
+
+        def record_processor_error(error, record_metadata)
+          increment_pipeline_counter(:processor_error)
+          notify_failure(error, phase: :processor, record_metadata: record_metadata)
+        end
+
+        def emit_internal_error_record(error)
+          emit_prepared_draft(
+            Diagnostics::InternalRecords.emit_error(error, error_backtrace_lines: @error_backtrace_lines),
+            enforce_level: false
+          )
+        rescue StandardError => e
+          notify_failure(e, phase: :internal_error_record)
+          nil
+        end
+
+        def emit_to_destinations(record)
+          @destinations.emit(record)
+        end
+
+        def record_no_output_drop
+          increment_pipeline_counter(:no_output_dropped)
+          nil
+        end
+
+        def notify_failure(error, **metadata)
+          @health.record_failure(error, callback: @on_failure, **metadata)
+        end
+
+        def record_destination_drop(reason, **metadata)
+          callback_result = Diagnostics::CallbackNotifier.call(@on_drop, reason, metadata.merge(reason: reason))
+          return unless Diagnostics::CallbackNotifier.failure?(callback_result)
+
+          @health.record_callback_failure(callback_result)
+        end
+
+        def record_emit_failure(error, record)
+          notify_failure(error, phase: :emit_record, record_metadata: Records::Metadata.call(record))
+          nil
+        end
+
+        def pipeline_counts_snapshot
+          @health.counts
+        end
+
+        def increment_pipeline_counter(key)
+          @health.increment(key)
+        end
+      end
+    end
+  end
+end

data/lib/julewire/core/processing/processor_chain.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+module Julewire
+  module Core
+    module Processing
+      class ProcessorChain
+        DROP = Core.sentinel(:drop)
+        ErrorResult = Data.define(:draft)
+
+        def initialize(processors:, error_backtrace_lines:, on_error:)
+          @processors = processors
+          @error_backtrace_lines = error_backtrace_lines
+          @on_error = on_error
+        end
+
+        def empty? = @processors.empty?
+
+        def call(draft)
+          current = draft
+
+          @processors.each do |processor|
+            current = apply_processor_result(current, processor.call(current))
+            break if current == :drop
+          rescue StandardError => e
+            action = handle_processor_error(processor, e, current)
+            case action
+            when :continue
+              next
+            when :drop
+              return DROP
+            else
+              return action
+            end
+          end
+
+          current == :drop ? DROP : current
+        end
+
+        private
+
+        def apply_processor_result(current, result)
+          return :drop if result == :drop
+          return result if result.is_a?(Records::Draft)
+
+          current
+        end
+
+        def handle_processor_error(processor, error, current)
+          record_metadata = Records::Metadata.call(current)
+          @on_error.call(error, record_metadata)
+
+          return :continue if processor.on_error == ProcessorWrapper::FAIL_OPEN
+          return :drop if processor.on_error == ProcessorWrapper::DROP
+
+          ErrorResult.new(processor_error_record(processor, error, record_metadata))
+        end
+
+        def processor_error_record(processor, error, record_metadata)
+          Diagnostics::InternalRecords.processor_error(
+            processor_name: processor.processor_name,
+            error: error,
+            record_metadata: record_metadata,
+            error_backtrace_lines: @error_backtrace_lines
+          )
+        end
+      end
+    end
+  end
+end

data/lib/julewire/core/processing/processor_registry.rb

@@ -0,0 +1,115 @@
+# frozen_string_literal: true
+
+module Julewire
+  module Core
+    module Processing
+      class ProcessorRegistry
+        Entry = Data.define(:entry, :arguments, :options, :on_error, :factory)
+
+        def initialize(entries = [])
+          @entries = entries.map { normalize_entry(it) }
+        end
+
+        def use(processor = nil, *arguments, **options, &block)
+          add(resolve_processor(processor, block), arguments, options, prepend: false)
+        end
+
+        def prepend(processor = nil, *arguments, **options, &block)
+          add(resolve_processor(processor, block), arguments, options, prepend: true)
+        end
+
+        def clear
+          @entries.clear
+          self
+        end
+
+        def to_a
+          @entries.map { materialize(it) }
+        end
+
+        def copy
+          self.class.new(@entries)
+        end
+
+        def freeze
+          @entries.freeze
+          super
+        end
+
+        private
+
+        def resolve_processor(processor, block)
+          raise ArgumentError, "pass processor or block, not both" if processor && block
+
+          processor || block
+        end
+
+        def add(processor, arguments, options, prepend:)
+          raise ArgumentError, "processor or block is required" unless processor
+
+          entry = build_entry(processor, arguments, options)
+          prepend ? @entries.unshift(entry) : @entries << entry
+          self
+        end
+
+        def build_entry(processor, arguments = [], options = {})
+          options = options.dup
+          on_error = ProcessorWrapper.normalize_policy(
+            options.delete(:on_error) { ProcessorWrapper::FAIL_CLOSED }
+          )
+
+          if factory_processor?(processor)
+            Entry.new(
+              entry: processor.to_sym,
+              arguments: arguments.dup.freeze,
+              options: options.freeze,
+              on_error: on_error,
+              factory: true
+            )
+          elsif processor.is_a?(Class)
+            Entry.new(
+              entry: processor,
+              arguments: arguments.dup.freeze,
+              options: options.freeze,
+              on_error: on_error,
+              factory: false
+            )
+          else
+            validate_processor_object!(processor, arguments, options)
+            Entry.new(entry: processor, arguments: [].freeze, options: {}.freeze, on_error: on_error, factory: false)
+          end
+        end
+
+        def normalize_entry(entry)
+          entry.is_a?(Entry) ? entry : build_entry(entry)
+        end
+
+        def materialize(entry)
+          processor = if entry.factory
+                        Processing.build(entry.entry, *entry.arguments, **entry.options)
+                      elsif entry.entry.is_a?(Class)
+                        entry.entry.new(*entry.arguments, **entry.options)
+                      else
+                        entry.entry
+                      end
+          ProcessorWrapper.new(processor, on_error: entry.on_error)
+        end
+
+        def factory_processor?(processor)
+          processor.respond_to?(:to_sym) && Processing.factory_for(processor)
+        end
+
+        def validate_processor_object!(processor, arguments, options)
+          raise ArgumentError, "unknown processor kind #{processor.to_sym.inspect}" if processor.respond_to?(:to_sym)
+
+          raise ArgumentError, "processor constructor arguments require a class" unless arguments.empty?
+          raise ArgumentError, "processor options require a class" unless options.empty?
+
+          return if processor.respond_to?(:call)
+
+          raise ArgumentError, "processor must respond to call"
+        end
+      end
+    end
+  end
+end

data/lib/julewire/core/processing/processor_wrapper.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Julewire
+  module Core
+    module Processing
+      class ProcessorWrapper
+        FAIL_CLOSED = :fail_closed
+        FAIL_OPEN = :fail_open
+        DROP = :drop
+        POLICIES = [FAIL_CLOSED, FAIL_OPEN, DROP].freeze
+
+        attr_reader :on_error
+
+        class << self
+          def normalize_policy(value)
+            Validation.validate_symbol_choice!(value, name: "processor on_error", choices: POLICIES)
+          end
+        end
+
+        def initialize(processor, on_error: FAIL_CLOSED)
+          validate_processor!(processor)
+          @processor = processor
+          @on_error = self.class.normalize_policy(on_error)
+        end
+
+        def call(...)
+          @processor.call(...)
+        end
+
+        def processor_name
+          @processor.class.name
+        end
+
+        private
+
+        def validate_processor!(processor)
+          return if processor.respond_to?(:call)
+
+          raise ArgumentError, "processor must respond to call"
+        end
+      end
+    end
+  end
+end

data/lib/julewire/core/processing/record_field_transform.rb

@@ -0,0 +1,124 @@
+# frozen_string_literal: true
+
+module Julewire
+  module Core
+    module Processing
+      class RecordFieldTransform
+        CONTAINER_KEYS = Fields::Bags.transform_container_sections
+        SCALAR_KEYS = (Records::Record::REQUIRED_KEYS - CONTAINER_KEYS).freeze
+        CONTAINER_KEY_SET = CONTAINER_KEYS.to_h { [it, true] }.freeze
+        SCALAR_KEY_SET = SCALAR_KEYS.to_h { [it, true] }.freeze
+        private_constant :CONTAINER_KEYS
+        private_constant :SCALAR_KEYS
+        private_constant :CONTAINER_KEY_SET
+        private_constant :SCALAR_KEY_SET
+
+        class << self
+          def container_keys = CONTAINER_KEYS
+
+          def scalar_keys = SCALAR_KEYS
+
+          def container_key?(key) = CONTAINER_KEY_SET.key?(key)
+
+          def scalar_key?(key) = SCALAR_KEY_SET.key?(key)
+        end
+
+        def initialize(
+          max_array_items: nil,
+          max_depth: nil,
+          max_hash_keys: nil,
+          max_string_bytes: nil,
+          preserve_top_level_keys: nil,
+          track_paths: false
+        )
+          @bounded_options = bounded_options(
+            max_array_items: max_array_items,
+            max_depth: max_depth,
+            max_hash_keys: max_hash_keys,
+            max_string_bytes: max_string_bytes
+          )
+          @preserve_top_level_key_set = Array(preserve_top_level_keys).to_h { [it, true] }
+          @track_paths = track_paths
+        end
+
+        def call(record, &)
+          record.each_with_object({}) do |(key, value), result|
+            result[key] = transform_record_field(key, value, record, &)
+          end
+        end
+
+        private
+
+        def transform_record_field(key, value, record, &)
+          return transform_container(key, value, record, &) if transform_container?(key, value)
+          return value unless transform_scalar?(key)
+
+          transform_scalar(key, value, record, &)
+        end
+
+        def transform_container?(key, value)
+          value.is_a?(Hash) && self.class.container_key?(key)
+        end
+
+        def transform_scalar?(key)
+          self.class.scalar_key?(key) && !@preserve_top_level_key_set.key?(key)
+        end
+
+        def transform_container(top_level_key, value, record, &)
+          Serialization::BoundedTransform.call(
+            value,
+            **@bounded_options,
+            track_paths: @track_paths
+          ) do |item, key:, path:, depth:, **|
+            yield(
+              item,
+              key: key,
+              path: path,
+              prefixed_path: prefixed_path(path, top_level_key),
+              original: record,
+              depth: depth,
+              top_level_key: top_level_key
+            )
+          end
+        end
+
+        def transform_scalar(top_level_key, value, record, &)
+          path = top_level_key.to_s if @track_paths
+          transformed = yield(
+            value,
+            key: top_level_key,
+            path: path,
+            prefixed_path: path,
+            original: record,
+            depth: 1,
+            top_level_key: top_level_key
+          )
+          transformed = value if transformed.equal?(Serialization::BoundedTransform::CONTINUE)
+          bound_value(transformed)
+        end
+
+        def bound_value(value)
+          Serialization::BoundedTransform.call(
+            value,
+            **@bounded_options
+          )
+        end
+
+        def prefixed_path(path, top_level_key)
+          return unless path
+
+          "#{top_level_key}.#{path}"
+        end
+
+        def bounded_options(max_array_items:, max_depth:, max_hash_keys:, max_string_bytes:)
+          {
+            max_array_items: max_array_items,
+            max_depth: max_depth,
+            max_hash_keys: max_hash_keys,
+            max_string_bytes: max_string_bytes
+          }.compact
+        end
+      end
+    end
+  end
+end