julewire-core 1.0.0

data/lib/julewire/core/destinations/tail_sampling.rb

@@ -0,0 +1,321 @@
+# frozen_string_literal: true
+
+module Julewire
+  module Core
+    module Destinations
+      # @api extension
+      class TailSampling # rubocop:disable Metrics/ClassLength -- Owns execution buffering plus destination lifecycle.
+        COUNTER_KEYS = %i[
+          buffered
+          emitted
+          failures
+          immediate
+          overflow_dropped
+          policy_dropped
+          received
+        ].freeze
+        DEFAULT_MAX_EXECUTIONS = 1024
+        DEFAULT_MAX_RECORDS_PER_EXECUTION = 1000
+        DEFAULT_SAMPLE_RATE = 0.1
+        ERROR_RANK = Records::Severity.rank(:error)
+        OPTION_KEYS = %i[
+          decider
+          max_executions
+          max_records_per_execution
+          name
+          on_drop
+          on_failure
+          sample_rate
+          slow_ms
+        ].freeze
+        private_constant :COUNTER_KEYS, :ERROR_RANK, :OPTION_KEYS
+
+        TailOptions = Data.define(:decider, :max_executions, :max_records_per_execution, :name, :on_drop, :on_failure,
+                                  :sample_rate, :slow_ms)
+        private_constant :TailOptions
+
+        attr_reader :name
+
+        def initialize(destination:, **options)
+          Validation.validate_options!(options, OPTION_KEYS, name: :tail_sampling)
+          options = tail_options(options)
+          @destination = Registry.validate!(destination)
+          @name = Destinations.normalize_name(options.name)
+          @sample_rate = options.sample_rate
+          Processing::Sampling.threshold_for(options.sample_rate)
+          @slow_ms = validate_slow_ms(options.slow_ms)
+          @max_executions = Validation.validate_integer_limit!(
+            options.max_executions,
+            name: :max_executions,
+            positive: true
+          )
+          @max_records_per_execution = Validation.validate_integer_limit!(
+            options.max_records_per_execution,
+            name: :max_records_per_execution,
+            positive: true
+          )
+          Validation.validate_callable!(options.decider, name: :decider, allow_nil: true)
+          Validation.validate_callable!(options.on_drop, name: :on_drop, allow_nil: true)
+          Validation.validate_callable!(options.on_failure, name: :on_failure, allow_nil: true)
+          @decider = options.decider
+          @on_drop = options.on_drop
+          @on_failure = options.on_failure
+          initialize_state
+        end
+
+        def emit(record)
+          result = @mutex.synchronize { accept_record(record) }
+          result.losses.compact.each { notify_drop(it) }
+          result.records.each { emit_target(it) }
+          nil
+        rescue StandardError => e
+          record_failure(e, record)
+          nil
+        end
+
+        def flush(timeout: nil)
+          drain_and_lifecycle(:flush, timeout: timeout)
+        end
+
+        def close(timeout: nil)
+          drain_and_lifecycle(:close, timeout: timeout)
+        end
+
+        def after_fork!
+          @mutex.synchronize { initialize_buffer }
+          @destination.after_fork! if @destination.respond_to?(:after_fork!)
+          self
+        rescue StandardError => e
+          record_failure(e, nil, phase: :after_fork)
+          self
+        end
+
+        def resource_identity = self
+
+        def health
+          buffered_executions = @mutex.synchronize { @buffers.length }
+          destination = destination_health
+          status = health_status
+          @health.snapshot(
+            buffered_executions: buffered_executions,
+            destination: destination,
+            max_executions: @max_executions,
+            max_records_per_execution: @max_records_per_execution,
+            sample_rate: @sample_rate,
+            slow_ms: @slow_ms,
+            status: status
+          )
+        end
+
+        private
+
+        EmitResult = Data.define(:records, :losses)
+        Buffer = Data.define(:records)
+        private_constant :Buffer, :EmitResult
+
+        def tail_options(options)
+          TailOptions.new(
+            decider: options[:decider],
+            max_executions: options.fetch(:max_executions, DEFAULT_MAX_EXECUTIONS),
+            max_records_per_execution: options.fetch(:max_records_per_execution, DEFAULT_MAX_RECORDS_PER_EXECUTION),
+            name: options.fetch(:name, :tail_sampling),
+            on_drop: options[:on_drop],
+            on_failure: options[:on_failure],
+            sample_rate: options.fetch(:sample_rate, DEFAULT_SAMPLE_RATE),
+            slow_ms: options[:slow_ms]
+          )
+        end
+
+        def initialize_state
+          @mutex = Mutex.new
+          initialize_buffer
+          @health = Integration::DestinationHealth.new(counter_keys: COUNTER_KEYS)
+        end
+
+        def initialize_buffer
+          @buffers = {}
+          @order = []
+        end
+
+        def accept_record(record)
+          @health.increment(:received)
+          key = execution_key(record)
+          return immediate(record) unless key
+
+          if summary_record?(record)
+            finish_execution(key, record)
+          else
+            buffer_record(key, record)
+          end
+        end
+
+        def immediate(record)
+          @health.increment(:immediate)
+          EmitResult.new([record], [])
+        end
+
+        def buffer_record(key, record)
+          losses = ensure_execution_capacity(key)
+          buffer = (@buffers[key] ||= begin
+            @order << key
+            Buffer.new([])
+          end)
+          if buffer.records.length >= @max_records_per_execution
+            dropped = buffer.records.shift
+            losses << record_loss(:overflow_dropped, dropped)
+          end
+          buffer.records << record
+          @health.increment(:buffered)
+          EmitResult.new([], losses)
+        end
+
+        def ensure_execution_capacity(key)
+          return [] if @buffers.key?(key)
+          return [] if @buffers.length < @max_executions
+
+          oldest = @order.shift
+          buffer = @buffers.delete(oldest)
+          Array(buffer&.records).map { record_loss(:overflow_dropped, it) }
+        end
+
+        def finish_execution(key, summary)
+          buffer = @buffers.delete(key)
+          @order.delete(key)
+          records = Array(buffer&.records)
+          records << summary
+          if keep_execution?(summary, key)
+            @health.increment(:emitted, by: records.length)
+            EmitResult.new(records, [])
+          else
+            EmitResult.new([], records.map { record_loss(:policy_dropped, it) })
+          end
+        end
+
+        def drain_buffered_records
+          records = @order.flat_map { @buffers.fetch(it).records }
+          initialize_buffer
+          @health.increment(:emitted, by: records.length)
+          records
+        end
+
+        def drain_and_lifecycle(method_name, timeout:)
+          records = @mutex.synchronize { drain_buffered_records }
+          records.each { emit_target(it) }
+          lifecycle(method_name, timeout: timeout)
+        end
+
+        def keep_execution?(record, key)
+          return !!@decider.call(record, key: key) if @decider
+
+          default_keep_execution?(record, key)
+        rescue StandardError => e
+          record_failure(e, record, phase: :tail_sampling_decider)
+          false
+        end
+
+        def default_keep_execution?(record, key)
+          error_record?(record) || slow_record?(record) || Processing::Sampling.keep?(rate: @sample_rate, key: key)
+        end
+
+        def error_record?(record)
+          return true if record[:error]
+
+          Records::Severity.rank(record[:severity]) >= ERROR_RANK
+        rescue StandardError
+          false
+        end
+
+        def slow_record?(record)
+          return false unless @slow_ms
+
+          duration = field_value(record[:metrics], :duration_ms)
+          duration.is_a?(Numeric) && duration >= @slow_ms
+        end
+
+        def summary_record?(record) = record[:kind] == :summary
+
+        def execution_key(record)
+          reference = record.respond_to?(:lineage) ? record.lineage.root_reference : nil
+          reference = record[:execution] unless reference.is_a?(Hash)
+          id = field_value(reference, :id)
+          return unless id
+
+          [field_value(reference, :type), id].freeze
+        rescue StandardError
+          nil
+        end
+
+        def field_value(hash, key)
+          Fields::FieldSet.value_for(hash, key)
+        end
+
+        def emit_target(record)
+          @destination.emit(record)
+        rescue StandardError => e
+          record_failure(e, record, phase: :tail_sampling_destination)
+        end
+
+        def lifecycle(method_name, timeout:)
+          @destination.public_send(method_name, timeout: timeout) != false
+        rescue StandardError => e
+          record_failure(e, nil, phase: :tail_sampling_lifecycle, action: method_name)
+          false
+        end
+
+        def destination_health
+          @destination.health
+        rescue StandardError => e
+          Diagnostics::FailureSnapshot.build(e, destination: @destination.name, phase: :tail_sampling_health)
+        end
+
+        def record_loss(reason, record)
+          @health.record_loss(reason: reason, record_metadata: Records::Metadata.call(record))
+        rescue StandardError => e
+          record_failure(e, record, phase: :tail_sampling_drop)
+        end
+
+        def notify_drop(loss)
+          @on_drop&.call(loss.fetch(:reason), loss)
+        rescue StandardError => e
+          record_failure(e, nil, phase: :tail_sampling_drop_callback)
+        end
+
+        def record_failure(error, record, phase: :tail_sampling, **metadata)
+          failure_metadata = if @mutex.owned?
+                               record_failure_state(error, record, phase: phase, **metadata)
+                             else
+                               @mutex.synchronize { record_failure_state(error, record, phase: phase, **metadata) }
+                             end
+          @on_failure&.call(error, **failure_metadata)
+        rescue StandardError
+          nil
+        end
+
+        def record_failure_state(error, record, phase:, **metadata)
+          @health.record_failure(
+            error,
+            **metadata,
+            destination: @name,
+            phase: phase,
+            record_metadata: record ? Records::Metadata.call(record) : nil
+          )
+          metadata.merge(destination: @name, phase: phase)
+        end
+
+        def health_status
+          return :degraded if @health.last_failure || @health.last_loss&.fetch(:reason) == :overflow_dropped
+
+          :ok
+        end
+
+        def validate_slow_ms(value)
+          return if value.nil?
+          raise ArgumentError, "slow_ms must be a non-negative Numeric" unless value.is_a?(Numeric) && value.finite?
+          raise ArgumentError, "slow_ms must be non-negative" if value.negative?
+
+          value
+        end
+      end
+    end
+  end
+end

data/lib/julewire/core/destinations/write_step.rb

@@ -0,0 +1,119 @@
+# frozen_string_literal: true
+
+module Julewire
+  module Core
+    module Destinations
+      # @api integration_spi
+      class WriteStep
+        def initialize(
+          formatter:,
+          encoder:,
+          output:,
+          max_record_bytes:,
+          increment:,
+          failure:,
+          loss:,
+          output_class_name:
+        )
+          @formatter = formatter
+          @encoder = encoder
+          @output = output
+          @max_record_bytes = max_record_bytes
+          @increment = increment
+          @failure = failure
+          @loss = loss
+          @output_class_name = output_class_name
+        end
+
+        def call(record)
+          increment(:received)
+          payload = format_record(record)
+          return :dropped unless payload
+
+          increment(:formatted)
+          encoded = encode_payload(payload, record)
+          return :dropped unless encoded
+          return :dropped unless within_limit?(encoded, record)
+          return :dropped unless write(encoded, record)
+
+          increment(:output_accepted)
+          :accepted
+        end
+
+        private
+
+        def format_record(record)
+          payload = @formatter.call(record)
+          raise TypeError, "formatter must return a payload object" if payload.nil?
+
+          payload
+        rescue StandardError => e
+          transform_error(e, phase: :formatter, counter: :formatter_error, record: record)
+        end
+
+        def encode_payload(payload, record)
+          encoded = @encoder.call(payload)
+          raise TypeError, "encoder must return a String" unless encoded.is_a?(String)
+
+          encoded
+        rescue StandardError => e
+          transform_error(e, phase: :encode, counter: :encode_error, record: record)
+        end
+
+        def transform_error(error, phase:, counter:, record:)
+          increment(counter)
+          failure(error, phase: phase, record: record)
+          loss(counter, record: record)
+          nil
+        end
+
+        def within_limit?(encoded, record)
+          return true unless @max_record_bytes
+
+          bytesize = encoded.bytesize
+          return true if bytesize <= @max_record_bytes
+
+          increment(:record_too_large)
+          loss(:record_too_large, bytesize: bytesize, max_record_bytes: @max_record_bytes, record: record)
+          false
+        end
+
+        def write(encoded, record)
+          result = @output.write(encoded)
+          return true unless result == false
+
+          increment(:output_rejected)
+          output_error
+          loss(:output_rejected, record: record)
+          false
+        rescue StandardError => e
+          increment(:output_exception)
+          output_error
+          failure(e, phase: :output, action: :write, output_class: output_class_name, record: record)
+          loss(:output_exception, action: :write, output_class: output_class_name, record: record)
+          false
+        end
+
+        def output_error
+          increment(:output_error)
+        end
+
+        def increment(counter)
+          @increment.call(counter)
+        end
+
+        def failure(error, **metadata)
+          @failure.call(error, metadata)
+        end
+
+        def loss(reason, **metadata)
+          @loss.call(reason, metadata)
+        end
+
+        def output_class_name
+          @output_class_name.call
+        end
+      end
+    end
+  end
+end

data/lib/julewire/core/destinations.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Julewire
+  module Core
+    module Destinations
+      @factories = {}
+
+      class << self
+        def register(kind, &factory)
+          raise ArgumentError, "destination factory block required" unless factory
+
+          @factories[normalize_name(kind)] = factory
+          nil
+        end
+
+        # Private testing seam for `Julewire::Testing.unregister_destination`.
+        def unregister(kind)
+          @factories.delete(normalize_name(kind))
+          nil
+        end
+        private :unregister
+
+        def factory_for(kind)
+          @factories[normalize_name(kind)]
+        end
+
+        def normalize_name(value)
+          Core.normalize_name(value, name: "destination name")
+        end
+      end
+    end
+  end
+end

data/lib/julewire/core/diagnostics/callback_notifier.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+module Julewire
+  module Core
+    module Diagnostics
+      module CallbackNotifier
+        ACTIVE_KEY = :__julewire_core_callback_active__
+        private_constant :ACTIVE_KEY
+
+        class NestedCallback < StandardError; end
+
+        private_constant :NestedCallback
+
+        Failure = Data.define(:at, :class_name, :metadata) do
+          def to_h
+            {
+              action: metadata[:action],
+              at: at,
+              class: class_name,
+              destination: metadata[:destination],
+              phase: metadata[:phase],
+              reason: metadata[:reason]
+            }.compact.freeze
+          end
+        end
+
+        class << self
+          def call(callback, first_argument, metadata)
+            return unless callback
+            return nested_callback_result(metadata) if callback_active?
+
+            previous = Fiber[ACTIVE_KEY]
+            begin
+              Fiber[ACTIVE_KEY] = true
+              callback.call(first_argument, metadata)
+              true
+            rescue StandardError => e
+              failure(e.class.name, metadata)
+            ensure
+              Fiber[ACTIVE_KEY] = previous
+            end
+          end
+
+          def failure?(result)
+            result.is_a?(Failure)
+          end
+
+          def nested_callback_result(metadata)
+            failure(NestedCallback.name, metadata)
+          end
+
+          def callback_active?
+            Fiber[ACTIVE_KEY] == true
+          end
+
+          def failure(class_name, metadata)
+            Failure.new(at: Time.now.utc, class_name: class_name, metadata: metadata || {})
+          end
+        end
+      end
+    end
+  end
+end

data/lib/julewire/core/diagnostics/doctor.rb

@@ -0,0 +1,114 @@
+# frozen_string_literal: true
+
+module Julewire
+  module Core
+    module Diagnostics
+      class Doctor
+        class << self
+          def call(runtime)
+            new(runtime).call
+          end
+        end
+
+        def initialize(runtime)
+          @runtime = runtime
+        end
+
+        def call
+          health = @runtime.health
+          config = @runtime.config
+          {
+            status: health.fetch(:status),
+            runtime: runtime_info(health, config),
+            pipeline: pipeline_info(health.fetch(:pipeline)),
+            integrations: integration_info(health.fetch(:integrations)),
+            process_integrations: integration_info(health.fetch(:process_integrations)),
+            warnings: warnings(health)
+          }
+        end
+
+        private
+
+        def runtime_info(health, config)
+          {
+            closed: health.fetch(:closed),
+            counts: health.fetch(:counts),
+            generation: health.fetch(:generation),
+            level: config.level,
+            last_failure: health[:last_failure],
+            status: health.fetch(:status)
+          }
+        end
+
+        def pipeline_info(pipeline)
+          {
+            configured: pipeline.fetch(:configured),
+            counts: pipeline.fetch(:counts),
+            destinations: destination_info(pipeline.fetch(:destinations)),
+            last_failure: pipeline[:last_failure],
+            status: pipeline.fetch(:status)
+          }
+        end
+
+        def destination_info(destinations)
+          destinations.transform_values do |destination|
+            {
+              counts: destination[:counts],
+              last_failure: destination[:last_failure],
+              last_loss: destination[:last_loss],
+              status: destination[:status]
+            }.compact
+          end
+        end
+
+        def integration_info(integrations)
+          integrations.transform_values do |integration|
+            {
+              counts: integration[:counts],
+              last_failure: integration[:last_failure],
+              status: integration[:status]
+            }.compact
+          end
+        end
+
+        def warnings(health)
+          [].tap do |items|
+            items << warning(:runtime_closed, "runtime is closed") if health.fetch(:closed)
+            pipeline_warnings(health.fetch(:pipeline), items)
+            integration_warnings(health.fetch(:integrations), items, label: :integration)
+            integration_warnings(health.fetch(:process_integrations), items, label: :process_integration)
+          end
+        end
+
+        def pipeline_warnings(pipeline, items)
+          items << warning(:no_destinations, "pipeline has no destinations") unless pipeline.fetch(:configured)
+          if pipeline.fetch(:configured) && pipeline.fetch(:status) != :ok
+            items << warning(:pipeline_degraded, "pipeline is #{pipeline.fetch(:status)}")
+          end
+          component_warnings(
+            pipeline.fetch(:destinations),
+            items,
+            code: :destination_degraded,
+            label: :destination
+          )
+        end
+
+        def integration_warnings(integrations, items, label:)
+          component_warnings(integrations, items, code: :integration_degraded, label: label)
+        end
+
+        def component_warnings(components, items, code:, label:)
+          components.each do |name, component|
+            next if component[:status] == :ok
+
+            items << warning(code, "#{label} #{name} is #{component[:status]}")
+          end
+        end
+
+        def warning(code, message)
+          { code: code, message: message }.freeze
+        end
+      end
+    end
+  end
+end

data/lib/julewire/core/diagnostics/failure_snapshot.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Julewire
+  module Core
+    module Diagnostics
+      module FailureSnapshot
+        class << self
+          def build(error, **metadata)
+            {
+              at: Time.now.utc,
+              action: metadata[:action],
+              class: error.class.name,
+              component: metadata[:component],
+              destination: metadata[:destination],
+              event: metadata[:event],
+              integration: metadata[:integration],
+              output_class: metadata[:output_class],
+              phase: metadata[:phase],
+              record: record_metadata(metadata[:record_metadata])
+            }.compact.freeze
+          end
+
+          private
+
+          def record_metadata(value)
+            return unless value.is_a?(Hash)
+
+            {
+              event: value[:event],
+              labels: value[:labels].is_a?(Hash) ? Fields::FieldSet.deep_dup(value[:labels]) : nil,
+              severity: value[:severity],
+              source: value[:source]
+            }.compact
+          end
+        end
+      end
+    end
+  end
+end