julewire-core 1.0.1 → 1.1.0

data/lib/julewire/core/destinations/synchronized_output.rb

@@ -17,8 +17,6 @@ module Julewire
         end
 
         def after_fork!
-          @mutex = Mutex.new
-          @lifecycle_mutex = Mutex.new
           @output.after_fork! if @output.respond_to?(:after_fork!)
           @lifecycle = lifecycle_methods
           self
@@ -41,7 +39,7 @@ module Julewire
             lifecycle = @lifecycle[:flush]
             return true unless lifecycle
 
-            call_lifecycle(:flush, lifecycle, timeout: timeout) != false
+            lifecycle_succeeded?(call_lifecycle(:flush, lifecycle, timeout: timeout))
           end
         end
 
@@ -52,12 +50,13 @@ module Julewire
             @mutex.synchronize do
               return true if output_closed?
 
-              result = if @close_output && @lifecycle[:close]
-                         call_lifecycle(:close, @lifecycle.fetch(:close), timeout: timeout)
-                       elsif @lifecycle[:flush]
-                         call_lifecycle(:flush, @lifecycle.fetch(:flush), timeout: timeout)
-                       end
-              result != false
+              if @close_output && (close_lifecycle = @lifecycle[:close])
+                return lifecycle_succeeded?(call_lifecycle(:close, close_lifecycle, timeout: timeout))
+              end
+
+              return true unless (flush_lifecycle = @lifecycle[:flush])
+
+              lifecycle_succeeded?(call_lifecycle(:flush, flush_lifecycle, timeout: timeout))
             end
           end
         end
@@ -65,9 +64,11 @@ module Julewire
         private
 
         def output_closed?
-          @output.respond_to?(:closed?) ? @output.closed? : false
+          @output.respond_to?(:closed?) && @output.closed?
         end
 
+        def lifecycle_succeeded?(result) = !result.equal?(false)
+
         def call_lifecycle(name, lifecycle, timeout:)
           return @output.public_send(name, timeout: timeout) if lifecycle.fetch(:timeout)
 
@@ -78,9 +79,9 @@ module Julewire
           %i[flush close].each_with_object({}) do |name, methods|
             next unless @output.respond_to?(name)
 
-            method = @output.method(name)
-            methods[name] = { timeout: accepts_timeout_keyword?(method) }.freeze
-          end.freeze
+            method = @output.public_method(name)
+            methods[name] = { timeout: accepts_timeout_keyword?(method) }
+          end
         end
 
         def accepts_timeout_keyword?(method)

data/lib/julewire/core/destinations/tail_sampling.rb

@@ -29,7 +29,8 @@ module Julewire
           slow_ms
         ].freeze
         private_constant :COUNTER_KEYS, :ERROR_RANK, :OPTION_KEYS
-
+        Sampling = Processing::Sampling
+        private_constant :Sampling
         TailOptions = Data.define(:decider, :max_executions, :max_records_per_execution, :name, :on_drop, :on_failure,
                                   :sample_rate, :slow_ms)
         private_constant :TailOptions
@@ -42,7 +43,7 @@ module Julewire
           @destination = Registry.validate!(destination)
           @name = Destinations.normalize_name(options.name)
           @sample_rate = options.sample_rate
-          Processing::Sampling.threshold_for(options.sample_rate)
+          Sampling.threshold_for(options.sample_rate)
           @slow_ms = validate_slow_ms(options.slow_ms)
           @max_executions = Validation.validate_integer_limit!(
             options.max_executions,
@@ -65,7 +66,7 @@ module Julewire
 
         def emit(record)
           result = @mutex.synchronize { accept_record(record) }
-          result.losses.compact.each { notify_drop(it) }
+          result.losses.each { notify_drop(it) }
           result.records.each { emit_target(it) }
           nil
         rescue StandardError => e
@@ -160,7 +161,7 @@ module Julewire
             @order << key
             Buffer.new([])
           end)
-          if buffer.records.length >= @max_records_per_execution
+          if buffer.records.length == @max_records_per_execution
             dropped = buffer.records.shift
             losses << record_loss(:overflow_dropped, dropped)
           end
@@ -174,8 +175,7 @@ module Julewire
           return [] if @buffers.length < @max_executions
 
           oldest = @order.shift
-          buffer = @buffers.delete(oldest)
-          Array(buffer&.records).map { record_loss(:overflow_dropped, it) }
+          @buffers.delete(oldest).records.map { record_loss(:overflow_dropped, it) }
         end
 
         def finish_execution(key, summary)
@@ -205,7 +205,7 @@ module Julewire
         end
 
         def keep_execution?(record, key)
-          return !!@decider.call(record, key: key) if @decider
+          return @decider.call(record, key: key) if @decider
 
           default_keep_execution?(record, key)
         rescue StandardError => e
@@ -214,7 +214,7 @@ module Julewire
         end
 
         def default_keep_execution?(record, key)
-          error_record?(record) || slow_record?(record) || Processing::Sampling.keep?(rate: @sample_rate, key: key)
+          error_record?(record) || slow_record?(record) || Sampling.keep?(rate: @sample_rate, key: key)
         end
 
         def error_record?(record)
@@ -235,7 +235,7 @@ module Julewire
         def summary_record?(record) = record[:kind] == :summary
 
         def execution_key(record)
-          reference = record.respond_to?(:lineage) ? record.lineage.root_reference : nil
+          reference = record.lineage.root_reference if record.respond_to?(:lineage)
           reference = record[:execution] unless reference.is_a?(Hash)
           id = field_value(reference, :id)
           return unless id
@@ -256,7 +256,7 @@ module Julewire
         end
 
         def lifecycle(method_name, timeout:)
-          @destination.public_send(method_name, timeout: timeout) != false
+          !@destination.public_send(method_name, timeout: timeout).eql?(false)
         rescue StandardError => e
           record_failure(e, nil, phase: :tail_sampling_lifecycle, action: method_name)
           false
@@ -270,8 +270,6 @@ module Julewire
 
         def record_loss(reason, record)
           @health.record_loss(reason: reason, record_metadata: Records::Metadata.call(record))
-        rescue StandardError => e
-          record_failure(e, record, phase: :tail_sampling_drop)
         end
 
         def notify_drop(loss)
@@ -281,29 +279,22 @@ module Julewire
         end
 
         def record_failure(error, record, phase: :tail_sampling, **metadata)
-          failure_metadata = if @mutex.owned?
-                               record_failure_state(error, record, phase: phase, **metadata)
-                             else
-                               @mutex.synchronize { record_failure_state(error, record, phase: phase, **metadata) }
-                             end
-          @on_failure&.call(error, **failure_metadata)
+          failure_metadata = record_failure_state(error, record, phase: phase, **metadata)
+          @on_failure.call(error, **failure_metadata)
         rescue StandardError
           nil
         end
 
         def record_failure_state(error, record, phase:, **metadata)
-          @health.record_failure(
-            error,
-            **metadata,
-            destination: @name,
-            phase: phase,
-            record_metadata: record ? Records::Metadata.call(record) : nil
-          )
-          metadata.merge(destination: @name, phase: phase)
+          metadata = metadata.merge(destination: @name, phase: phase)
+          record_metadata = record && Records::Metadata.call(record)
+          metadata[:record_metadata] = record_metadata if record_metadata
+          @health.record_failure(error, **metadata)
+          metadata
         end
 
         def health_status
-          return :degraded if @health.last_failure || @health.last_loss&.fetch(:reason) == :overflow_dropped
+          return if @health.last_failure || @health.last_loss&.fetch(:reason) == :overflow_dropped
 
           :ok
         end

data/lib/julewire/core/destinations/write_step.rb

@@ -28,16 +28,16 @@ module Julewire
         def call(record)
           increment(:received)
           payload = format_record(record)
-          return :dropped unless payload
+          return false unless payload
 
           increment(:formatted)
           encoded = encode_payload(payload, record)
-          return :dropped unless encoded
-          return :dropped unless within_limit?(encoded, record)
-          return :dropped unless write(encoded, record)
+          return false unless encoded
+          return false unless within_limit?(encoded, record)
+          return false unless write(encoded, record)
 
           increment(:output_accepted)
-          :accepted
+          true
         end
 
         private

data/lib/julewire/core/destinations.rb

@@ -13,13 +13,6 @@ module Julewire
           nil
         end
 
-        # Private testing seam for `Julewire::Testing.unregister_destination`.
-        def unregister(kind)
-          @factories.delete(normalize_name(kind))
-          nil
-        end
-        private :unregister
-
         def factory_for(kind)
           @factories[normalize_name(kind)]
         end

data/lib/julewire/core/diagnostics/callback_notifier.rb

@@ -11,7 +11,9 @@ module Julewire
 
         private_constant :NestedCallback
 
-        Failure = Data.define(:at, :class_name, :metadata) do
+        Failure = Data.define(:at, :class_name, :metadata)
+
+        class Failure
           def to_h
             {
               action: metadata[:action],
@@ -42,7 +44,7 @@ module Julewire
           end
 
           def failure?(result)
-            result.is_a?(Failure)
+            result.instance_of?(Failure)
           end
 
           def nested_callback_result(metadata)

data/lib/julewire/core/diagnostics/doctor.rb

@@ -52,7 +52,7 @@ module Julewire
 
         def destination_info(destinations)
           destinations.transform_values do |destination|
-            component_info(destination, include_loss: true)
+            destination_component_info(destination)
           end
         end
 
@@ -62,12 +62,18 @@ module Julewire
           end
         end
 
-        def component_info(component, include_loss: false)
+        def destination_component_info(component)
+          info = component_info(component)
+          last_loss = component[:last_loss]
+          info[:last_loss] = last_loss if last_loss
+          info
+        end
+
+        def component_info(component)
           {
-            counts: component[:counts],
+            counts: component.fetch(:counts),
             last_failure: component[:last_failure],
-            last_loss: include_loss ? component[:last_loss] : nil,
-            status: component[:status]
+            status: component.fetch(:status)
           }.compact
         end
 
@@ -82,7 +88,7 @@ module Julewire
 
         def pipeline_warnings(pipeline, items)
           items << warning(:no_destinations, "pipeline has no destinations") unless pipeline.fetch(:configured)
-          if pipeline.fetch(:configured) && pipeline.fetch(:status) != :ok
+          if pipeline.fetch(:configured) && !pipeline.fetch(:status).equal?(:ok)
             items << warning(:pipeline_degraded, "pipeline is #{pipeline.fetch(:status)}")
           end
           component_warnings(
@@ -99,14 +105,15 @@ module Julewire
 
         def component_warnings(components, items, code:, label:)
           components.each do |name, component|
-            next if component[:status] == :ok
+            status = component.fetch(:status)
+            next if status == :ok
 
-            items << warning(code, "#{label} #{name} is #{component[:status]}")
+            items << warning(code, "#{label} #{name} is #{status}")
           end
         end
 
         def warning(code, message)
-          { code: code, message: message }.freeze
+          { code: code, message: message }
         end
       end
     end

data/lib/julewire/core/diagnostics/failure_snapshot.rb

@@ -16,8 +16,10 @@ module Julewire
               integration: metadata[:integration],
               output_class: metadata[:output_class],
               phase: metadata[:phase],
+              processor: metadata[:processor],
               reason: metadata[:reason],
               record: record_metadata(metadata[:record_metadata]),
+              result_class: metadata[:result_class],
               status: metadata[:status]
             }.compact.freeze
           end
@@ -27,12 +29,14 @@ module Julewire
           def record_metadata(value)
             return unless value.is_a?(Hash)
 
-            {
+            labels = value[:labels]
+            metadata = {
               event: value[:event],
-              labels: value[:labels].is_a?(Hash) ? Fields::FieldSet.deep_dup(value[:labels]) : nil,
               severity: value[:severity],
               source: value[:source]
             }.compact
+            metadata[:labels] = Fields::FieldSet.deep_dup(labels) if labels.is_a?(Hash)
+            metadata
           end
         end
       end

data/lib/julewire/core/diagnostics/health.rb

@@ -1,5 +1,8 @@
 # frozen_string_literal: true
 
+require "concurrent/atomic/atomic_fixnum"
+require "concurrent/atomic/atomic_reference"
+
 module Julewire
   module Core
     module Diagnostics
@@ -15,106 +18,91 @@ module Julewire
           @callback_metadata = callback_metadata
           @failure_counter = failure_counter
           @track_failures = track_failures
-          @mutex = Mutex.new
-          counter_keys = counter_keys.to_a
+          counter_keys = counter_keys.map { it }
           counter_keys = counter_keys.union([:failures]) if @track_failures
-          @counts = counter_keys.to_h { [it, 0] }
-          @current_degradation = nil
-          @last_callback_failure = nil
-          @last_failure = nil
-          @last_loss = nil
+          @counts = counter_keys.to_h { [it, Concurrent::AtomicFixnum.new] }
+          @current_degradation = Concurrent::AtomicReference.new
+          @last_callback_failure = Concurrent::AtomicReference.new
+          @last_failure = Concurrent::AtomicReference.new
+          @last_loss = Concurrent::AtomicReference.new
         end
 
         def increment(key, by: 1)
-          @mutex.synchronize { increment_unlocked(key, by: by) }
+          @counts.fetch(key).increment(by)
         end
 
         def counts
-          @mutex.synchronize { @counts.dup.freeze }
+          @counts.to_h { |key, counter| [key, counter.value] }.freeze
         end
 
         def degradation_marker
-          @mutex.synchronize { @current_degradation }
+          @current_degradation.get
         end
 
         def degraded?(status_from: :current)
-          @mutex.synchronize { degraded_unlocked?(status_from) }
+          degraded_from?(status_from)
         end
 
         def last_callback_failure
-          @mutex.synchronize { @last_callback_failure }
+          @last_callback_failure.get
         end
 
         def last_failure
-          @mutex.synchronize { @last_failure }
+          @last_failure.get
         end
 
         def last_loss
-          @mutex.synchronize { @last_loss }
-        end
-
-        def clear_degradation
-          @mutex.synchronize { @current_degradation = nil }
+          @last_loss.get
         end
 
         def clear_degradation_if_unchanged(marker)
-          @mutex.synchronize { @current_degradation = nil if @current_degradation.equal?(marker) }
+          @current_degradation.compare_and_set(marker, nil)
         end
 
         def clear_failures!
-          @mutex.synchronize do
-            @current_degradation = nil
-            @last_callback_failure = nil
-            @last_failure = nil
-            @last_loss = nil
-          end
+          @current_degradation.set(nil)
+          @last_callback_failure.set(nil)
+          @last_failure.set(nil)
+          @last_loss.set(nil)
           self
         end
 
         def record_failure(error, callback: nil, counter: @failure_counter, degrade: true, **metadata)
           failure = FailureSnapshot.build(error, **metadata)
-          @mutex.synchronize do
-            increment_unlocked(:failures) if @track_failures
-            increment_unlocked(counter) if counter && counter != :failures && @counts.key?(counter)
-            @last_failure = failure
-            @current_degradation = failure if degrade
-          end
+          increment(:failures) if @track_failures
+          increment(counter) if counter && !counter.equal?(:failures) && @counts.key?(counter)
+          @last_failure.set(failure)
+          @current_degradation.set(failure) if degrade
           notify_failure_callback(callback, error, metadata)
           failure
         end
 
         def record_callback_failure(callback_failure)
-          @mutex.synchronize do
-            @last_callback_failure = callback_failure.to_h
-            increment_unlocked(@callback_failure_counter) if @callback_failure_counter
-          end
+          @last_callback_failure.set(callback_failure.to_h)
+          increment(@callback_failure_counter) if @callback_failure_counter
         end
 
         def record_loss(reason:, counter: reason, degrade: true, **metadata)
           loss = { reason: reason }.merge(metadata).compact.freeze
-          @mutex.synchronize do
-            increment_unlocked(counter) if counter && @counts.key?(counter)
-            @last_loss = loss
-            @current_degradation = loss if degrade
-          end
+          increment(counter) if counter && @counts.key?(counter)
+          @last_loss.set(loss)
+          @current_degradation.set(loss) if degrade
           loss
         end
 
         def record_success
-          @mutex.synchronize { @current_degradation = nil }
+          @current_degradation.set(nil)
           self
         end
 
         def snapshot(status: nil, status_from: :current, include_loss: false, **fields)
-          @mutex.synchronize do
-            result = {
-              counts: @counts.dup.freeze,
-              last_failure: @last_failure,
-              status: status || (degraded_unlocked?(status_from) ? :degraded : :ok)
-            }
-            result[:last_loss] = @last_loss if include_loss
-            result.merge(fields).compact.freeze
-          end
+          result = {
+            counts: counts,
+            last_failure: last_failure,
+            status: status || (degraded_from?(status_from) ? :degraded : :ok)
+          }
+          result[:last_loss] = last_loss if include_loss
+          result.merge(fields).compact.freeze
         end
 
         private
@@ -124,16 +112,12 @@ module Julewire
           record_callback_failure(callback_result) if CallbackNotifier.failure?(callback_result)
         end
 
-        def increment_unlocked(key, by: 1)
-          @counts[key] = @counts.fetch(key) + by
-        end
-
-        def degraded_unlocked?(status_from)
+        def degraded_from?(status_from)
           case status_from
           when :current
-            !!@current_degradation
+            !!degradation_marker
           when :failure_or_loss
-            !!(@last_failure || @last_loss)
+            !!(last_failure || last_loss)
           else
             raise ArgumentError, "unknown health status source: #{status_from.inspect}"
           end

data/lib/julewire/core/diagnostics/integration_health_store.rb

@@ -1,60 +1,47 @@
 # frozen_string_literal: true
 
+require "concurrent/map"
+
 module Julewire
   module Core
     module Diagnostics
       class IntegrationHealthStore
         def initialize
-          @mutex = Mutex.new
-          @entries = {}
+          @entries = Concurrent::Map.new
         end
 
         def record_failure(integration, error, **metadata)
           name = normalize_name(integration)
           metadata = { phase: :integration, integration: name }.merge(metadata)
-          @mutex.synchronize do
-            entry_for(name).record_failure(error, **metadata)
-          end
-          nil
-        rescue StandardError
+          entry_for(name).record_failure(error, **metadata)
           nil
         end
 
         def record_success(integration)
           name = normalize_name(integration)
-          @mutex.synchronize do
-            entry_for(name).record_success
-          end
-          nil
-        rescue StandardError
+          entry_for(name).record_success
           nil
         end
 
         def health
-          @mutex.synchronize do
-            @entries.to_h { |name, entry| [name, entry.snapshot] }.freeze
-          end
+          @entries.each_pair.with_object({}) do |(name, entry), snapshot|
+            snapshot[name] = entry.snapshot
+          end.freeze
         end
 
         def reset!
-          @mutex.synchronize { @entries.clear }
-          nil
-        end
-
-        def after_fork!
-          @mutex = Mutex.new
-          @entries = {}
+          @entries.clear
           nil
         end
 
         private
 
         def entry_for(name)
-          @entries[name] ||= Health.new(counter_keys: [:failures])
+          @entries.compute_if_absent(name) { Health.new(counter_keys: []) }
         end
 
         def normalize_name(value)
-          Core.normalize_name(value, name: :integration)
+          Core.normalize_name(value)
         rescue StandardError
           :unknown
         end

data/lib/julewire/core/diagnostics/internal_records.rb

@@ -5,11 +5,10 @@ module Julewire
     module Diagnostics
       module InternalRecords
         class << self
-          def emit_error(error, error_backtrace_lines:)
-            Core::Records::Draft.build(
+          def emit_error(error)
+            Records::Draft.build(
               {
                 severity: :error,
-                kind: :point,
                 event: "julewire.emit_error",
                 source: "julewire",
                 message: "Julewire emit failed",
@@ -17,17 +16,15 @@ module Julewire
                   error: failure_details(error)
                 }
               },
-              context: {},
-              scope: nil,
-              error_backtrace_lines: error_backtrace_lines
+              context: nil,
+              scope: nil
             )
           end
 
-          def processor_error(processor_name:, error:, record_metadata:, error_backtrace_lines:)
-            Core::Records::Draft.build(
+          def processor_error(processor_name:, error:, record_metadata:)
+            Records::Draft.build(
               {
                 severity: :error,
-                kind: :point,
                 event: "julewire.processor_error",
                 source: "julewire",
                 message: "Julewire processor failed",
@@ -38,9 +35,8 @@ module Julewire
                   record: record_metadata
                 }
               },
-              context: {},
-              scope: nil,
-              error_backtrace_lines: error_backtrace_lines
+              context: nil,
+              scope: nil
             )
           end