julewire-core 1.0.0 → 1.1.0

data/lib/julewire/core/destinations/sink.rb

@@ -9,7 +9,6 @@ module Julewire
             reject_output_array!(output)
             return output if wrapped?(output)
 
-            validate_writeable!(output)
             SynchronizedOutput.new(output, close_output: close_output)
           end
 

data/lib/julewire/core/destinations/synchronized_output.rb

@@ -4,16 +4,21 @@ module Julewire
   module Core
     module Destinations
       class SynchronizedOutput
+        TIMEOUT_PARAMETER_TYPES = %i[key keyreq].freeze
+        private_constant :TIMEOUT_PARAMETER_TYPES
+
         def initialize(output, close_output: false)
           Sink.validate_writeable!(output)
           @output = output
           @close_output = close_output
           @mutex = Mutex.new
+          @lifecycle_mutex = Mutex.new
+          @lifecycle = lifecycle_methods
         end
 
         def after_fork!
-          @mutex = Mutex.new
           @output.after_fork! if @output.respond_to?(:after_fork!)
+          @lifecycle = lifecycle_methods
           self
         end
 
@@ -22,34 +27,66 @@ module Julewire
         def resource_identity = @output
 
         def write(value)
-          @mutex.synchronize { @output.write(value) }
+          @mutex.synchronize do
+            return false if output_closed?
+
+            @output.write(value)
+          end
         end
 
-        def flush
-          @mutex.synchronize do
-            return true unless @output.respond_to?(:flush)
+        def flush(timeout: nil)
+          @lifecycle_mutex.synchronize do
+            lifecycle = @lifecycle[:flush]
+            return true unless lifecycle
 
-            @output.flush != false
+            lifecycle_succeeded?(call_lifecycle(:flush, lifecycle, timeout: timeout))
           end
         end
 
-        def close
-          @mutex.synchronize do
-            return true if output_closed?
-
-            result = if @close_output && @output.respond_to?(:close)
-                       @output.close
-                     elsif @output.respond_to?(:flush)
-                       @output.flush
-                     end
-            result != false
+        def close(timeout: nil)
+          @lifecycle_mutex.synchronize do
+            # Close is terminal: lifecycle calls stay serialized, and the write mutex
+            # keeps the underlying output from being closed while a write is in flight.
+            @mutex.synchronize do
+              return true if output_closed?
+
+              if @close_output && (close_lifecycle = @lifecycle[:close])
+                return lifecycle_succeeded?(call_lifecycle(:close, close_lifecycle, timeout: timeout))
+              end
+
+              return true unless (flush_lifecycle = @lifecycle[:flush])
+
+              lifecycle_succeeded?(call_lifecycle(:flush, flush_lifecycle, timeout: timeout))
+            end
           end
         end
 
         private
 
         def output_closed?
-          @output.respond_to?(:closed?) ? @output.closed? : false
+          @output.respond_to?(:closed?) && @output.closed?
+        end
+
+        def lifecycle_succeeded?(result) = !result.equal?(false)
+
+        def call_lifecycle(name, lifecycle, timeout:)
+          return @output.public_send(name, timeout: timeout) if lifecycle.fetch(:timeout)
+
+          @output.public_send(name)
+        end
+
+        def lifecycle_methods
+          %i[flush close].each_with_object({}) do |name, methods|
+            next unless @output.respond_to?(name)
+
+            method = @output.public_method(name)
+            methods[name] = { timeout: accepts_timeout_keyword?(method) }
+          end
+        end
+
+        def accepts_timeout_keyword?(method)
+          method.parameters.any? { |type, name| TIMEOUT_PARAMETER_TYPES.include?(type) && name == :timeout } ||
+            method.parameters.any? { |type, _name| type == :keyrest }
         end
       end
     end

data/lib/julewire/core/destinations/tail_sampling.rb

@@ -29,7 +29,8 @@ module Julewire
           slow_ms
         ].freeze
         private_constant :COUNTER_KEYS, :ERROR_RANK, :OPTION_KEYS
-
+        Sampling = Processing::Sampling
+        private_constant :Sampling
         TailOptions = Data.define(:decider, :max_executions, :max_records_per_execution, :name, :on_drop, :on_failure,
                                   :sample_rate, :slow_ms)
         private_constant :TailOptions
@@ -42,7 +43,7 @@ module Julewire
           @destination = Registry.validate!(destination)
           @name = Destinations.normalize_name(options.name)
           @sample_rate = options.sample_rate
-          Processing::Sampling.threshold_for(options.sample_rate)
+          Sampling.threshold_for(options.sample_rate)
           @slow_ms = validate_slow_ms(options.slow_ms)
           @max_executions = Validation.validate_integer_limit!(
             options.max_executions,
@@ -65,7 +66,7 @@ module Julewire
 
         def emit(record)
           result = @mutex.synchronize { accept_record(record) }
-          result.losses.compact.each { notify_drop(it) }
+          result.losses.each { notify_drop(it) }
           result.records.each { emit_target(it) }
           nil
         rescue StandardError => e
@@ -160,7 +161,7 @@ module Julewire
             @order << key
             Buffer.new([])
           end)
-          if buffer.records.length >= @max_records_per_execution
+          if buffer.records.length == @max_records_per_execution
             dropped = buffer.records.shift
             losses << record_loss(:overflow_dropped, dropped)
           end
@@ -174,8 +175,7 @@ module Julewire
           return [] if @buffers.length < @max_executions
 
           oldest = @order.shift
-          buffer = @buffers.delete(oldest)
-          Array(buffer&.records).map { record_loss(:overflow_dropped, it) }
+          @buffers.delete(oldest).records.map { record_loss(:overflow_dropped, it) }
         end
 
         def finish_execution(key, summary)
@@ -205,7 +205,7 @@ module Julewire
         end
 
         def keep_execution?(record, key)
-          return !!@decider.call(record, key: key) if @decider
+          return @decider.call(record, key: key) if @decider
 
           default_keep_execution?(record, key)
         rescue StandardError => e
@@ -214,7 +214,7 @@ module Julewire
         end
 
         def default_keep_execution?(record, key)
-          error_record?(record) || slow_record?(record) || Processing::Sampling.keep?(rate: @sample_rate, key: key)
+          error_record?(record) || slow_record?(record) || Sampling.keep?(rate: @sample_rate, key: key)
         end
 
         def error_record?(record)
@@ -235,7 +235,7 @@ module Julewire
         def summary_record?(record) = record[:kind] == :summary
 
         def execution_key(record)
-          reference = record.respond_to?(:lineage) ? record.lineage.root_reference : nil
+          reference = record.lineage.root_reference if record.respond_to?(:lineage)
           reference = record[:execution] unless reference.is_a?(Hash)
           id = field_value(reference, :id)
           return unless id
@@ -256,7 +256,7 @@ module Julewire
         end
 
         def lifecycle(method_name, timeout:)
-          @destination.public_send(method_name, timeout: timeout) != false
+          !@destination.public_send(method_name, timeout: timeout).eql?(false)
         rescue StandardError => e
           record_failure(e, nil, phase: :tail_sampling_lifecycle, action: method_name)
           false
@@ -270,8 +270,6 @@ module Julewire
 
         def record_loss(reason, record)
           @health.record_loss(reason: reason, record_metadata: Records::Metadata.call(record))
-        rescue StandardError => e
-          record_failure(e, record, phase: :tail_sampling_drop)
         end
 
         def notify_drop(loss)
@@ -281,29 +279,22 @@ module Julewire
         end
 
         def record_failure(error, record, phase: :tail_sampling, **metadata)
-          failure_metadata = if @mutex.owned?
-                               record_failure_state(error, record, phase: phase, **metadata)
-                             else
-                               @mutex.synchronize { record_failure_state(error, record, phase: phase, **metadata) }
-                             end
-          @on_failure&.call(error, **failure_metadata)
+          failure_metadata = record_failure_state(error, record, phase: phase, **metadata)
+          @on_failure.call(error, **failure_metadata)
         rescue StandardError
           nil
         end
 
         def record_failure_state(error, record, phase:, **metadata)
-          @health.record_failure(
-            error,
-            **metadata,
-            destination: @name,
-            phase: phase,
-            record_metadata: record ? Records::Metadata.call(record) : nil
-          )
-          metadata.merge(destination: @name, phase: phase)
+          metadata = metadata.merge(destination: @name, phase: phase)
+          record_metadata = record && Records::Metadata.call(record)
+          metadata[:record_metadata] = record_metadata if record_metadata
+          @health.record_failure(error, **metadata)
+          metadata
         end
 
         def health_status
-          return :degraded if @health.last_failure || @health.last_loss&.fetch(:reason) == :overflow_dropped
+          return if @health.last_failure || @health.last_loss&.fetch(:reason) == :overflow_dropped
 
           :ok
         end

data/lib/julewire/core/destinations/write_step.rb

@@ -28,16 +28,16 @@ module Julewire
         def call(record)
           increment(:received)
           payload = format_record(record)
-          return :dropped unless payload
+          return false unless payload
 
           increment(:formatted)
           encoded = encode_payload(payload, record)
-          return :dropped unless encoded
-          return :dropped unless within_limit?(encoded, record)
-          return :dropped unless write(encoded, record)
+          return false unless encoded
+          return false unless within_limit?(encoded, record)
+          return false unless write(encoded, record)
 
           increment(:output_accepted)
-          :accepted
+          true
         end
 
         private

data/lib/julewire/core/destinations.rb

@@ -13,13 +13,6 @@ module Julewire
           nil
         end
 
-        # Private testing seam for `Julewire::Testing.unregister_destination`.
-        def unregister(kind)
-          @factories.delete(normalize_name(kind))
-          nil
-        end
-        private :unregister
-
         def factory_for(kind)
           @factories[normalize_name(kind)]
         end

data/lib/julewire/core/diagnostics/callback_notifier.rb

@@ -11,7 +11,9 @@ module Julewire
 
         private_constant :NestedCallback
 
-        Failure = Data.define(:at, :class_name, :metadata) do
+        Failure = Data.define(:at, :class_name, :metadata)
+
+        class Failure
           def to_h
             {
               action: metadata[:action],
@@ -42,7 +44,7 @@ module Julewire
           end
 
           def failure?(result)
-            result.is_a?(Failure)
+            result.instance_of?(Failure)
           end
 
           def nested_callback_result(metadata)

data/lib/julewire/core/diagnostics/doctor.rb

@@ -52,25 +52,31 @@ module Julewire
 
         def destination_info(destinations)
           destinations.transform_values do |destination|
-            {
-              counts: destination[:counts],
-              last_failure: destination[:last_failure],
-              last_loss: destination[:last_loss],
-              status: destination[:status]
-            }.compact
+            destination_component_info(destination)
           end
         end
 
         def integration_info(integrations)
           integrations.transform_values do |integration|
-            {
-              counts: integration[:counts],
-              last_failure: integration[:last_failure],
-              status: integration[:status]
-            }.compact
+            component_info(integration)
           end
         end
 
+        def destination_component_info(component)
+          info = component_info(component)
+          last_loss = component[:last_loss]
+          info[:last_loss] = last_loss if last_loss
+          info
+        end
+
+        def component_info(component)
+          {
+            counts: component.fetch(:counts),
+            last_failure: component[:last_failure],
+            status: component.fetch(:status)
+          }.compact
+        end
+
         def warnings(health)
           [].tap do |items|
             items << warning(:runtime_closed, "runtime is closed") if health.fetch(:closed)
@@ -82,7 +88,7 @@ module Julewire
 
         def pipeline_warnings(pipeline, items)
           items << warning(:no_destinations, "pipeline has no destinations") unless pipeline.fetch(:configured)
-          if pipeline.fetch(:configured) && pipeline.fetch(:status) != :ok
+          if pipeline.fetch(:configured) && !pipeline.fetch(:status).equal?(:ok)
             items << warning(:pipeline_degraded, "pipeline is #{pipeline.fetch(:status)}")
           end
           component_warnings(
@@ -99,14 +105,15 @@ module Julewire
 
         def component_warnings(components, items, code:, label:)
           components.each do |name, component|
-            next if component[:status] == :ok
+            status = component.fetch(:status)
+            next if status == :ok
 
-            items << warning(code, "#{label} #{name} is #{component[:status]}")
+            items << warning(code, "#{label} #{name} is #{status}")
           end
         end
 
         def warning(code, message)
-          { code: code, message: message }.freeze
+          { code: code, message: message }
         end
       end
     end

data/lib/julewire/core/diagnostics/failure_snapshot.rb

@@ -16,7 +16,11 @@ module Julewire
               integration: metadata[:integration],
               output_class: metadata[:output_class],
               phase: metadata[:phase],
-              record: record_metadata(metadata[:record_metadata])
+              processor: metadata[:processor],
+              reason: metadata[:reason],
+              record: record_metadata(metadata[:record_metadata]),
+              result_class: metadata[:result_class],
+              status: metadata[:status]
             }.compact.freeze
           end
 
@@ -25,12 +29,14 @@ module Julewire
           def record_metadata(value)
             return unless value.is_a?(Hash)
 
-            {
+            labels = value[:labels]
+            metadata = {
               event: value[:event],
-              labels: value[:labels].is_a?(Hash) ? Fields::FieldSet.deep_dup(value[:labels]) : nil,
               severity: value[:severity],
               source: value[:source]
             }.compact
+            metadata[:labels] = Fields::FieldSet.deep_dup(labels) if labels.is_a?(Hash)
+            metadata
           end
         end
       end

data/lib/julewire/core/diagnostics/health.rb

@@ -1,5 +1,8 @@
 # frozen_string_literal: true
 
+require "concurrent/atomic/atomic_fixnum"
+require "concurrent/atomic/atomic_reference"
+
 module Julewire
   module Core
     module Diagnostics
@@ -15,106 +18,91 @@ module Julewire
           @callback_metadata = callback_metadata
           @failure_counter = failure_counter
           @track_failures = track_failures
-          @mutex = Mutex.new
-          counter_keys = counter_keys.to_a
+          counter_keys = counter_keys.map { it }
           counter_keys = counter_keys.union([:failures]) if @track_failures
-          @counts = counter_keys.to_h { [it, 0] }
-          @current_degradation = nil
-          @last_callback_failure = nil
-          @last_failure = nil
-          @last_loss = nil
+          @counts = counter_keys.to_h { [it, Concurrent::AtomicFixnum.new] }
+          @current_degradation = Concurrent::AtomicReference.new
+          @last_callback_failure = Concurrent::AtomicReference.new
+          @last_failure = Concurrent::AtomicReference.new
+          @last_loss = Concurrent::AtomicReference.new
         end
 
         def increment(key, by: 1)
-          @mutex.synchronize { increment_unlocked(key, by: by) }
+          @counts.fetch(key).increment(by)
         end
 
         def counts
-          @mutex.synchronize { @counts.dup.freeze }
+          @counts.to_h { |key, counter| [key, counter.value] }.freeze
         end
 
         def degradation_marker
-          @mutex.synchronize { @current_degradation }
+          @current_degradation.get
         end
 
         def degraded?(status_from: :current)
-          @mutex.synchronize { degraded_unlocked?(status_from) }
+          degraded_from?(status_from)
         end
 
         def last_callback_failure
-          @mutex.synchronize { @last_callback_failure }
+          @last_callback_failure.get
         end
 
         def last_failure
-          @mutex.synchronize { @last_failure }
+          @last_failure.get
         end
 
         def last_loss
-          @mutex.synchronize { @last_loss }
-        end
-
-        def clear_degradation
-          @mutex.synchronize { @current_degradation = nil }
+          @last_loss.get
         end
 
         def clear_degradation_if_unchanged(marker)
-          @mutex.synchronize { @current_degradation = nil if @current_degradation.equal?(marker) }
+          @current_degradation.compare_and_set(marker, nil)
         end
 
         def clear_failures!
-          @mutex.synchronize do
-            @current_degradation = nil
-            @last_callback_failure = nil
-            @last_failure = nil
-            @last_loss = nil
-          end
+          @current_degradation.set(nil)
+          @last_callback_failure.set(nil)
+          @last_failure.set(nil)
+          @last_loss.set(nil)
           self
         end
 
         def record_failure(error, callback: nil, counter: @failure_counter, degrade: true, **metadata)
           failure = FailureSnapshot.build(error, **metadata)
-          @mutex.synchronize do
-            increment_unlocked(:failures) if @track_failures
-            increment_unlocked(counter) if counter && counter != :failures && @counts.key?(counter)
-            @last_failure = failure
-            @current_degradation = failure if degrade
-          end
+          increment(:failures) if @track_failures
+          increment(counter) if counter && !counter.equal?(:failures) && @counts.key?(counter)
+          @last_failure.set(failure)
+          @current_degradation.set(failure) if degrade
           notify_failure_callback(callback, error, metadata)
           failure
         end
 
         def record_callback_failure(callback_failure)
-          @mutex.synchronize do
-            @last_callback_failure = callback_failure.to_h
-            increment_unlocked(@callback_failure_counter) if @callback_failure_counter
-          end
+          @last_callback_failure.set(callback_failure.to_h)
+          increment(@callback_failure_counter) if @callback_failure_counter
         end
 
         def record_loss(reason:, counter: reason, degrade: true, **metadata)
           loss = { reason: reason }.merge(metadata).compact.freeze
-          @mutex.synchronize do
-            increment_unlocked(counter) if counter && @counts.key?(counter)
-            @last_loss = loss
-            @current_degradation = loss if degrade
-          end
+          increment(counter) if counter && @counts.key?(counter)
+          @last_loss.set(loss)
+          @current_degradation.set(loss) if degrade
           loss
         end
 
         def record_success
-          @mutex.synchronize { @current_degradation = nil }
+          @current_degradation.set(nil)
           self
         end
 
         def snapshot(status: nil, status_from: :current, include_loss: false, **fields)
-          @mutex.synchronize do
-            result = {
-              counts: @counts.dup.freeze,
-              last_failure: @last_failure,
-              status: status || (degraded_unlocked?(status_from) ? :degraded : :ok)
-            }
-            result[:last_loss] = @last_loss if include_loss
-            result.merge(fields).compact.freeze
-          end
+          result = {
+            counts: counts,
+            last_failure: last_failure,
+            status: status || (degraded_from?(status_from) ? :degraded : :ok)
+          }
+          result[:last_loss] = last_loss if include_loss
+          result.merge(fields).compact.freeze
         end
 
         private
@@ -124,16 +112,12 @@ module Julewire
           record_callback_failure(callback_result) if CallbackNotifier.failure?(callback_result)
         end
 
-        def increment_unlocked(key, by: 1)
-          @counts[key] = @counts.fetch(key) + by
-        end
-
-        def degraded_unlocked?(status_from)
+        def degraded_from?(status_from)
           case status_from
           when :current
-            !!@current_degradation
+            !!degradation_marker
           when :failure_or_loss
-            !!(@last_failure || @last_loss)
+            !!(last_failure || last_loss)
           else
             raise ArgumentError, "unknown health status source: #{status_from.inspect}"
           end

data/lib/julewire/core/diagnostics/integration_health_store.rb

@@ -1,60 +1,47 @@
 # frozen_string_literal: true
 
+require "concurrent/map"
+
 module Julewire
   module Core
     module Diagnostics
       class IntegrationHealthStore
         def initialize
-          @mutex = Mutex.new
-          @entries = {}
+          @entries = Concurrent::Map.new
         end
 
         def record_failure(integration, error, **metadata)
           name = normalize_name(integration)
           metadata = { phase: :integration, integration: name }.merge(metadata)
-          @mutex.synchronize do
-            entry_for(name).record_failure(error, **metadata)
-          end
-          nil
-        rescue StandardError
+          entry_for(name).record_failure(error, **metadata)
           nil
         end
 
         def record_success(integration)
           name = normalize_name(integration)
-          @mutex.synchronize do
-            entry_for(name).record_success
-          end
-          nil
-        rescue StandardError
+          entry_for(name).record_success
           nil
         end
 
         def health
-          @mutex.synchronize do
-            @entries.to_h { |name, entry| [name, entry.snapshot] }.freeze
-          end
+          @entries.each_pair.with_object({}) do |(name, entry), snapshot|
+            snapshot[name] = entry.snapshot
+          end.freeze
         end
 
         def reset!
-          @mutex.synchronize { @entries.clear }
-          nil
-        end
-
-        def after_fork!
-          @mutex = Mutex.new
-          @entries = {}
+          @entries.clear
           nil
         end
 
         private
 
         def entry_for(name)
-          @entries[name] ||= Health.new(counter_keys: [:failures])
+          @entries.compute_if_absent(name) { Health.new(counter_keys: []) }
         end
 
         def normalize_name(value)
-          Core.normalize_name(value, name: :integration)
+          Core.normalize_name(value)
         rescue StandardError
           :unknown
         end