flare 0.3.1 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1adc85107fcbe273da78049b6882bda0f234913b743b5c67988b8b90f53261ae
-  data.tar.gz: c46bca41c454100e57ea005ad2a83af2bb7ada763ba48a9cb2ffb19e0811198f
+  metadata.gz: cee6e0910ebcd04f073efc41638823f7ed4d7abf511ab71b68caead994997af1
+  data.tar.gz: f28d669e444b2abb8b6043990b8f1cfc5cbd39abc14d25942cf5f41e08a80cb2
 SHA512:
-  metadata.gz: 76e48757e0e4dedffd5363b6cbb671271f2e1447cc1b06f1f1d1518315ab775a9198f72c522cbb74fc3969c54e93b2baca82d65919855025247e28897f015f9f
-  data.tar.gz: 89661ee1a5b0a1aa90cd21d410c087c359351b2714211f55d4a33bb35a739ef943348771fb7bf3a9cb7f4e8fd66446b0f213ac34f1c4da0e9f54b0babea74ec5
+  metadata.gz: 06647d35a64ae38dafb6e946b181fd87a8b8dc8e7065c5b0d4af3cb47419bd5be5dc9e930f3aa322740e824279169d9b2ae867c2b3c14dfda22c1d01be25507d
+  data.tar.gz: 837f70a2ef44f4d0f8c415cc15c6455ab5250b63e7dd06580ad640492385d8d1e6842cda3f56b036d918bb38c7b55438a2d60546a6c41fd7ce77a429b2d1793a

data/lib/flare/deadline.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Flare
+  # A small monotonic deadline shared by lifecycle operations. A nil timeout
+  # represents an unbounded operation.
+  class Deadline
+    def initialize(timeout)
+      @expires_at = monotonic_now + [timeout.to_f, 0].max unless timeout.nil?
+    end
+
+    def remaining
+      return nil unless @expires_at
+
+      [@expires_at - monotonic_now, 0].max
+    end
+
+    def expired?
+      remaining == 0
+    end
+
+    private
+
+    def monotonic_now
+      Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    end
+  end
+end

data/lib/flare/filtering_span_processor.rb

@@ -4,6 +4,8 @@ require "concurrent/atomic/atomic_fixnum"
 require "logger"
 require "opentelemetry/sdk"
 
+require_relative "deadline"
+
 module Flare
   # BSP-shaped span processor whose filter is `sampled OR marked` instead
   # of BSP's `sampled` (BSP early-returns on RECORD_ONLY spans -- our
@@ -17,6 +19,7 @@ module Flare
   class FilteringSpanProcessor
     SUCCESS = OpenTelemetry::SDK::Trace::Export::SUCCESS
     FAILURE = OpenTelemetry::SDK::Trace::Export::FAILURE
+    TIMEOUT = OpenTelemetry::SDK::Trace::Export::TIMEOUT
 
     DEFAULT_MAX_QUEUE = 5_000
     DEFAULT_FLUSH_INTERVAL = 5      # seconds
@@ -47,6 +50,10 @@ module Flare
       @mutex            = Mutex.new
       @cond             = ConditionVariable.new
       @stopped          = false
+      @active_exports   = 0
+      @export_completion_sequence = 0
+      @last_export_result = SUCCESS
+      @flush_owner      = nil
       @pid              = $$
 
       @dropped_count       = Concurrent::AtomicFixnum.new(0)
@@ -78,19 +85,43 @@ module Flare
     end
 
     def force_flush(timeout: nil)
-      drain_and_export(include_pending: true)
-      SUCCESS
+      detect_forking
+      deadline = Deadline.new(timeout)
+      return TIMEOUT unless begin_flush(deadline)
+      prior_result = @flush_prior_result
+
+      batch = snapshot_for_flush(deadline)
+      return TIMEOUT unless batch
+
+      operation = start_flush_export(batch, deadline)
+      result = wait_for_flush_export(operation, deadline)
+      [prior_result, result].max
+    ensure
+      finish_flush if @flush_owner == Thread.current
     end
 
     def shutdown(timeout: nil)
-      @mutex.synchronize do
+      detect_forking
+      deadline = Deadline.new(timeout)
+      return TIMEOUT unless lock_before_deadline(deadline)
+
+      begin
         @stopped = true
         @cond.broadcast
+      ensure
+        @mutex.unlock
       end
-      @worker.join(timeout || 5)
-      drain_and_export(include_pending: true)
-      @exporter.shutdown(timeout: timeout) if @exporter.respond_to?(:shutdown)
-      SUCCESS
+      @worker.join(deadline.remaining || 5)
+      return TIMEOUT if @worker.alive? || deadline.expired?
+
+      result = force_flush(timeout: deadline.remaining)
+      return result unless result == SUCCESS
+      return TIMEOUT if deadline.expired?
+
+      exporter_result = @exporter.shutdown(timeout: deadline.remaining) if @exporter.respond_to?(:shutdown)
+      return TIMEOUT if deadline.expired?
+
+      exporter_result || SUCCESS
     end
 
     def buffer_size
@@ -124,7 +155,8 @@ module Flare
       until stopped?
         @mutex.synchronize do
           timeout = next_wait_timeout
-          @cond.wait(@mutex, timeout) if @ready_queue.empty? && !@stopped
+          waiting_for_export = @flush_owner || @active_exports.positive?
+          @cond.wait(@mutex, timeout) if (@ready_queue.empty? || waiting_for_export) && !@stopped
         end
         drain_and_export
       end
@@ -134,30 +166,141 @@ module Flare
       @mutex.synchronize { @stopped }
     end
 
-    def drain_and_export(include_pending: false)
+    def drain_and_export
       batch = nil
       @mutex.synchronize do
         promote_due_delayed_traces
+        return if @ready_queue.empty? || @flush_owner || @active_exports.positive?
 
-        if include_pending
-          @ready_queue.concat(@pending_by_trace.values.flatten)
-          @pending_by_trace.clear
-          @trace_order.clear
-          @pending_count = 0
-          unmark_delayed_traces
-          @delayed_ready_by_trace.clear
+        batch = @ready_queue
+        @ready_queue = []
+        @active_exports += 1
+      end
+
+      result = export_batch(batch, timeout: @export_timeout)
+    ensure
+      export_finished(result || FAILURE) if batch
+    end
+
+    def begin_flush(deadline)
+      return false unless lock_before_deadline(deadline)
+
+      begin
+        initial_sequence = @export_completion_sequence
+        while @flush_owner && @flush_owner != Thread.current
+          return false if deadline.expired?
+
+          @cond.wait(@mutex, deadline.remaining)
         end
+        @flush_owner = Thread.current
+
+        while @active_exports.positive?
+          return false if deadline.expired?
 
-        return if @ready_queue.empty?
+          @cond.wait(@mutex, deadline.remaining)
+        end
+        @flush_prior_result = if @export_completion_sequence > initial_sequence
+          @last_export_result
+        else
+          SUCCESS
+        end
+      ensure
+        @mutex.unlock
+      end
+      true
+    end
+
+    def finish_flush
+      @mutex.synchronize do
+        @flush_owner = nil
+        @cond.broadcast
+      end
+    end
+
+    def snapshot_for_flush(deadline)
+      return unless lock_before_deadline(deadline)
+
+      begin
+        @ready_queue.concat(@pending_by_trace.values.flatten)
+        @pending_by_trace.clear
+        @trace_order.clear
+        @pending_count = 0
+        unmark_delayed_traces
+        @delayed_ready_by_trace.clear
         batch = @ready_queue
         @ready_queue = []
+        batch
+      ensure
+        @mutex.unlock
+      end
+    end
+
+    def start_flush_export(batch, deadline)
+      operation = { done: false, result: nil }
+      @mutex.synchronize { @active_exports += 1 }
+      Thread.new do
+        result = batch.empty? ? SUCCESS : export_batch(batch, timeout: deadline.remaining)
+        if result == SUCCESS && !deadline.expired? && @exporter.respond_to?(:force_flush)
+          result = @exporter.force_flush(timeout: deadline.remaining)
+        end
+        result = TIMEOUT if deadline.expired?
+        operation[:result] = result
+      rescue StandardError => e
+        @exception_count.increment
+        @logger.warn("[Flare::FilteringSpanProcessor] force flush failed: #{e.class}: #{e.message}")
+        operation[:result] = FAILURE
+      ensure
+        @mutex.synchronize do
+          operation[:done] = true
+          complete_export(operation[:result])
+        end
       end
+      operation
+    end
+
+    def wait_for_flush_export(operation, deadline)
+      @mutex.synchronize do
+        until operation[:done]
+          return TIMEOUT if deadline.expired?
 
-      result = @exporter.export(batch, timeout: @export_timeout)
+          @cond.wait(@mutex, deadline.remaining)
+        end
+      end
+      operation[:result]
+    end
+
+    def export_batch(batch, timeout:)
+      result = @exporter.export(batch, timeout: timeout)
       @failed_export_count.increment if result != SUCCESS
+      result
     rescue StandardError => e
       @exception_count.increment
       @logger.warn("[Flare::FilteringSpanProcessor] export failed: #{e.class}: #{e.message}")
+      FAILURE
+    end
+
+    def export_finished(result)
+      @mutex.synchronize do
+        complete_export(result)
+      end
+    end
+
+    def complete_export(result)
+      @active_exports -= 1
+      @export_completion_sequence += 1
+      @last_export_result = result || FAILURE
+      @cond.broadcast
+    end
+
+    def lock_before_deadline(deadline)
+      return @mutex.lock unless deadline.remaining
+
+      until @mutex.try_lock
+        return false if deadline.expired?
+
+        sleep([deadline.remaining, 0.001].min)
+      end
+      true
     end
 
     def mark_trace_ready(trace_id)
@@ -255,18 +398,25 @@ module Flare
     def detect_forking
       return if @pid == $$
 
-      @mutex.synchronize do
-        return if @pid == $$
-
-        @pid = $$
-        @pending_by_trace.clear
-        @trace_order.clear
-        @ready_queue.clear
-        @delayed_ready_by_trace.clear
-        @pending_count = 0
-        @stopped = false
-        start_worker
-      end
+      # The child only retains the forking thread. Replace synchronization
+      # objects so it cannot inherit locks or active-export bookkeeping owned
+      # by vanished threads.
+      @pid = $$
+      @mutex = Mutex.new
+      @cond = ConditionVariable.new
+      @pending_by_trace = {}
+      @trace_order = []
+      @ready_queue = []
+      @delayed_ready_by_trace = {}
+      @pending_count = 0
+      @active_exports = 0
+      @export_completion_sequence = 0
+      @last_export_result = SUCCESS
+      @flush_prior_result = SUCCESS
+      @flush_owner = nil
+      @stopped = false
+      @worker = nil
+      start_worker
     end
 
     def start_worker

data/lib/flare/http_transport.rb

@@ -3,12 +3,15 @@
 require "net/http"
 require "uri"
 
+require_relative "deadline"
+
 module Flare
   # Tiny HTTP wrapper used by TraceExporter (and anything else that wants
   # to PUT/POST without pulling in a heavy client). Designed for injection
   # at the boundary so tests can swap in a recording fake; no other moving
   # parts.
   class HttpTransport
+    DeadlineExceeded = Class.new(StandardError)
     DEFAULT_OPEN_TIMEOUT  = 2
     DEFAULT_READ_TIMEOUT  = 5
     DEFAULT_WRITE_TIMEOUT = 5
@@ -28,35 +31,46 @@ module Flare
       @write_timeout = write_timeout
     end
 
-    def get(url, headers = {})
-      request(url, nil, headers, Net::HTTP::Get)
+    def get(url, headers = {}, timeout: nil)
+      request(url, nil, headers, Net::HTTP::Get, timeout: timeout)
     end
 
-    def put(url, body, headers = {})
-      request(url, body, headers, Net::HTTP::Put)
+    def put(url, body, headers = {}, timeout: nil)
+      request(url, body, headers, Net::HTTP::Put, timeout: timeout)
     end
 
-    def post(url, body, headers = {})
-      request(url, body, headers, Net::HTTP::Post)
+    def post(url, body, headers = {}, timeout: nil)
+      request(url, body, headers, Net::HTTP::Post, timeout: timeout)
     end
 
     private
 
-    def request(url, body, headers, klass)
+    def request(url, body, headers, klass, timeout: nil)
+      deadline = Deadline.new(timeout)
+      raise DeadlineExceeded if deadline.expired?
+
       uri = URI(url)
       http = Net::HTTP.new(uri.host, uri.port)
       http.use_ssl     = uri.scheme == "https"
-      http.open_timeout = @open_timeout
-      http.read_timeout = @read_timeout
-      http.write_timeout = @write_timeout if http.respond_to?(:write_timeout=)
+      http.open_timeout = effective_timeout(@open_timeout, deadline.remaining)
+      http.read_timeout = effective_timeout(@read_timeout, deadline.remaining)
+      http.write_timeout = effective_timeout(@write_timeout, deadline.remaining) if http.respond_to?(:write_timeout=)
 
       req = klass.new(uri.request_uri == "" ? "/" : uri.request_uri)
       headers.each { |k, v| req[k] = v }
       req.body = body if body
 
       response = http.request(req)
+      raise DeadlineExceeded if deadline.expired?
+
       hash = response.each_header.to_h
       Response.new(code: response.code.to_s, body: response.body, headers: hash)
     end
+
+    def effective_timeout(configured_timeout, remaining)
+      return configured_timeout unless remaining
+
+      [configured_timeout, remaining].min
+    end
   end
 end

data/lib/flare/lifecycle.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require "opentelemetry/sdk"
+
+require_relative "deadline"
+
+module Flare
+  module Lifecycle
+    # Flush all trace processors and the separate aggregated metric pipeline
+    # using one monotonic timeout budget. This is safe to call from lifecycle
+    # hooks for short-lived and fork-per-job workers.
+    def force_flush(timeout: nil)
+      deadline = Deadline.new(timeout)
+      results = []
+
+      results << tracer_provider_for_flush.force_flush(timeout: deadline.remaining)
+      return OpenTelemetry::SDK::Trace::Export::TIMEOUT if deadline.expired?
+
+      flusher = metric_flusher_for_flush
+      results << flusher.force_flush(timeout: deadline.remaining) if flusher
+      return OpenTelemetry::SDK::Trace::Export::TIMEOUT if deadline.expired?
+
+      results.max || OpenTelemetry::SDK::Trace::Export::SUCCESS
+    rescue => e
+      warn "[Flare] Telemetry flush error: #{e.message}"
+      OpenTelemetry::SDK::Trace::Export::FAILURE
+    end
+
+    def tracer_provider_for_flush
+      OpenTelemetry.tracer_provider
+    end
+
+    def metric_flusher_for_flush
+      @metric_flusher
+    end
+  end
+
+  extend Lifecycle
+end

data/lib/flare/metric_flusher.rb

@@ -2,6 +2,9 @@
 
 require "concurrent/timer_task"
 require "concurrent/executor/fixed_thread_pool"
+require "opentelemetry/sdk"
+
+require_relative "deadline"
 
 module Flare
   # Background threads that periodically drain in-memory metrics and submit
@@ -10,6 +13,9 @@ module Flare
   #
   # Fork-safe: detects forked processes and restarts automatically.
   class MetricFlusher
+    SUCCESS = OpenTelemetry::SDK::Trace::Export::SUCCESS
+    FAILURE = OpenTelemetry::SDK::Trace::Export::FAILURE
+    TIMEOUT = OpenTelemetry::SDK::Trace::Export::TIMEOUT
     DEFAULT_INTERVAL = 60 # seconds
     DEFAULT_SHUTDOWN_TIMEOUT = 5 # seconds
 
@@ -23,6 +29,7 @@ module Flare
       @health_reporters = Array(health_reporters)
       @pid = $$
       @stopped = false
+      initialize_synchronization
     end
 
     def start
@@ -40,23 +47,25 @@ module Flare
       }) { post_to_pool }
     end
 
-    def stop
+    def stop(timeout: @shutdown_timeout)
       return if @stopped
 
+      deadline = Deadline.new(timeout)
       @stopped = true
 
       log "Shutting down metrics flusher, draining remaining metrics..."
 
       if @timer
         @timer.shutdown
-        @timer.wait_for_termination(1)
+        @timer.wait_for_termination([deadline.remaining || 1, 1].min)
         @timer.kill unless @timer.shutdown?
       end
 
+      force_flush(timeout: deadline.remaining)
+
       if @pool
-        post_to_pool # one last drain
         @pool.shutdown
-        pool_terminated = @pool.wait_for_termination(@shutdown_timeout)
+        pool_terminated = @pool.wait_for_termination(deadline.remaining || @shutdown_timeout)
         @pool.kill unless pool_terminated
       end
 
@@ -70,14 +79,11 @@ module Flare
     end
 
     # Manually trigger a flush (useful for testing or forced flushes).
-    def flush_now
+    def flush_now(timeout: nil)
       return 0 unless @storage && @submitter
 
-      record_health_metrics
-      drained = @storage.drain
-      return 0 if drained.empty?
-
-      count, error = @submitter.submit(drained)
+      detect_forking
+      count, error, = flush_synchronously(Deadline.new(timeout))
       if error
         warn "[Flare] Metric submission error: #{error.message}"
       end
@@ -87,6 +93,21 @@ module Flare
       0
     end
 
+    def force_flush(timeout: nil)
+      return SUCCESS unless @storage && @submitter
+
+      detect_forking
+      deadline = Deadline.new(timeout)
+      _count, error, timed_out = flush_synchronously(deadline)
+      return TIMEOUT if timed_out || deadline.expired?
+      return FAILURE if error
+
+      SUCCESS
+    rescue => e
+      warn "[Flare] Metric flush error: #{e.message}"
+      FAILURE
+    end
+
     def running?
       @timer&.running? || false
     end
@@ -96,22 +117,46 @@ module Flare
     # after_fork hooks.
     def after_fork
       @pid = $$
-      restart
+      @storage.after_fork if @storage.respond_to?(:after_fork)
+      initialize_synchronization
+      @timer = nil
+      @pool = nil
+      start
     end
 
     private
 
+    def detect_forking
+      after_fork if @pid != $$
+    end
+
+    def initialize_synchronization
+      @submission_mutex = Mutex.new
+      @submission_condition = ConditionVariable.new
+      @pending_submissions = 0
+      @flush_owner = nil
+    end
+
     def post_to_pool
+      return unless reserve_background_submission
+
       record_health_metrics
       drained = @storage.drain
       if drained.empty?
         log "No metrics to flush"
+        background_submission_finished
         return
       end
 
       log "Drained #{drained.size} metric keys for submission"
-      @pool.post { submit_to_cloud(drained) }
+      posted = @pool.post do
+        submit_to_cloud(drained)
+      ensure
+        background_submission_finished
+      end
+      background_submission_finished unless posted
     rescue => e
+      background_submission_finished
       warn "[Flare] Metric drain error: #{e.message}"
     end
 
@@ -124,6 +169,103 @@ module Flare
       warn "[Flare] Metric submission error: #{e.message}"
     end
 
+    def reserve_background_submission
+      @submission_mutex.synchronize do
+        return false if @flush_owner || @pending_submissions.positive?
+
+        @pending_submissions += 1
+        true
+      end
+    end
+
+    def background_submission_finished
+      @submission_mutex.synchronize do
+        @pending_submissions -= 1 if @pending_submissions.positive?
+        @submission_condition.broadcast
+      end
+    end
+
+    def flush_synchronously(deadline)
+      return [0, nil, true] unless begin_synchronous_flush(deadline)
+
+      record_health_metrics
+      drained = @storage.drain
+      return [0, nil, false] if drained.empty?
+
+      submit_with_deadline(drained, deadline)
+    ensure
+      finish_synchronous_flush if @flush_owner == Thread.current
+    end
+
+    def begin_synchronous_flush(deadline)
+      @submission_mutex.synchronize do
+        while @flush_owner && @flush_owner != Thread.current
+          return false if deadline.expired?
+
+          @submission_condition.wait(@submission_mutex, deadline.remaining)
+        end
+        @flush_owner = Thread.current
+
+        while @pending_submissions.positive?
+          return false if deadline.expired?
+
+          @submission_condition.wait(@submission_mutex, deadline.remaining)
+        end
+      end
+      true
+    end
+
+    def finish_synchronous_flush
+      @submission_mutex.synchronize do
+        @flush_owner = nil
+        @submission_condition.broadcast
+      end
+    end
+
+    def submit_metrics(drained, timeout:)
+      parameters = @submitter.method(:submit).parameters
+      accepts_timeout = parameters.any? do |type, name|
+        type == :keyrest || ([:key, :keyreq].include?(type) && name == :timeout)
+      end
+
+      if accepts_timeout
+        @submitter.submit(drained, timeout: timeout)
+      else
+        @submitter.submit(drained)
+      end
+    end
+
+    def submit_with_deadline(drained, deadline)
+      operation = { done: false, count: 0, error: nil }
+      @submission_mutex.synchronize { @pending_submissions += 1 }
+      Thread.new do
+        operation[:count], operation[:error] = submit_metrics(drained, timeout: deadline.remaining)
+      rescue => e
+        operation[:error] = e
+      ensure
+        @submission_mutex.synchronize do
+          operation[:done] = true
+          @pending_submissions -= 1
+          @submission_condition.broadcast
+        end
+      end
+
+      @submission_mutex.synchronize do
+        until operation[:done]
+          return [0, nil, true] if deadline.expired?
+
+          @submission_condition.wait(@submission_mutex, deadline.remaining)
+        end
+      end
+
+      timed_out = deadline.expired? || deadline_error?(operation[:error])
+      [operation[:count], operation[:error], timed_out]
+    end
+
+    def deadline_error?(error)
+      defined?(MetricSubmitter::DeadlineExceeded) && error.is_a?(MetricSubmitter::DeadlineExceeded)
+    end
+
     def record_health_metrics
       @health_reporters.each { |reporter| reporter.record(@storage) }
     rescue => e