flare 0.3.1 → 0.4.1

data/lib/flare/metric_flusher.rb

@@ -2,6 +2,9 @@
 
 require "concurrent/timer_task"
 require "concurrent/executor/fixed_thread_pool"
+require "opentelemetry/sdk"
+
+require_relative "deadline"
 
 module Flare
   # Background threads that periodically drain in-memory metrics and submit
@@ -10,6 +13,9 @@ module Flare
   #
   # Fork-safe: detects forked processes and restarts automatically.
   class MetricFlusher
+    SUCCESS = OpenTelemetry::SDK::Trace::Export::SUCCESS
+    FAILURE = OpenTelemetry::SDK::Trace::Export::FAILURE
+    TIMEOUT = OpenTelemetry::SDK::Trace::Export::TIMEOUT
     DEFAULT_INTERVAL = 60 # seconds
     DEFAULT_SHUTDOWN_TIMEOUT = 5 # seconds
 
@@ -23,6 +29,7 @@ module Flare
       @health_reporters = Array(health_reporters)
       @pid = $$
       @stopped = false
+      initialize_synchronization
     end
 
     def start
@@ -40,23 +47,25 @@ module Flare
       }) { post_to_pool }
     end
 
-    def stop
+    def stop(timeout: @shutdown_timeout)
       return if @stopped
 
+      deadline = Deadline.new(timeout)
       @stopped = true
 
       log "Shutting down metrics flusher, draining remaining metrics..."
 
       if @timer
         @timer.shutdown
-        @timer.wait_for_termination(1)
+        @timer.wait_for_termination([deadline.remaining || 1, 1].min)
         @timer.kill unless @timer.shutdown?
       end
 
+      force_flush(timeout: deadline.remaining)
+
       if @pool
-        post_to_pool # one last drain
         @pool.shutdown
-        pool_terminated = @pool.wait_for_termination(@shutdown_timeout)
+        pool_terminated = @pool.wait_for_termination(deadline.remaining || @shutdown_timeout)
         @pool.kill unless pool_terminated
       end
 
@@ -70,14 +79,11 @@ module Flare
     end
 
     # Manually trigger a flush (useful for testing or forced flushes).
-    def flush_now
+    def flush_now(timeout: nil)
       return 0 unless @storage && @submitter
 
-      record_health_metrics
-      drained = @storage.drain
-      return 0 if drained.empty?
-
-      count, error = @submitter.submit(drained)
+      detect_forking
+      count, error, = flush_synchronously(Deadline.new(timeout))
       if error
         warn "[Flare] Metric submission error: #{error.message}"
       end
@@ -87,6 +93,21 @@ module Flare
       0
     end
 
+    def force_flush(timeout: nil)
+      return SUCCESS unless @storage && @submitter
+
+      detect_forking
+      deadline = Deadline.new(timeout)
+      _count, error, timed_out = flush_synchronously(deadline)
+      return TIMEOUT if timed_out || deadline.expired?
+      return FAILURE if error
+
+      SUCCESS
+    rescue => e
+      warn "[Flare] Metric flush error: #{e.message}"
+      FAILURE
+    end
+
     def running?
       @timer&.running? || false
     end
@@ -95,23 +116,49 @@ module Flare
     # on first span in the new process, or manually from Puma/Unicorn
     # after_fork hooks.
     def after_fork
+      return if @pid == $$
+
       @pid = $$
-      restart
+      @storage.after_fork if @storage.respond_to?(:after_fork)
+      initialize_synchronization
+      @timer = nil
+      @pool = nil
+      start
     end
 
     private
 
+    def detect_forking
+      after_fork if @pid != $$
+    end
+
+    def initialize_synchronization
+      @submission_mutex = Mutex.new
+      @submission_condition = ConditionVariable.new
+      @pending_submissions = 0
+      @flush_owner = nil
+    end
+
     def post_to_pool
+      return unless reserve_background_submission
+
       record_health_metrics
       drained = @storage.drain
       if drained.empty?
         log "No metrics to flush"
+        background_submission_finished
         return
       end
 
       log "Drained #{drained.size} metric keys for submission"
-      @pool.post { submit_to_cloud(drained) }
+      posted = @pool.post do
+        submit_to_cloud(drained)
+      ensure
+        background_submission_finished
+      end
+      background_submission_finished unless posted
     rescue => e
+      background_submission_finished
       warn "[Flare] Metric drain error: #{e.message}"
     end
 
@@ -124,6 +171,103 @@ module Flare
       warn "[Flare] Metric submission error: #{e.message}"
     end
 
+    def reserve_background_submission
+      @submission_mutex.synchronize do
+        return false if @flush_owner || @pending_submissions.positive?
+
+        @pending_submissions += 1
+        true
+      end
+    end
+
+    def background_submission_finished
+      @submission_mutex.synchronize do
+        @pending_submissions -= 1 if @pending_submissions.positive?
+        @submission_condition.broadcast
+      end
+    end
+
+    def flush_synchronously(deadline)
+      return [0, nil, true] unless begin_synchronous_flush(deadline)
+
+      record_health_metrics
+      drained = @storage.drain
+      return [0, nil, false] if drained.empty?
+
+      submit_with_deadline(drained, deadline)
+    ensure
+      finish_synchronous_flush if @flush_owner == Thread.current
+    end
+
+    def begin_synchronous_flush(deadline)
+      @submission_mutex.synchronize do
+        while @flush_owner && @flush_owner != Thread.current
+          return false if deadline.expired?
+
+          @submission_condition.wait(@submission_mutex, deadline.remaining)
+        end
+        @flush_owner = Thread.current
+
+        while @pending_submissions.positive?
+          return false if deadline.expired?
+
+          @submission_condition.wait(@submission_mutex, deadline.remaining)
+        end
+      end
+      true
+    end
+
+    def finish_synchronous_flush
+      @submission_mutex.synchronize do
+        @flush_owner = nil
+        @submission_condition.broadcast
+      end
+    end
+
+    def submit_metrics(drained, timeout:)
+      parameters = @submitter.method(:submit).parameters
+      accepts_timeout = parameters.any? do |type, name|
+        type == :keyrest || ([:key, :keyreq].include?(type) && name == :timeout)
+      end
+
+      if accepts_timeout
+        @submitter.submit(drained, timeout: timeout)
+      else
+        @submitter.submit(drained)
+      end
+    end
+
+    def submit_with_deadline(drained, deadline)
+      operation = { done: false, count: 0, error: nil }
+      @submission_mutex.synchronize { @pending_submissions += 1 }
+      Thread.new do
+        operation[:count], operation[:error] = submit_metrics(drained, timeout: deadline.remaining)
+      rescue => e
+        operation[:error] = e
+      ensure
+        @submission_mutex.synchronize do
+          operation[:done] = true
+          @pending_submissions -= 1
+          @submission_condition.broadcast
+        end
+      end
+
+      @submission_mutex.synchronize do
+        until operation[:done]
+          return [0, nil, true] if deadline.expired?
+
+          @submission_condition.wait(@submission_mutex, deadline.remaining)
+        end
+      end
+
+      timed_out = deadline.expired? || deadline_error?(operation[:error])
+      [operation[:count], operation[:error], timed_out]
+    end
+
+    def deadline_error?(error)
+      defined?(MetricSubmitter::DeadlineExceeded) && error.is_a?(MetricSubmitter::DeadlineExceeded)
+    end
+
     def record_health_metrics
       @health_reporters.each { |reporter| reporter.record(@storage) }
     rescue => e

data/lib/flare/metric_storage.rb

@@ -9,14 +9,17 @@ module Flare
   class MetricStorage
     def initialize
       @storage = Concurrent::Map.new
+      @pid = $$
     end
 
     def increment(key, duration_ms:, error: false)
+      detect_forking
       counter = @storage.compute_if_absent(key) { MetricCounter.new }
       counter.increment(duration_ms: duration_ms, error: error)
     end
 
     def add(key, count:, sum_ms:, error_count: 0)
+      detect_forking
       counter = @storage.compute_if_absent(key) { MetricCounter.new }
       counter.add(count: count, sum_ms: sum_ms, error_count: error_count)
     end
@@ -24,6 +27,7 @@ module Flare
     # Atomically retrieves and clears all metrics.
     # Returns a frozen hash of MetricKey => counter data.
     def drain
+      detect_forking
       result = {}
       @storage.keys.each do |key|
         counter = @storage.delete(key)
@@ -33,15 +37,31 @@ module Flare
     end
 
     def size
+      detect_forking
       @storage.size
     end
 
     def empty?
+      detect_forking
       @storage.empty?
     end
 
     def [](key)
+      detect_forking
       @storage[key]
     end
+
+    def after_fork
+      return if @pid == $$
+
+      @pid = $$
+      @storage = Concurrent::Map.new
+    end
+
+    private
+
+    def detect_forking
+      after_fork
+    end
   end
 end

data/lib/flare/metric_submitter.rb

@@ -7,6 +7,7 @@ require "stringio"
 require "securerandom"
 
 require_relative "client_headers"
+require_relative "deadline"
 
 module Flare
   # Submits metrics to the Flare metrics service via HTTP.
@@ -45,6 +46,8 @@ module Flare
       end
     end
 
+    class DeadlineExceeded < StandardError; end
+
     attr_reader :endpoint, :api_key, :backoff_policy
 
     def initialize(endpoint:, api_key:, project: nil, environment: nil, backoff_policy: nil, open_timeout: nil, read_timeout: nil, write_timeout: nil)
@@ -60,23 +63,28 @@ module Flare
 
     # Submit drained metrics to the server.
     # Returns [success_count, error] where error may be nil on success.
-    def submit(drained)
+    def submit(drained, timeout: nil)
       return [0, nil] if drained.empty?
 
+      deadline = Deadline.new(timeout)
+
       request_id = SecureRandom.uuid
-      Flare.log "Submitting #{drained.size} metrics to #{@endpoint} (request_id=#{request_id})"
+      Flare.log "Submitting #{drained.size} metrics to #{@endpoint} (request_id=#{request_id})" if Flare.respond_to?(:log)
 
       body = build_body(drained, request_id)
       return [0, nil] if body.nil?
+      return [0, DeadlineExceeded.new("metric submission deadline exceeded")] if deadline.expired?
 
       @backoff_policy.reset
-      response, error = retry_with_backoff(MAX_RETRIES) { post(body, request_id) }
+      response, error = retry_with_backoff(MAX_RETRIES, deadline: deadline) do
+        post(body, request_id, timeout: deadline.remaining)
+      end
 
       if error
-        Flare.log "Submission failed: #{error.message} (request_id=#{request_id})"
+        Flare.log "Submission failed: #{error.message} (request_id=#{request_id})" if Flare.respond_to?(:log)
         [0, error]
       else
-        Flare.log "Submission succeeded: #{response.code} (request_id=#{request_id})"
+        Flare.log "Submission succeeded: #{response.code} (request_id=#{request_id})" if Flare.respond_to?(:log)
         [drained.size, nil]
       end
     end
@@ -111,12 +119,14 @@ module Flare
       nil
     end
 
-    def post(body, request_id)
+    def post(body, request_id, timeout: nil)
+      raise DeadlineExceeded, "metric submission deadline exceeded" if timeout == 0
+
       http = Net::HTTP.new(@endpoint.host, @endpoint.port)
       http.use_ssl = @endpoint.scheme == "https"
-      http.open_timeout = @open_timeout
-      http.read_timeout = @read_timeout
-      http.write_timeout = @write_timeout if http.respond_to?(:write_timeout=)
+      http.open_timeout = effective_timeout(@open_timeout, timeout)
+      http.read_timeout = effective_timeout(@read_timeout, timeout)
+      http.write_timeout = effective_timeout(@write_timeout, timeout) if http.respond_to?(:write_timeout=)
 
       request_uri = @endpoint.request_uri
       request = Net::HTTP::Post.new(request_uri == "" ? "/" : request_uri)
@@ -158,20 +168,27 @@ module Flare
       )
     end
 
-    def retry_with_backoff(max_attempts)
+    def retry_with_backoff(max_attempts, deadline:)
       attempts_remaining = max_attempts
       last_error = nil
 
       while attempts_remaining > 0
         begin
+          return [nil, DeadlineExceeded.new("metric submission deadline exceeded")] if deadline.expired?
+
           result, should_retry = yield
+          return [nil, DeadlineExceeded.new("metric submission deadline exceeded")] if deadline.expired?
           return [result, nil] unless should_retry
-        rescue SubmissionError, Net::OpenTimeout, Net::ReadTimeout, Errno::ECONNREFUSED, Errno::ECONNRESET => e
+        rescue SubmissionError, Net::OpenTimeout, Net::ReadTimeout, Net::WriteTimeout, Errno::ECONNREFUSED, Errno::ECONNRESET => e
           last_error = e
           attempts_remaining -= 1
 
           if attempts_remaining > 0
             sleep_time = @backoff_policy.next_interval / 1000.0
+            remaining = deadline.remaining
+            if remaining && sleep_time >= remaining
+              return [nil, DeadlineExceeded.new("metric submission deadline exceeded")]
+            end
             sleep(sleep_time)
           end
           next
@@ -184,6 +201,12 @@ module Flare
       [nil, last_error]
     end
 
+    def effective_timeout(configured_timeout, remaining)
+      return configured_timeout unless remaining
+
+      [configured_timeout, remaining].min
+    end
+
     def gzip(string)
       io = StringIO.new
       io.set_encoding("BINARY")

data/lib/flare/recording_batch_span_processor.rb

@@ -0,0 +1,238 @@
+# frozen_string_literal: true
+
+require "logger"
+require "opentelemetry/sdk"
+
+require_relative "deadline"
+
+module Flare
+  # An asynchronous, bounded span processor that exports every ended recording
+  # span. OpenTelemetry's BatchSpanProcessor only accepts sampled spans, which
+  # excludes RECORD_ONLY spans needed by Flare's local development dashboard.
+  class RecordingBatchSpanProcessor
+    SUCCESS = OpenTelemetry::SDK::Trace::Export::SUCCESS
+    FAILURE = OpenTelemetry::SDK::Trace::Export::FAILURE
+    TIMEOUT = OpenTelemetry::SDK::Trace::Export::TIMEOUT
+
+    def initialize(exporter, exporter_timeout: 30_000, schedule_delay: 5_000,
+                   max_queue_size: 2_048, max_export_batch_size: 512, logger: nil)
+      raise ArgumentError if max_export_batch_size > max_queue_size
+
+      @exporter = exporter
+      @exporter_timeout = exporter_timeout / 1_000.0
+      @schedule_delay = schedule_delay / 1_000.0
+      @max_queue_size = max_queue_size
+      @max_export_batch_size = max_export_batch_size
+      @logger = logger || Logger.new($stderr, level: Logger::WARN)
+      @pid = $$
+      initialize_synchronization
+      start_worker
+    end
+
+    def on_start(_span, _parent_context); end
+
+    def on_finish(span)
+      detect_forking
+
+      @mutex.synchronize do
+        overflow = @queue.length + 1 - @max_queue_size
+        @queue.shift(overflow) if overflow.positive?
+        @queue << span
+        @condition.signal if @queue.length >= @max_export_batch_size
+      end
+    end
+
+    def force_flush(timeout: nil)
+      detect_forking
+      deadline = Deadline.new(timeout)
+      return TIMEOUT unless begin_flush(deadline)
+
+      snapshot = snapshot_for_flush
+      operation = start_flush_export(snapshot, deadline)
+      result = wait_for_flush_export(operation, deadline)
+      [@flush_prior_result, result].max
+    rescue StandardError => e
+      log_export_error(e)
+      FAILURE
+    ensure
+      finish_flush if @flush_owner == Thread.current
+    end
+
+    def shutdown(timeout: nil)
+      detect_forking
+      deadline = Deadline.new(timeout)
+
+      worker = @mutex.synchronize do
+        @stopped = true
+        @condition.broadcast
+        @worker
+      end
+      worker&.join(deadline.remaining)
+      return TIMEOUT if worker&.alive? || deadline.expired?
+
+      result = force_flush(timeout: deadline.remaining)
+      return result unless result == SUCCESS
+      return TIMEOUT if deadline.expired?
+
+      exporter_result = @exporter.shutdown(timeout: deadline.remaining)
+      deadline.expired? ? TIMEOUT : exporter_result
+    rescue StandardError => e
+      log_export_error(e)
+      FAILURE
+    end
+
+    private
+
+    def initialize_synchronization
+      @mutex = Mutex.new
+      @condition = ConditionVariable.new
+      @queue = []
+      @active_exports = 0
+      @export_completion_sequence = 0
+      @last_export_result = SUCCESS
+      @flush_prior_result = SUCCESS
+      @flush_owner = nil
+      @stopped = false
+      @worker = nil
+    end
+
+    def worker_loop
+      loop do
+        batch = @mutex.synchronize do
+          while !@stopped && (@queue.empty? || @flush_owner || @active_exports.positive?)
+            @condition.wait(@mutex, @schedule_delay)
+            break if !@queue.empty? && !@flush_owner && @active_exports.zero?
+          end
+          return if @stopped
+
+          @active_exports += 1
+          @queue.shift(@max_export_batch_size)
+        end
+
+        result = export_batch(batch, timeout: @exporter_timeout)
+      ensure
+        export_finished(result || FAILURE) if batch
+      end
+    end
+
+    def begin_flush(deadline)
+      @mutex.synchronize do
+        initial_sequence = @export_completion_sequence
+        while @flush_owner && @flush_owner != Thread.current
+          return false if deadline.expired?
+
+          @condition.wait(@mutex, deadline.remaining)
+        end
+        @flush_owner = Thread.current
+
+        while @active_exports.positive?
+          return false if deadline.expired?
+
+          @condition.wait(@mutex, deadline.remaining)
+        end
+        @flush_prior_result = if @export_completion_sequence > initial_sequence
+          @last_export_result
+        else
+          SUCCESS
+        end
+      end
+      true
+    end
+
+    def finish_flush
+      @mutex.synchronize do
+        @flush_owner = nil
+        @condition.broadcast
+      end
+    end
+
+    def snapshot_for_flush
+      @mutex.synchronize { @queue.shift(@queue.length) }
+    end
+
+    def export_snapshot(snapshot, deadline)
+      until snapshot.empty?
+        return TIMEOUT if deadline.expired?
+
+        batch = snapshot.shift(@max_export_batch_size)
+        result = export_batch(batch, timeout: deadline.remaining)
+        return result unless result == SUCCESS
+      end
+      SUCCESS
+    ensure
+      @mutex.synchronize { @queue.unshift(*snapshot) } if snapshot&.any?
+    end
+
+    def start_flush_export(snapshot, deadline)
+      operation = { done: false, result: nil }
+      @mutex.synchronize { @active_exports += 1 }
+      Thread.new do
+        result = export_snapshot(snapshot, deadline)
+        if result == SUCCESS && !deadline.expired?
+          result = @exporter.force_flush(timeout: deadline.remaining)
+        end
+        operation[:result] = deadline.expired? ? TIMEOUT : result
+      rescue StandardError => e
+        log_export_error(e)
+        operation[:result] = FAILURE
+      ensure
+        @mutex.synchronize do
+          operation[:done] = true
+          complete_export(operation[:result])
+        end
+      end
+      operation
+    end
+
+    def wait_for_flush_export(operation, deadline)
+      @mutex.synchronize do
+        until operation[:done]
+          return TIMEOUT if deadline.expired?
+
+          @condition.wait(@mutex, deadline.remaining)
+        end
+      end
+      operation[:result]
+    end
+
+    def export_batch(spans, timeout:)
+      span_data = spans.map { |span| span.respond_to?(:to_span_data) ? span.to_span_data : span }
+      @exporter.export(span_data, timeout: timeout)
+    rescue StandardError => e
+      log_export_error(e)
+      FAILURE
+    end
+
+    def export_finished(result)
+      @mutex.synchronize do
+        complete_export(result)
+      end
+    end
+
+    def complete_export(result)
+      @active_exports -= 1
+      @export_completion_sequence += 1
+      @last_export_result = result || FAILURE
+      @condition.broadcast
+    end
+
+    def detect_forking
+      return if @pid == $$
+
+      # Only the forking thread survives. Replacing synchronization objects
+      # avoids waiting on locks or in-flight state owned by vanished threads.
+      @pid = $$
+      initialize_synchronization
+      start_worker
+    end
+
+    def start_worker
+      @worker = Thread.new { worker_loop }
+      @worker.name = "flare-recording-batch-span-processor"
+    end
+
+    def log_export_error(error)
+      @logger.warn("[Flare::RecordingBatchSpanProcessor] export failed: #{error.class}: #{error.message}")
+    end
+  end
+end

data/lib/flare/rule_manager.rb

@@ -60,10 +60,10 @@ module Flare
       self
     end
 
-    def stop
+    def stop(timeout: 1)
       if @timer
         @timer.shutdown
-        @timer.wait_for_termination(1)
+        @timer.wait_for_termination(timeout)
         @timer.kill unless @timer.shutdown?
         @timer = nil
       end