flare 0.3.1 → 0.4.0

data/lib/flare/metric_storage.rb

@@ -9,14 +9,17 @@ module Flare
   class MetricStorage
     def initialize
       @storage = Concurrent::Map.new
+      @pid = $$
     end
 
     def increment(key, duration_ms:, error: false)
+      detect_forking
       counter = @storage.compute_if_absent(key) { MetricCounter.new }
       counter.increment(duration_ms: duration_ms, error: error)
     end
 
     def add(key, count:, sum_ms:, error_count: 0)
+      detect_forking
       counter = @storage.compute_if_absent(key) { MetricCounter.new }
       counter.add(count: count, sum_ms: sum_ms, error_count: error_count)
     end
@@ -24,6 +27,7 @@ module Flare
     # Atomically retrieves and clears all metrics.
     # Returns a frozen hash of MetricKey => counter data.
     def drain
+      detect_forking
       result = {}
       @storage.keys.each do |key|
         counter = @storage.delete(key)
@@ -33,15 +37,31 @@ module Flare
     end
 
     def size
+      detect_forking
       @storage.size
     end
 
     def empty?
+      detect_forking
       @storage.empty?
     end
 
     def [](key)
+      detect_forking
       @storage[key]
     end
+
+    def after_fork
+      return if @pid == $$
+
+      @pid = $$
+      @storage = Concurrent::Map.new
+    end
+
+    private
+
+    def detect_forking
+      after_fork
+    end
   end
 end

data/lib/flare/metric_submitter.rb

@@ -7,6 +7,7 @@ require "stringio"
 require "securerandom"
 
 require_relative "client_headers"
+require_relative "deadline"
 
 module Flare
   # Submits metrics to the Flare metrics service via HTTP.
@@ -45,6 +46,8 @@ module Flare
       end
     end
 
+    class DeadlineExceeded < StandardError; end
+
     attr_reader :endpoint, :api_key, :backoff_policy
 
     def initialize(endpoint:, api_key:, project: nil, environment: nil, backoff_policy: nil, open_timeout: nil, read_timeout: nil, write_timeout: nil)
@@ -60,23 +63,28 @@ module Flare
 
     # Submit drained metrics to the server.
     # Returns [success_count, error] where error may be nil on success.
-    def submit(drained)
+    def submit(drained, timeout: nil)
       return [0, nil] if drained.empty?
 
+      deadline = Deadline.new(timeout)
+
       request_id = SecureRandom.uuid
-      Flare.log "Submitting #{drained.size} metrics to #{@endpoint} (request_id=#{request_id})"
+      Flare.log "Submitting #{drained.size} metrics to #{@endpoint} (request_id=#{request_id})" if Flare.respond_to?(:log)
 
       body = build_body(drained, request_id)
       return [0, nil] if body.nil?
+      return [0, DeadlineExceeded.new("metric submission deadline exceeded")] if deadline.expired?
 
       @backoff_policy.reset
-      response, error = retry_with_backoff(MAX_RETRIES) { post(body, request_id) }
+      response, error = retry_with_backoff(MAX_RETRIES, deadline: deadline) do
+        post(body, request_id, timeout: deadline.remaining)
+      end
 
       if error
-        Flare.log "Submission failed: #{error.message} (request_id=#{request_id})"
+        Flare.log "Submission failed: #{error.message} (request_id=#{request_id})" if Flare.respond_to?(:log)
         [0, error]
       else
-        Flare.log "Submission succeeded: #{response.code} (request_id=#{request_id})"
+        Flare.log "Submission succeeded: #{response.code} (request_id=#{request_id})" if Flare.respond_to?(:log)
         [drained.size, nil]
       end
     end
@@ -111,12 +119,14 @@ module Flare
       nil
     end
 
-    def post(body, request_id)
+    def post(body, request_id, timeout: nil)
+      raise DeadlineExceeded, "metric submission deadline exceeded" if timeout == 0
+
       http = Net::HTTP.new(@endpoint.host, @endpoint.port)
       http.use_ssl = @endpoint.scheme == "https"
-      http.open_timeout = @open_timeout
-      http.read_timeout = @read_timeout
-      http.write_timeout = @write_timeout if http.respond_to?(:write_timeout=)
+      http.open_timeout = effective_timeout(@open_timeout, timeout)
+      http.read_timeout = effective_timeout(@read_timeout, timeout)
+      http.write_timeout = effective_timeout(@write_timeout, timeout) if http.respond_to?(:write_timeout=)
 
       request_uri = @endpoint.request_uri
       request = Net::HTTP::Post.new(request_uri == "" ? "/" : request_uri)
@@ -158,20 +168,27 @@ module Flare
       )
     end
 
-    def retry_with_backoff(max_attempts)
+    def retry_with_backoff(max_attempts, deadline:)
       attempts_remaining = max_attempts
       last_error = nil
 
       while attempts_remaining > 0
         begin
+          return [nil, DeadlineExceeded.new("metric submission deadline exceeded")] if deadline.expired?
+
           result, should_retry = yield
+          return [nil, DeadlineExceeded.new("metric submission deadline exceeded")] if deadline.expired?
           return [result, nil] unless should_retry
-        rescue SubmissionError, Net::OpenTimeout, Net::ReadTimeout, Errno::ECONNREFUSED, Errno::ECONNRESET => e
+        rescue SubmissionError, Net::OpenTimeout, Net::ReadTimeout, Net::WriteTimeout, Errno::ECONNREFUSED, Errno::ECONNRESET => e
           last_error = e
           attempts_remaining -= 1
 
           if attempts_remaining > 0
             sleep_time = @backoff_policy.next_interval / 1000.0
+            remaining = deadline.remaining
+            if remaining && sleep_time >= remaining
+              return [nil, DeadlineExceeded.new("metric submission deadline exceeded")]
+            end
             sleep(sleep_time)
           end
           next
@@ -184,6 +201,12 @@ module Flare
       [nil, last_error]
     end
 
+    def effective_timeout(configured_timeout, remaining)
+      return configured_timeout unless remaining
+
+      [configured_timeout, remaining].min
+    end
+
     def gzip(string)
       io = StringIO.new
       io.set_encoding("BINARY")

data/lib/flare/recording_batch_span_processor.rb

@@ -0,0 +1,238 @@
+# frozen_string_literal: true
+
+require "logger"
+require "opentelemetry/sdk"
+
+require_relative "deadline"
+
+module Flare
+  # An asynchronous, bounded span processor that exports every ended recording
+  # span. OpenTelemetry's BatchSpanProcessor only accepts sampled spans, which
+  # excludes RECORD_ONLY spans needed by Flare's local development dashboard.
+  class RecordingBatchSpanProcessor
+    SUCCESS = OpenTelemetry::SDK::Trace::Export::SUCCESS
+    FAILURE = OpenTelemetry::SDK::Trace::Export::FAILURE
+    TIMEOUT = OpenTelemetry::SDK::Trace::Export::TIMEOUT
+
+    def initialize(exporter, exporter_timeout: 30_000, schedule_delay: 5_000,
+                   max_queue_size: 2_048, max_export_batch_size: 512, logger: nil)
+      raise ArgumentError if max_export_batch_size > max_queue_size
+
+      @exporter = exporter
+      @exporter_timeout = exporter_timeout / 1_000.0
+      @schedule_delay = schedule_delay / 1_000.0
+      @max_queue_size = max_queue_size
+      @max_export_batch_size = max_export_batch_size
+      @logger = logger || Logger.new($stderr, level: Logger::WARN)
+      @pid = $$
+      initialize_synchronization
+      start_worker
+    end
+
+    def on_start(_span, _parent_context); end
+
+    def on_finish(span)
+      detect_forking
+
+      @mutex.synchronize do
+        overflow = @queue.length + 1 - @max_queue_size
+        @queue.shift(overflow) if overflow.positive?
+        @queue << span
+        @condition.signal if @queue.length >= @max_export_batch_size
+      end
+    end
+
+    def force_flush(timeout: nil)
+      detect_forking
+      deadline = Deadline.new(timeout)
+      return TIMEOUT unless begin_flush(deadline)
+
+      snapshot = snapshot_for_flush
+      operation = start_flush_export(snapshot, deadline)
+      result = wait_for_flush_export(operation, deadline)
+      [@flush_prior_result, result].max
+    rescue StandardError => e
+      log_export_error(e)
+      FAILURE
+    ensure
+      finish_flush if @flush_owner == Thread.current
+    end
+
+    def shutdown(timeout: nil)
+      detect_forking
+      deadline = Deadline.new(timeout)
+
+      worker = @mutex.synchronize do
+        @stopped = true
+        @condition.broadcast
+        @worker
+      end
+      worker&.join(deadline.remaining)
+      return TIMEOUT if worker&.alive? || deadline.expired?
+
+      result = force_flush(timeout: deadline.remaining)
+      return result unless result == SUCCESS
+      return TIMEOUT if deadline.expired?
+
+      exporter_result = @exporter.shutdown(timeout: deadline.remaining)
+      deadline.expired? ? TIMEOUT : exporter_result
+    rescue StandardError => e
+      log_export_error(e)
+      FAILURE
+    end
+
+    private
+
+    def initialize_synchronization
+      @mutex = Mutex.new
+      @condition = ConditionVariable.new
+      @queue = []
+      @active_exports = 0
+      @export_completion_sequence = 0
+      @last_export_result = SUCCESS
+      @flush_prior_result = SUCCESS
+      @flush_owner = nil
+      @stopped = false
+      @worker = nil
+    end
+
+    def worker_loop
+      loop do
+        batch = @mutex.synchronize do
+          while !@stopped && (@queue.empty? || @flush_owner || @active_exports.positive?)
+            @condition.wait(@mutex, @schedule_delay)
+            break if !@queue.empty? && !@flush_owner && @active_exports.zero?
+          end
+          return if @stopped
+
+          @active_exports += 1
+          @queue.shift(@max_export_batch_size)
+        end
+
+        result = export_batch(batch, timeout: @exporter_timeout)
+      ensure
+        export_finished(result || FAILURE) if batch
+      end
+    end
+
+    def begin_flush(deadline)
+      @mutex.synchronize do
+        initial_sequence = @export_completion_sequence
+        while @flush_owner && @flush_owner != Thread.current
+          return false if deadline.expired?
+
+          @condition.wait(@mutex, deadline.remaining)
+        end
+        @flush_owner = Thread.current
+
+        while @active_exports.positive?
+          return false if deadline.expired?
+
+          @condition.wait(@mutex, deadline.remaining)
+        end
+        @flush_prior_result = if @export_completion_sequence > initial_sequence
+          @last_export_result
+        else
+          SUCCESS
+        end
+      end
+      true
+    end
+
+    def finish_flush
+      @mutex.synchronize do
+        @flush_owner = nil
+        @condition.broadcast
+      end
+    end
+
+    def snapshot_for_flush
+      @mutex.synchronize { @queue.shift(@queue.length) }
+    end
+
+    def export_snapshot(snapshot, deadline)
+      until snapshot.empty?
+        return TIMEOUT if deadline.expired?
+
+        batch = snapshot.shift(@max_export_batch_size)
+        result = export_batch(batch, timeout: deadline.remaining)
+        return result unless result == SUCCESS
+      end
+      SUCCESS
+    ensure
+      @mutex.synchronize { @queue.unshift(*snapshot) } if snapshot&.any?
+    end
+
+    def start_flush_export(snapshot, deadline)
+      operation = { done: false, result: nil }
+      @mutex.synchronize { @active_exports += 1 }
+      Thread.new do
+        result = export_snapshot(snapshot, deadline)
+        if result == SUCCESS && !deadline.expired?
+          result = @exporter.force_flush(timeout: deadline.remaining)
+        end
+        operation[:result] = deadline.expired? ? TIMEOUT : result
+      rescue StandardError => e
+        log_export_error(e)
+        operation[:result] = FAILURE
+      ensure
+        @mutex.synchronize do
+          operation[:done] = true
+          complete_export(operation[:result])
+        end
+      end
+      operation
+    end
+
+    def wait_for_flush_export(operation, deadline)
+      @mutex.synchronize do
+        until operation[:done]
+          return TIMEOUT if deadline.expired?
+
+          @condition.wait(@mutex, deadline.remaining)
+        end
+      end
+      operation[:result]
+    end
+
+    def export_batch(spans, timeout:)
+      span_data = spans.map { |span| span.respond_to?(:to_span_data) ? span.to_span_data : span }
+      @exporter.export(span_data, timeout: timeout)
+    rescue StandardError => e
+      log_export_error(e)
+      FAILURE
+    end
+
+    def export_finished(result)
+      @mutex.synchronize do
+        complete_export(result)
+      end
+    end
+
+    def complete_export(result)
+      @active_exports -= 1
+      @export_completion_sequence += 1
+      @last_export_result = result || FAILURE
+      @condition.broadcast
+    end
+
+    def detect_forking
+      return if @pid == $$
+
+      # Only the forking thread survives. Replacing synchronization objects
+      # avoids waiting on locks or in-flight state owned by vanished threads.
+      @pid = $$
+      initialize_synchronization
+      start_worker
+    end
+
+    def start_worker
+      @worker = Thread.new { worker_loop }
+      @worker.name = "flare-recording-batch-span-processor"
+    end
+
+    def log_export_error(error)
+      @logger.warn("[Flare::RecordingBatchSpanProcessor] export failed: #{error.class}: #{error.message}")
+    end
+  end
+end

data/lib/flare/sqlite_exporter.rb

@@ -3,6 +3,8 @@
 require "sqlite3"
 require "json"
 
+require_relative "deadline"
+
 module Flare
   class SQLiteExporter
     SUCCESS = OpenTelemetry::SDK::Trace::Export::SUCCESS
@@ -16,45 +18,61 @@ module Flare
       @database_path = database_path
       @mutex = Mutex.new
       @setup = false
+      @pid = $$
     end
 
     # Maximum number of retry attempts when the database is busy.
     # Mirrors ActiveRecord's retry strategy for SQLite.
     MAX_RETRIES = 3
+    ExportDeadlineExceeded = Class.new(StandardError)
 
     def export(span_datas, timeout: nil)
-      setup_database unless @setup
+      detect_forking
+      deadline = Deadline.new(timeout)
 
       retries = 0
       exported = 0
 
       begin
-        @mutex.synchronize do
+        setup_database(deadline) unless @setup
+        raise ExportDeadlineExceeded if deadline.expired?
+        raise ExportDeadlineExceeded unless lock_before_deadline(deadline)
+
+        begin
+          apply_busy_timeout(deadline)
           connection.transaction do
             span_datas.each do |span_data|
+              raise ExportDeadlineExceeded if deadline.expired?
               next if should_ignore_span?(span_data)
 
               create_span(span_data)
               exported += 1
             end
           end
+        ensure
+          @mutex.unlock
         end
+      rescue ExportDeadlineExceeded
+        return TIMEOUT
       rescue ::SQLite3::BusyException
         retries += 1
         if retries <= MAX_RETRIES
-          sleep 0.1 * retries
+          sleep_time = 0.1 * retries
+          return TIMEOUT if deadline.remaining && sleep_time >= deadline.remaining
+
+          sleep(sleep_time)
           retry
         end
         warn "[Flare] SQLite export error: database is busy after #{MAX_RETRIES} retries"
         return FAILURE
       end
 
-      Flare.log "Exported #{exported} spans to SQLite" if exported > 0
+      Flare.log "Exported #{exported} spans to SQLite" if exported > 0 && Flare.respond_to?(:log)
 
       # Periodically prune old data
-      maybe_prune
+      maybe_prune unless deadline.expired?
 
-      SUCCESS
+      deadline.expired? ? TIMEOUT : SUCCESS
     rescue => e
       warn "[Flare] SQLite export error: #{e.message}"
       FAILURE
@@ -70,6 +88,31 @@ module Flare
 
     private
 
+    def detect_forking
+      return if @pid == $$
+
+      @pid = $$
+      @mutex = Mutex.new
+      close_connection
+    end
+
+    def lock_before_deadline(deadline)
+      return @mutex.lock unless deadline.remaining
+
+      until @mutex.try_lock
+        return false if deadline.expired?
+
+        sleep([deadline.remaining, 0.001].min)
+      end
+      true
+    end
+
+    def apply_busy_timeout(deadline)
+      remaining = deadline.remaining
+      timeout_ms = remaining ? [(remaining * 1_000).floor, 1].max : 5_000
+      connection.busy_timeout = [timeout_ms, 5_000].min
+    end
+
     def maybe_prune
       return unless rand < PRUNE_PROBABILITY
 
@@ -162,14 +205,16 @@ module Flare
       end
     end
 
-    def setup_database
-      @mutex.synchronize do
+    def setup_database(deadline)
+      raise ExportDeadlineExceeded unless lock_before_deadline(deadline)
+
+      begin
         return if @setup
 
         db = connection
-        configure_pragmas(db)
+        configure_pragmas(db, deadline)
 
-        db.execute(<<~SQL)
+        execute_setup(db, deadline, <<~SQL)
           CREATE TABLE IF NOT EXISTS flare_spans (
             id INTEGER PRIMARY KEY AUTOINCREMENT,
             name TEXT NOT NULL,
@@ -187,23 +232,23 @@ module Flare
           )
         SQL
 
-        db.execute(<<~SQL)
+        execute_setup(db, deadline, <<~SQL)
           CREATE INDEX IF NOT EXISTS idx_spans_span_id ON flare_spans(span_id)
         SQL
 
-        db.execute(<<~SQL)
+        execute_setup(db, deadline, <<~SQL)
           CREATE INDEX IF NOT EXISTS idx_spans_trace_id ON flare_spans(trace_id)
         SQL
 
-        db.execute(<<~SQL)
+        execute_setup(db, deadline, <<~SQL)
           CREATE INDEX IF NOT EXISTS idx_spans_parent_span_id ON flare_spans(parent_span_id)
         SQL
 
-        db.execute(<<~SQL)
+        execute_setup(db, deadline, <<~SQL)
           CREATE INDEX IF NOT EXISTS idx_spans_created_at ON flare_spans(created_at)
         SQL
 
-        db.execute(<<~SQL)
+        execute_setup(db, deadline, <<~SQL)
           CREATE TABLE IF NOT EXISTS flare_events (
             id INTEGER PRIMARY KEY AUTOINCREMENT,
             span_id INTEGER NOT NULL,
@@ -214,11 +259,11 @@ module Flare
           )
         SQL
 
-        db.execute(<<~SQL)
+        execute_setup(db, deadline, <<~SQL)
           CREATE INDEX IF NOT EXISTS idx_events_span_id ON flare_events(span_id)
         SQL
 
-        db.execute(<<~SQL)
+        execute_setup(db, deadline, <<~SQL)
           CREATE TABLE IF NOT EXISTS flare_properties (
             id INTEGER PRIMARY KEY AUTOINCREMENT,
             key TEXT NOT NULL,
@@ -231,27 +276,37 @@ module Flare
           )
         SQL
 
-        db.execute(<<~SQL)
+        execute_setup(db, deadline, <<~SQL)
           CREATE INDEX IF NOT EXISTS idx_properties_owner ON flare_properties(owner_type, owner_id)
         SQL
 
-        db.execute(<<~SQL)
+        execute_setup(db, deadline, <<~SQL)
           CREATE INDEX IF NOT EXISTS idx_properties_key ON flare_properties(key)
         SQL
 
         close_connection # avoid inheriting connection across fork
         @setup = true
+      ensure
+        @mutex.unlock
       end
     end
 
     # Applies the same SQLite pragmas that ActiveRecord uses for good
     # concurrency and performance with threaded/multi-process access.
-    def configure_pragmas(db)
-      db.execute("PRAGMA journal_mode=WAL")
-      db.execute("PRAGMA synchronous=NORMAL")
-      db.execute("PRAGMA mmap_size=134217728")        # 128MB
-      db.execute("PRAGMA journal_size_limit=67108864") # 64MB
-      db.execute("PRAGMA cache_size=2000")
+    def configure_pragmas(db, deadline)
+      execute_setup(db, deadline, "PRAGMA journal_mode=WAL")
+      execute_setup(db, deadline, "PRAGMA synchronous=NORMAL")
+      execute_setup(db, deadline, "PRAGMA mmap_size=134217728")         # 128MB
+      execute_setup(db, deadline, "PRAGMA journal_size_limit=67108864") # 64MB
+      execute_setup(db, deadline, "PRAGMA cache_size=2000")
+    end
+
+    def execute_setup(db, deadline, statement)
+      raise ExportDeadlineExceeded if deadline.expired?
+
+      apply_busy_timeout(deadline)
+      db.execute(statement)
+      raise ExportDeadlineExceeded if deadline.expired?
     end
 
     def connection_key