flare 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c659b175126da90baf0738ea221b54c25d9939ffbca3883b7756b2972574d0b6
-  data.tar.gz: 1b5d1f9f146a91ac1e7fc7a605fa2a3e3f22d28b906b92939362f99d9c63361d
+  metadata.gz: dcfa9283973a11f13fd17397781c76cc03d3e39c9bd08a92025417397eb74eb9
+  data.tar.gz: 30e2f84a92f9689aabdd64b7c6839598dc0c3c2bb1dd32a31e2ab9b3a6f5f85a
 SHA512:
-  metadata.gz: 2662255ed5b8c4b77315e348c6c334f02df3a0879fdba62b64bd9d20ead9c78add8d88847dd945c9a276138ebdaa222352e64b2f2d1b054acb91bb23e3c67da4
-  data.tar.gz: 6374272263432f9f3a75d7d291fb7099bc7bcd8c8bfe87b8d3eb1b6c1a7ff79d39d226e49d20f88018f17f0f1411df6849c7ae65faa73f15350612fe1a03880c
+  metadata.gz: 581e6e9ed512b82ac086ab76644ff0d618408649a110ef3b623a3bf4a1648e3695dfcedcefad01bd54b7a11687a1c5ed154ed01e74f2cf14c27fa400bf97f73a
+  data.tar.gz: 4a407b09a229f512d6cb1f35ab04dfb8de7ecaa982837fd356c41435c7758d3c4e05ad33812670b2e570830bef5a21e9bdffe297234cec9e33f9ab5250c42af1

data/CHANGELOG.md

@@ -1,18 +1,3 @@
-## [Unreleased]
+# Changelog
 
-## [0.2.0] - 2026-04-23
-
-- Auto-detect OTel instrumentation gems via `use_all`
-- Name Sidekiq job spans by worker class (via `job_class` attribute)
-- Support both old and new OTel semantic convention property keys in dashboard queries
-- Guard dashboard routes when sqlite3 is missing
-
-## [0.1.1] - 2025-12-17
-
-- Rename gem from caboose to flare
-- Warn instead of raising when sqlite3 gem is missing
-- Make sqlite3 an optional dependency for production compatibility
-
-## [0.1.0] - 2025-12-17
-
-- Initial release
+See [GitHub Releases](https://github.com/verygoodsoftwarellc/flare-gem/releases) for the changelog.

data/lib/flare/configuration.rb

@@ -12,9 +12,15 @@ module Flare
 
     # Spans: detailed trace data stored in SQLite (default: development only)
     # Metrics: aggregated counters in memory, flushed periodically (default: production only)
+    # Tracing: server-controlled per-route trace sampling. Polls /api/rules
+    # for which routes/jobs to capture, ships matched traces direct to R2
+    # via presigned URLs, self-notifies POST /api/traces.
     attr_accessor :spans_enabled
     attr_accessor :metrics_enabled
     attr_accessor :metrics_flush_interval # seconds between flushes (default: 60)
+    attr_accessor :tracing_enabled
+    attr_accessor :tracing_poll_interval  # seconds between /api/rules polls (default: 30)
+    attr_accessor :tracing_max_queue      # max traced spans buffered per process
 
     # Metrics HTTP submission settings
     attr_accessor :url        # URL of the Flare metrics service
@@ -53,6 +59,9 @@ module Flare
       @spans_enabled = rails_development?
       @metrics_enabled = !rails_test?
       @metrics_flush_interval = 60 # seconds
+      @tracing_enabled = !rails_test?
+      @tracing_poll_interval = 30  # seconds
+      @tracing_max_queue = 5_000
 
       # Metrics HTTP submission defaults
       @url = ENV.fetch("FLARE_URL", credentials_url || "https://flare.am")
@@ -67,6 +76,11 @@ module Flare
         !@key.nil? && !@key.empty?
     end
 
+    # Tracing reuses the same endpoint + key as metrics.
+    def tracing_submission_configured?
+      @tracing_enabled && metrics_submission_configured?
+    end
+
     def database_path
       @database_path || default_database_path
     end

data/lib/flare/engine.rb

@@ -25,9 +25,11 @@ module Flare
       Flare.configure_opentelemetry
     end
 
-    # Phase 2: Start the metrics flusher after all initializers have run
-    # so user config (metrics_enabled, flush_interval, etc.) is applied.
+    # Phase 2: Start background threads after all initializers have run
+    # so user config (metrics_enabled, tracing_enabled, flush_interval,
+    # tracing_poll_interval, etc.) is applied.
     config.after_initialize do
+      Flare.start_rule_manager
       Flare.start_metrics_flusher
     end
 

data/lib/flare/filtering_span_processor.rb

@@ -0,0 +1,279 @@
+# frozen_string_literal: true
+
+require "concurrent/atomic/atomic_fixnum"
+require "logger"
+require "opentelemetry/sdk"
+
+module Flare
+  # BSP-shaped span processor whose filter is `sampled OR marked` instead
+  # of BSP's `sampled` (BSP early-returns on RECORD_ONLY spans -- our
+  # Path 2 spans have sampled=false so they'd be dropped). Forwards
+  # matching spans to a trace exporter on a background worker thread so
+  # the exporter never runs on the request/job thread (CAF-3).
+  #
+  # On every on_finish we also check marker.owner?(trace_id, span_id) and
+  # unmark when the owning rack span finishes (CAF-2). Cleanup runs even
+  # for spans we don't export.
+  class FilteringSpanProcessor
+    SUCCESS = OpenTelemetry::SDK::Trace::Export::SUCCESS
+    FAILURE = OpenTelemetry::SDK::Trace::Export::FAILURE
+
+    DEFAULT_MAX_QUEUE = 5_000
+    DEFAULT_FLUSH_INTERVAL = 5      # seconds
+    DEFAULT_EXPORT_TIMEOUT = 30     # seconds
+    DEFAULT_MARKED_TRACE_GRACE_PERIOD = 1.0 # seconds
+
+    attr_reader :dropped_count, :failed_export_count, :exception_count, :buffer_high_watermark, :max_queue
+
+    def initialize(exporter:, marker:,
+                   max_queue: DEFAULT_MAX_QUEUE,
+                   flush_interval: DEFAULT_FLUSH_INTERVAL,
+                   export_timeout: DEFAULT_EXPORT_TIMEOUT,
+                   marked_trace_grace_period: DEFAULT_MARKED_TRACE_GRACE_PERIOD,
+                   logger: nil)
+      @exporter       = exporter
+      @marker         = marker
+      @max_queue      = max_queue
+      @flush_interval = flush_interval
+      @export_timeout = export_timeout
+      @marked_trace_grace_period = marked_trace_grace_period.to_f
+      @logger         = logger || Logger.new($stderr, level: Logger::WARN)
+
+      @pending_by_trace = {}
+      @trace_order      = []
+      @pending_count    = 0
+      @ready_queue      = []
+      @delayed_ready_by_trace = {}
+      @mutex            = Mutex.new
+      @cond             = ConditionVariable.new
+      @stopped          = false
+      @pid              = $$
+
+      @dropped_count       = Concurrent::AtomicFixnum.new(0)
+      @failed_export_count = Concurrent::AtomicFixnum.new(0)
+      @exception_count     = Concurrent::AtomicFixnum.new(0)
+      @buffer_high_watermark = Concurrent::AtomicFixnum.new(0)
+
+      start_worker
+    end
+
+    def on_start(_span, _parent_context); end
+
+    def on_finish(span)
+      detect_forking
+
+      ctx = span.context
+      sampled = ctx&.trace_flags&.sampled?
+      marked  = ctx && @marker.marked?(ctx.trace_id)
+      owner_finished = marked && @marker.owner?(ctx.trace_id, ctx.span_id)
+
+      return unless sampled || marked
+
+      span_data = span.respond_to?(:to_span_data) ? span.to_span_data : span
+      enqueue(
+        span_data,
+        complete: owner_finished || sampled_completion_span?(span_data),
+        delay: owner_finished ? @marked_trace_grace_period : 0
+      )
+    end
+
+    def force_flush(timeout: nil)
+      drain_and_export(include_pending: true)
+      SUCCESS
+    end
+
+    def shutdown(timeout: nil)
+      @mutex.synchronize do
+        @stopped = true
+        @cond.broadcast
+      end
+      @worker.join(timeout || 5)
+      drain_and_export(include_pending: true)
+      @exporter.shutdown(timeout: timeout) if @exporter.respond_to?(:shutdown)
+      SUCCESS
+    end
+
+    def buffer_size
+      @mutex.synchronize { queued_span_count }
+    end
+
+    def reset_buffer_high_watermark
+      @buffer_high_watermark.value = buffer_size
+    end
+
+    private
+
+    def enqueue(span_data, complete:, delay: 0)
+      @mutex.synchronize do
+        trace_id = span_data.trace_id
+        @trace_order << trace_id unless @pending_by_trace.key?(trace_id)
+        @pending_by_trace[trace_id] ||= []
+        @pending_by_trace[trace_id] << span_data
+        @pending_count += 1
+        evict_oldest_spans
+
+        if complete
+          delay.positive? ? delay_trace_ready(trace_id, delay) : mark_trace_ready(trace_id)
+        end
+        evict_oldest_spans
+        update_buffer_high_watermark
+      end
+    end
+
+    def worker_loop
+      until stopped?
+        @mutex.synchronize do
+          timeout = next_wait_timeout
+          @cond.wait(@mutex, timeout) if @ready_queue.empty? && !@stopped
+        end
+        drain_and_export
+      end
+    end
+
+    def stopped?
+      @mutex.synchronize { @stopped }
+    end
+
+    def drain_and_export(include_pending: false)
+      batch = nil
+      @mutex.synchronize do
+        promote_due_delayed_traces
+
+        if include_pending
+          @ready_queue.concat(@pending_by_trace.values.flatten)
+          @pending_by_trace.clear
+          @trace_order.clear
+          @pending_count = 0
+          unmark_delayed_traces
+          @delayed_ready_by_trace.clear
+        end
+
+        return if @ready_queue.empty?
+        batch = @ready_queue
+        @ready_queue = []
+      end
+
+      result = @exporter.export(batch, timeout: @export_timeout)
+      @failed_export_count.increment if result != SUCCESS
+    rescue StandardError => e
+      @exception_count.increment
+      @logger.warn("[Flare::FilteringSpanProcessor] export failed: #{e.class}: #{e.message}")
+    end
+
+    def mark_trace_ready(trace_id)
+      batch = @pending_by_trace.delete(trace_id)
+      return unless batch
+
+      @trace_order.delete(trace_id)
+      @delayed_ready_by_trace.delete(trace_id)
+      @pending_count -= batch.length
+      @ready_queue.concat(batch)
+      @cond.signal
+    end
+
+    def delay_trace_ready(trace_id, delay)
+      @delayed_ready_by_trace[trace_id] = monotonic_now + delay
+      @cond.signal
+    end
+
+    def promote_due_delayed_traces
+      now = monotonic_now
+      ready_trace_ids = @delayed_ready_by_trace.select { |_, ready_at| ready_at <= now }.keys
+      ready_trace_ids.each do |trace_id|
+        mark_trace_ready(trace_id)
+        @marker.unmark(trace_id)
+      end
+    end
+
+    def unmark_delayed_traces
+      @delayed_ready_by_trace.each_key { |trace_id| @marker.unmark(trace_id) }
+    end
+
+    def next_wait_timeout
+      next_ready_at = @delayed_ready_by_trace.values.min
+      return @flush_interval unless next_ready_at
+
+      [next_ready_at - monotonic_now, 0].max
+    end
+
+    def evict_oldest_spans
+      while queued_span_count > @max_queue
+        trace_id = @trace_order.first
+        unless trace_id
+          @ready_queue.shift
+          @dropped_count.increment
+          next
+        end
+
+        spans = @pending_by_trace[trace_id]
+        if spans.nil? || spans.empty?
+          @trace_order.shift
+          next
+        end
+
+        spans.shift
+        @pending_count -= 1
+        @dropped_count.increment
+
+        if spans.empty?
+          @pending_by_trace.delete(trace_id)
+          @trace_order.shift
+        end
+      end
+    end
+
+    def queued_span_count
+      @pending_count + @ready_queue.length
+    end
+
+    def update_buffer_high_watermark
+      current = queued_span_count
+      @buffer_high_watermark.update { |previous| current > previous ? current : previous }
+    end
+
+    def monotonic_now
+      Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    end
+
+    def sampled_completion_span?(span_data)
+      root_span?(span_data) || entry_span?(span_data)
+    end
+
+    def root_span?(span_data)
+      parent_id = span_data.parent_span_id if span_data.respond_to?(:parent_span_id)
+      parent_id.nil? ||
+        (parent_id.respond_to?(:empty?) && parent_id.empty?) ||
+        parent_id == OpenTelemetry::Trace::INVALID_SPAN_ID
+    end
+
+    def entry_span?(span_data)
+      return false unless span_data.respond_to?(:kind)
+
+      span_data.kind == :server || span_data.kind == :consumer
+    end
+
+    def detect_forking
+      return if @pid == $$
+
+      @mutex.synchronize do
+        return if @pid == $$
+
+        @pid = $$
+        @pending_by_trace.clear
+        @trace_order.clear
+        @ready_queue.clear
+        @delayed_ready_by_trace.clear
+        @pending_count = 0
+        @stopped = false
+        start_worker
+      end
+    end
+
+    def start_worker
+      return if @worker&.alive?
+
+      @worker = Thread.new { worker_loop }
+      @worker.name = "flare-filtering-span-processor"
+    end
+  end
+end

data/lib/flare/http_transport.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "uri"
+
+module Flare
+  # Tiny HTTP wrapper used by TraceExporter (and anything else that wants
+  # to PUT/POST without pulling in a heavy client). Designed for injection
+  # at the boundary so tests can swap in a recording fake; no other moving
+  # parts.
+  class HttpTransport
+    DEFAULT_OPEN_TIMEOUT  = 2
+    DEFAULT_READ_TIMEOUT  = 5
+    DEFAULT_WRITE_TIMEOUT = 5
+
+    Response = Struct.new(:code, :body, :headers, keyword_init: true) do
+      def header(name)
+        return nil unless headers
+        headers[name] || headers[name.downcase] || headers[name.upcase]
+      end
+    end
+
+    def initialize(open_timeout: DEFAULT_OPEN_TIMEOUT,
+                   read_timeout: DEFAULT_READ_TIMEOUT,
+                   write_timeout: DEFAULT_WRITE_TIMEOUT)
+      @open_timeout  = open_timeout
+      @read_timeout  = read_timeout
+      @write_timeout = write_timeout
+    end
+
+    def get(url, headers = {})
+      request(url, nil, headers, Net::HTTP::Get)
+    end
+
+    def put(url, body, headers = {})
+      request(url, body, headers, Net::HTTP::Put)
+    end
+
+    def post(url, body, headers = {})
+      request(url, body, headers, Net::HTTP::Post)
+    end
+
+    private
+
+    def request(url, body, headers, klass)
+      uri = URI(url)
+      http = Net::HTTP.new(uri.host, uri.port)
+      http.use_ssl     = uri.scheme == "https"
+      http.open_timeout = @open_timeout
+      http.read_timeout = @read_timeout
+      http.write_timeout = @write_timeout if http.respond_to?(:write_timeout=)
+
+      req = klass.new(uri.request_uri == "" ? "/" : uri.request_uri)
+      headers.each { |k, v| req[k] = v }
+      req.body = body if body
+
+      response = http.request(req)
+      hash = response.each_header.to_h
+      Response.new(code: response.code.to_s, body: response.body, headers: hash)
+    end
+  end
+end

data/lib/flare/marker.rb

@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+
+require "concurrent/map"
+require "concurrent/atomic/atomic_fixnum"
+
+module Flare
+  # Thread-safe registry of trace_ids that Path 2 (the WebMarkerSubscriber)
+  # has marked for export. FilteringSpanProcessor checks marked? on every
+  # on_finish; matching spans get forwarded to the trace exporter, the rest
+  # are dropped.
+  #
+  # Each entry records the OWNER span_id (the local rack server span the
+  # subscriber was inside when it marked the trace). Cleanup is keyed on
+  # the owner finishing, not the trace root finishing -- remote-parented
+  # rack spans aren't trace roots, and child spans can outlive their parent
+  # in OTel, so root-driven cleanup would leak on the dominant production
+  # case (web app behind a load balancer or service mesh).
+  #
+  # Bounded by:
+  #   - sweep(): drops entries older than max_age (default 5 min) so a rack
+  #     span that never finishes (process killed mid-request, exception path
+  #     that skips ensure) doesn't leak forever.
+  #   - hard ceiling at max_entries (default 10k): on overflow, drop oldest
+  #     10% by marked_at.
+  class Marker
+    Entry = Struct.new(:owner_span_id, :rule_id, :marked_at, keyword_init: true)
+
+    DEFAULT_MAX_ENTRIES = 10_000
+    DEFAULT_MAX_AGE = 5 * 60 # seconds
+
+    attr_reader :evicted_count
+
+    def initialize(max_entries: DEFAULT_MAX_ENTRIES, max_age: DEFAULT_MAX_AGE)
+      @entries = Concurrent::Map.new
+      @max_entries = max_entries
+      @max_age = max_age
+      @evicted_count = Concurrent::AtomicFixnum.new(0)
+    end
+
+    def mark(trace_id, owner_span_id:, rule_id:)
+      @entries[trace_id] = Entry.new(
+        owner_span_id: owner_span_id,
+        rule_id: rule_id,
+        marked_at: monotonic_now
+      )
+      maybe_evict_oldest
+    end
+
+    def marked?(trace_id)
+      @entries.key?(trace_id)
+    end
+
+    # True only when span_id matches the marker's owner -- the rack span
+    # that originally marked this trace. Used by FilteringSpanProcessor to
+    # decide when to unmark (only when that exact span finishes, not on
+    # every span that happens to have this trace_id).
+    def owner?(trace_id, span_id)
+      entry = @entries[trace_id]
+      !entry.nil? && entry.owner_span_id == span_id
+    end
+
+    def rule_id(trace_id)
+      entry = @entries[trace_id]
+      entry&.rule_id
+    end
+
+    def unmark(trace_id)
+      @entries.delete(trace_id)
+    end
+
+    def size
+      @entries.size
+    end
+
+    # Drop entries older than max_age. Call periodically (the RuleManager's
+    # scheduler is the natural place) to handle the rack-span-never-finishes
+    # leak case (CAF-7).
+    def sweep
+      threshold = monotonic_now - @max_age
+      evicted = 0
+      @entries.each_pair do |trace_id, entry|
+        if entry.marked_at < threshold
+          @entries.delete(trace_id)
+          evicted += 1
+        end
+      end
+      @evicted_count.increment(evicted) if evicted.positive?
+      evicted
+    end
+
+    private
+
+    def maybe_evict_oldest
+      return if @entries.size <= @max_entries
+
+      to_drop = (@max_entries * 0.1).ceil
+      sorted = @entries.each_pair.to_a.sort_by { |_, entry| entry.marked_at }
+      sorted.first(to_drop).each { |trace_id, _| @entries.delete(trace_id) }
+      @evicted_count.increment(to_drop)
+    end
+
+    def monotonic_now
+      Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    end
+  end
+end

data/lib/flare/metric_counter.rb

@@ -22,6 +22,12 @@ module Flare
       @error_count.increment if error
     end
 
+    def add(count:, sum_ms:, error_count: 0)
+      @count.increment(count.to_i)
+      @sum_ms.increment(sum_ms.to_i)
+      @error_count.increment(error_count.to_i)
+    end
+
     def count
       @count.value
     end

data/lib/flare/metric_flusher.rb

@@ -15,11 +15,12 @@ module Flare
 
     attr_reader :interval, :shutdown_timeout
 
-    def initialize(storage:, submitter:, interval: DEFAULT_INTERVAL, shutdown_timeout: DEFAULT_SHUTDOWN_TIMEOUT)
+    def initialize(storage:, submitter:, interval: DEFAULT_INTERVAL, shutdown_timeout: DEFAULT_SHUTDOWN_TIMEOUT, health_reporters: [])
       @storage = storage
       @submitter = submitter
       @interval = interval
       @shutdown_timeout = shutdown_timeout
+      @health_reporters = Array(health_reporters)
       @pid = $$
       @stopped = false
     end
@@ -44,7 +45,7 @@ module Flare
 
       @stopped = true
 
-      Flare.log "Shutting down metrics flusher, draining remaining metrics..."
+      log "Shutting down metrics flusher, draining remaining metrics..."
 
       if @timer
         @timer.shutdown
@@ -59,7 +60,7 @@ module Flare
         @pool.kill unless pool_terminated
       end
 
-      Flare.log "Metrics flusher stopped"
+      log "Metrics flusher stopped"
     end
 
     def restart
@@ -72,6 +73,7 @@ module Flare
     def flush_now
       return 0 unless @storage && @submitter
 
+      record_health_metrics
       drained = @storage.drain
       return 0 if drained.empty?
 
@@ -100,13 +102,14 @@ module Flare
     private
 
     def post_to_pool
+      record_health_metrics
       drained = @storage.drain
       if drained.empty?
-        Flare.log "No metrics to flush"
+        log "No metrics to flush"
         return
       end
 
-      Flare.log "Drained #{drained.size} metric keys for submission"
+      log "Drained #{drained.size} metric keys for submission"
       @pool.post { submit_to_cloud(drained) }
     rescue => e
       warn "[Flare] Metric drain error: #{e.message}"
@@ -120,5 +123,15 @@ module Flare
     rescue => e
       warn "[Flare] Metric submission error: #{e.message}"
     end
+
+    def record_health_metrics
+      @health_reporters.each { |reporter| reporter.record(@storage) }
+    rescue => e
+      warn "[Flare] Health metric recording error: #{e.message}"
+    end
+
+    def log(message)
+      Flare.log(message) if Flare.respond_to?(:log)
+    end
   end
 end

data/lib/flare/metric_storage.rb

@@ -16,6 +16,11 @@ module Flare
       counter.increment(duration_ms: duration_ms, error: error)
     end
 
+    def add(key, count:, sum_ms:, error_count: 0)
+      counter = @storage.compute_if_absent(key) { MetricCounter.new }
+      counter.add(count: count, sum_ms: sum_ms, error_count: error_count)
+    end
+
     # Atomically retrieves and clears all metrics.
     # Returns a frozen hash of MetricKey => counter data.
     def drain