flare 0.1.1 → 0.3.0

data/lib/flare/trace_exporter.rb

@@ -0,0 +1,143 @@
+# frozen_string_literal: true
+
+require "json"
+require "zlib"
+require "stringio"
+require "logger"
+require "uri"
+require "concurrent/atomic/atomic_fixnum"
+require "opentelemetry/sdk"
+
+require_relative "sampler"
+require_relative "trace_blob"
+require_relative "http_transport"
+
+module Flare
+  # Custom OTel exporter. For each batch FilteringSpanProcessor hands over:
+  #
+  #   1. Group spans by trace_id.
+  #   2. For each trace, build a Flare::TraceBlob and gzip-JSON-encode it.
+  #   3. Check out a presigned R2 PUT URL from UploadUrlPool.
+  #   4. PUT the gzipped body straight to R2 -- Flare's server is NOT in
+  #      the trace-bytes path.
+  #   5. After R2 returns 200, POST /api/traces { key } using the
+  #      customer's push token + Flare-Project / Flare-Environment headers.
+  #      That's the self-notify hop the design swapped in for the CF Worker.
+  #
+  # 403 from R2 means the presigned URL expired between issue and use;
+  # discard, check out the next URL, retry once. Pool empty -> FAILURE.
+  # Notify-POST failure is logged + counted but doesn't fail the export
+  # (the blob is in R2, just won't be processed; incoming/* lifecycle
+  # cleans it up in 1hr).
+  class TraceExporter
+    SUCCESS = OpenTelemetry::SDK::Trace::Export::SUCCESS
+    FAILURE = OpenTelemetry::SDK::Trace::Export::FAILURE
+
+    PUT_HEADERS = {
+      "Content-Type"     => "application/json",
+      "Content-Encoding" => "gzip"
+    }.freeze
+
+    attr_reader :put_failure_count, :notify_failure_count, :pool_empty_count, :exception_count
+
+    def initialize(pool:, notify_url:, api_key:, project:, environment:,
+                   transport: nil, logger: nil)
+      @pool         = pool
+      @notify_url   = notify_url.to_s
+      @api_key      = api_key
+      @project      = project
+      @environment  = environment
+      @transport    = transport || HttpTransport.new
+      @logger       = logger || Logger.new($stderr, level: Logger::WARN)
+
+      @put_failure_count    = Concurrent::AtomicFixnum.new(0)
+      @notify_failure_count = Concurrent::AtomicFixnum.new(0)
+      @pool_empty_count     = Concurrent::AtomicFixnum.new(0)
+      @exception_count      = Concurrent::AtomicFixnum.new(0)
+    end
+
+    def export(spans, timeout: nil)
+      grouped = spans.group_by(&:trace_id)
+      return SUCCESS if grouped.empty?
+
+      overall = SUCCESS
+      grouped.each do |trace_id, group|
+        result = ship(TraceBlob.build(trace_id: trace_id, spans: group))
+        overall = FAILURE if result == FAILURE
+      end
+      overall
+    rescue StandardError => e
+      @exception_count.increment
+      @logger.warn("[Flare::TraceExporter] export raised: #{e.class}: #{e.message}")
+      FAILURE
+    end
+
+    def force_flush(timeout: nil)
+      SUCCESS
+    end
+
+    def shutdown(timeout: nil)
+      SUCCESS
+    end
+
+    private
+
+    def ship(blob, retried: false)
+      return FAILURE if blob.nil?
+
+      entry = @pool.checkout
+      if entry.nil?
+        @pool_empty_count.increment
+        return FAILURE
+      end
+
+      body = gzip(JSON.generate(blob.to_h))
+      response = @transport.put(entry[:put_url], body, PUT_HEADERS)
+
+      case response.code
+      when "200", "204"
+        notify(entry[:key])
+        SUCCESS
+      when "403"
+        # Presigned URL probably expired; try once more with the next one.
+        retried ? record_put_failure(response) : ship(blob, retried: true)
+      else
+        record_put_failure(response)
+      end
+    end
+
+    def notify(key)
+      response = @transport.post(@notify_url, JSON.generate(key: key), notify_headers)
+      return if response.code == "202"
+
+      @notify_failure_count.increment
+      @logger.warn("[Flare::TraceExporter] notify failed: HTTP #{response.code}")
+    rescue StandardError => e
+      @notify_failure_count.increment
+      @logger.warn("[Flare::TraceExporter] notify exception: #{e.class}: #{e.message}")
+    end
+
+    def notify_headers
+      {
+        "Content-Type"      => "application/json",
+        "Authorization"     => "Bearer #{@api_key}",
+        "Flare-Project"     => @project,
+        "Flare-Environment" => @environment
+      }
+    end
+
+    def record_put_failure(response)
+      @put_failure_count.increment
+      @logger.warn("[Flare::TraceExporter] PUT failed: HTTP #{response.code}")
+      FAILURE
+    end
+
+    def gzip(body)
+      io = StringIO.new
+      gz = Zlib::GzipWriter.new(io)
+      gz.write(body)
+      gz.close
+      io.string
+    end
+  end
+end

data/lib/flare/trace_health_reporter.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+require_relative "metric_key"
+
+module Flare
+  # Records client-side tracing health into MetricStorage so flare-web can
+  # warn when local buffering, URL exhaustion, or export errors reduce trace
+  # fidelity.
+  class TraceHealthReporter
+    NAMESPACE = "sdk"
+    SERVICE = "flare-ruby"
+    TARGET = "tracing"
+
+    def initialize(processor:, pool:, exporter:)
+      @processor = processor
+      @pool = pool
+      @exporter = exporter
+      @last = {}
+      @mutex = Mutex.new
+    end
+
+    def record(storage, bucket: Time.now.utc)
+      @mutex.synchronize do
+        record_counter(storage, bucket, "dropped_spans", @processor.dropped_count.value)
+        record_counter(storage, bucket, "export_failures", @processor.failed_export_count.value)
+        record_counter(storage, bucket, "processor_exceptions", @processor.exception_count.value)
+
+        record_counter(storage, bucket, "upload_url_pool_empty", @pool.empty_count.value)
+        record_counter(storage, bucket, "upload_url_expired", @pool.expired_count.value)
+
+        record_counter(storage, bucket, "r2_put_failures", @exporter.put_failure_count.value)
+        record_counter(storage, bucket, "notify_failures", @exporter.notify_failure_count.value)
+        record_counter(storage, bucket, "trace_pool_empty", @exporter.pool_empty_count.value)
+        record_counter(storage, bucket, "trace_export_exceptions", @exporter.exception_count.value)
+
+        buffer_size = @processor.buffer_size
+        buffer_high_watermark = @processor.buffer_high_watermark.value
+        record_gauge(storage, bucket, "buffer_size", buffer_size)
+        record_gauge(storage, bucket, "buffer_high_watermark", buffer_high_watermark)
+        record_gauge(storage, bucket, "buffer_limit", @processor.max_queue) if buffer_size.positive? || buffer_high_watermark.positive?
+        @processor.reset_buffer_high_watermark
+      end
+    end
+
+    private
+
+    def record_counter(storage, bucket, operation, current)
+      previous = @last.fetch(operation, 0)
+      @last[operation] = current
+      delta = current - previous
+      return unless delta.positive?
+
+      storage.add(key(bucket, operation), count: delta, sum_ms: 0, error_count: 0)
+    end
+
+    def record_gauge(storage, bucket, operation, value)
+      storage.add(key(bucket, operation), count: 1, sum_ms: value, error_count: 0)
+    end
+
+    def key(bucket, operation)
+      MetricKey.new(
+        bucket: bucket_time(bucket),
+        namespace: NAMESPACE,
+        service: SERVICE,
+        target: TARGET,
+        operation: operation
+      )
+    end
+
+    def bucket_time(time)
+      Time.utc(time.year, time.month, time.day, time.hour, time.min, 0)
+    end
+  end
+end

data/lib/flare/upload_url_pool.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+require "concurrent/atomic/atomic_reference"
+
+module Flare
+  # Thread-safe pool of presigned R2 PUT URLs the RuleManager fills from
+  # the /api/rules response. TraceExporter checks one out before each
+  # upload; if the pool is empty (no active rules, no fresh URLs) it
+  # returns nil and the exporter gives up on that batch -- caller decides
+  # what to do.
+  #
+  # Each entry is a Hash: { upload_id:, key:, put_url:, expires_at: }.
+  # expires_at is a Time; entries past their expiry are skipped on checkout.
+  #
+  # Fork-safe: after_fork clears the pool so child processes don't reuse
+  # parent URLs (each child polls its own copy from /api/rules anyway).
+  class UploadUrlPool
+    attr_reader :checkouts, :empty_count, :expired_count
+
+    def initialize
+      @entries_ref   = Concurrent::AtomicReference.new([].freeze)
+      @checkouts     = Concurrent::AtomicFixnum.new(0)
+      @empty_count   = Concurrent::AtomicFixnum.new(0)
+      @expired_count = Concurrent::AtomicFixnum.new(0)
+    end
+
+    def replace(entries)
+      normalized = (entries || []).filter_map { |raw| normalize(raw) }
+      @entries_ref.set(normalized.freeze)
+    end
+
+    def checkout
+      now = Time.now
+      loop do
+        current = @entries_ref.get
+        if current.empty?
+          @empty_count.increment
+          return nil
+        end
+
+        candidate, *rest = current
+        next_state = rest.freeze
+        next unless @entries_ref.compare_and_set(current, next_state)
+
+        if expired?(candidate, now)
+          @expired_count.increment
+          next # try the next one
+        end
+
+        @checkouts.increment
+        return candidate
+      end
+    end
+
+    def size
+      @entries_ref.get.length
+    end
+
+    def empty?
+      size.zero?
+    end
+
+    def clear
+      @entries_ref.set([].freeze)
+    end
+
+    # Drop URLs that have already passed their expires_at. Cheap; safe to
+    # call from RuleManager's scheduler in between polls.
+    def sweep
+      now = Time.now
+      current = @entries_ref.get
+      live    = current.reject { |e| expired?(e, now) }
+      return 0 if live.length == current.length
+
+      @entries_ref.set(live.freeze)
+      current.length - live.length
+    end
+
+    # Call from Flare.after_fork. Parent's URLs aren't usable from the
+    # child's point of view (each child should get its own from a fresh
+    # /api/rules poll), so just drop them.
+    def after_fork
+      clear
+    end
+
+    private
+
+    def normalize(raw)
+      h = raw.is_a?(Hash) ? raw : nil
+      return nil unless h
+
+      upload_id  = h[:upload_id] || h["upload_id"]
+      key        = h[:key]       || h["key"]
+      put_url    = h[:put_url]   || h["put_url"]
+      expires_at = h[:expires_at] || h["expires_at"]
+      return nil if upload_id.nil? || key.nil? || put_url.nil?
+
+      expires_at = Time.iso8601(expires_at) if expires_at.is_a?(String)
+      { upload_id: upload_id, key: key, put_url: put_url, expires_at: expires_at }
+    rescue StandardError
+      nil
+    end
+
+    def expired?(entry, now)
+      entry[:expires_at] && entry[:expires_at] <= now
+    end
+  end
+end

data/lib/flare/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Flare
-  VERSION = "0.1.1"
+  VERSION = "0.3.0"
 end

data/lib/flare/web_marker_subscriber.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+require "opentelemetry/sdk"
+
+module Flare
+  # Path 2: ActiveSupport::Notifications subscriber that fires on
+  # `start_processing.action_controller`, after Rails has routed to a
+  # controller#action. At that point the rack server span's start
+  # attributes don't yet carry code.namespace/code.function -- only the
+  # ActionPack instrumentation adds them, and Flare::Sampler's start-time
+  # decision (RECORD_ONLY) was already locked in.
+  #
+  # The subscriber consults the same sampler's rule set, finds any whose
+  # match_attributes match the now-known controller/action, applies the
+  # deterministic trace_id_ratio gate (CAF-1: no rate bypass on Path 2),
+  # and on pass calls marker.mark(trace_id, owner_span_id:, rule_id:).
+  # FilteringSpanProcessor then forwards every span in the trace to the
+  # exporter and unmarks when the owner (this rack span) finishes.
+  class WebMarkerSubscriber
+    NOTIFICATION = "start_processing.action_controller"
+
+    def initialize(sampler:, marker:)
+      @sampler = sampler
+      @marker = marker
+    end
+
+    def start
+      @subscriber = ActiveSupport::Notifications.subscribe(NOTIFICATION) do |*, payload|
+        handle(payload)
+      end
+      self
+    end
+
+    def stop
+      ActiveSupport::Notifications.unsubscribe(@subscriber) if @subscriber
+      @subscriber = nil
+      self
+    end
+
+    # Public for tests so they don't have to drive ActiveSupport::Notifications.
+    # current_span lets tests inject a context; in production it's the
+    # rack server span on the current thread.
+    def handle(payload, current_span: OpenTelemetry::Trace.current_span)
+      return unless current_span
+      ctx = current_span.context
+      return unless ctx && ctx.valid?
+
+      attrs = candidate_attributes(payload)
+      return if attrs.empty?
+
+      @sampler.rules.each do |rule|
+        next unless matches?(rule, attrs)
+        next unless @sampler.trace_id_ratio(ctx.trace_id) < rule.rate
+
+        current_span.set_attribute(Flare::Sampler::RULE_ID_ATTRIBUTE, rule.id) if current_span.respond_to?(:set_attribute)
+        @marker.mark(ctx.trace_id, owner_span_id: ctx.span_id, rule_id: rule.id)
+        break
+      end
+    end
+
+    private
+
+    def candidate_attributes(payload)
+      controller = payload[:controller] || payload["controller"]
+      action     = payload[:action]     || payload["action"]
+      {
+        "code.namespace" => controller,
+        "code.function"  => action
+      }.compact
+    end
+
+    def matches?(rule, attrs)
+      rule.match_attributes.all? { |k, v| attrs[k] == v }
+    end
+  end
+end

data/lib/flare.rb

@@ -13,6 +13,15 @@ require_relative "flare/metric_flusher"
 require_relative "flare/backoff_policy"
 require_relative "flare/metric_submitter"
 
+require_relative "flare/sampler"
+require_relative "flare/marker"
+require_relative "flare/web_marker_subscriber"
+require_relative "flare/filtering_span_processor"
+require_relative "flare/upload_url_pool"
+require_relative "flare/trace_exporter"
+require_relative "flare/rule_manager"
+require_relative "flare/trace_health_reporter"
+
 module Flare
   class Error < StandardError; end
 
@@ -114,15 +123,43 @@ module Flare
     @metric_flusher = flusher
   end
 
+  # Trace-sampling components, exposed for tests + manual after_fork wiring.
+  def sampler         = @sampler
+  def marker          = @marker
+  def upload_url_pool = @upload_url_pool
+  def rule_manager    = @rule_manager
+  def trace_span_processor = @trace_span_processor
+  def trace_health_reporter = @trace_health_reporter
+
   # Manually flush metrics (useful for testing or forced flushes).
   def flush_metrics
     @metric_flusher&.flush_now || 0
   end
 
-  # Re-initialize metric flusher after fork.
+  # Default project key, derived from the host Rails app's module name.
+  # Customers can override by configuring something else once we expose
+  # configuration.project; for v0.3 this matches MetricSubmitter's behavior.
+  def service_name_for_app
+    if defined?(Rails) && Rails.respond_to?(:application) && Rails.application
+      Rails.application.class.module_parent_name.underscore rescue "rails_app"
+    else
+      "app"
+    end
+  end
+
+  def rails_env_name
+    if defined?(Rails) && Rails.respond_to?(:env)
+      Rails.env.to_s
+    else
+      ENV.fetch("RACK_ENV", "development")
+    end
+  end
+
+  # Re-initialize background threads after fork.
   # Call this from Puma/Unicorn after_fork hooks.
   def after_fork
     @metric_flusher&.after_fork
+    @rule_manager&.after_fork
   end
 
   # Configure OpenTelemetry SDK and instrumentations. Must run before the
@@ -135,13 +172,9 @@ module Flare
     # Suppress noisy OTel INFO logs
     OpenTelemetry.logger = Logger.new(STDOUT, level: Logger::WARN)
 
-    service_name = if defined?(Rails) && Rails.application
-      Rails.application.class.module_parent_name.underscore rescue "rails_app"
-    else
-      "app"
-    end
+    service_name = service_name_for_app
 
-    # Require only the instrumentations we want
+    # Require flare's bundled instrumentations
     require "opentelemetry-instrumentation-rack"
     require "opentelemetry-instrumentation-net_http"
     require "opentelemetry-instrumentation-active_support"
@@ -165,19 +198,25 @@ module Flare
         log "Spans enabled (database=#{configuration.database_path})"
       end
 
-      # Configure specific instrumentations
-      c.use "OpenTelemetry::Instrumentation::Rack",
-        untraced_requests: ->(env) {
-          request = Rack::Request.new(env)
-          return true if request.path.start_with?("/flare")
-
-          configuration.ignore_request.call(request)
+      # Auto-detect and install all OTel instrumentation gems in the bundle.
+      # Apps can add gems like opentelemetry-instrumentation-sidekiq to their
+      # Gemfile and they'll be picked up automatically.
+      c.use_all(
+        "OpenTelemetry::Instrumentation::Rack" => {
+          untraced_requests: ->(env) {
+            request = Rack::Request.new(env)
+            return true if request.path.start_with?("/flare")
+
+            configuration.ignore_request.call(request)
+          }
+        },
+        # Name Sidekiq job spans after the worker class (e.g. "MyWorker
+        # process") instead of the upstream default of the queue name
+        # ("default process"), matching how ActiveJob spans are named.
+        "OpenTelemetry::Instrumentation::Sidekiq" => {
+          span_naming: :job_class,
         }
-      c.use "OpenTelemetry::Instrumentation::Net::HTTP"
-      c.use "OpenTelemetry::Instrumentation::ActiveSupport"
-      c.use "OpenTelemetry::Instrumentation::ActionPack" if defined?(ActionController)
-      c.use "OpenTelemetry::Instrumentation::ActionView" if defined?(ActionView)
-      c.use "OpenTelemetry::Instrumentation::ActiveJob" if defined?(ActiveJob)
+      )
     end
 
     # Subscribe to common ActiveSupport notification patterns
@@ -195,12 +234,98 @@ module Flare
         span_processor.shutdown
         log "Span processor flushed and stopped"
       end
+      if @trace_span_processor
+        @trace_span_processor.force_flush
+        @trace_span_processor.shutdown
+        log "Trace span processor flushed and stopped"
+      end
       log "Shutdown complete"
     end
 
     @otel_configured = true
   end
 
+  # Start the trace-rules poller. Polls GET /api/rules every
+  # tracing_poll_interval (default 30s) so the in-process sampler + URL
+  # pool stay current. Called from config.after_initialize -- after the
+  # user's configure block has run -- so configuration.url / .key /
+  # .tracing_enabled are settled.
+  def start_rule_manager
+    return unless configuration.tracing_submission_configured?
+
+    setup_tracing_components
+    return unless @sampler && @marker && @upload_url_pool
+
+    @rule_manager = RuleManager.new(
+      sampler:     @sampler,
+      marker:      @marker,
+      pool:        @upload_url_pool,
+      base_url:    configuration.url,
+      api_key:     configuration.key,
+      project:     service_name_for_app,
+      environment: rails_env_name,
+      interval:    configuration.tracing_poll_interval
+    )
+    @rule_manager.start
+    log "Rule manager started (poll=#{configuration.tracing_poll_interval}s)"
+
+    at_exit { @rule_manager&.stop }
+  end
+
+  def setup_tracing_components
+    return if @trace_span_processor
+
+    @sampler         = Sampler.new
+    @marker          = Marker.new
+    @upload_url_pool = UploadUrlPool.new
+
+    # Trace sampling: server-controlled per-route capture. The sampler runs
+    # at span start; for routes it can't decide there (Rails web spans get
+    # their controller#action attributes set post-routing) the marker +
+    # WebMarkerSubscriber handle it. The RECORD_ONLY delegates keep children
+    # of unsampled local and remote parents recording so processors still see
+    # web requests that arrive with an unsampled traceparent header.
+    #
+    # Sampler is set on the tracer_provider AFTER SDK.configure -- the SDK's
+    # Configurator block doesn't expose a `sampler=`; the provider does.
+    OpenTelemetry.tracer_provider.sampler =
+      OpenTelemetry::SDK::Trace::Samplers.parent_based(
+        root: @sampler,
+        remote_parent_sampled: ALWAYS_RECORD_ONLY,
+        remote_parent_not_sampled: ALWAYS_RECORD_ONLY,
+        local_parent_not_sampled: ALWAYS_RECORD_ONLY
+      )
+
+    @trace_exporter = TraceExporter.new(
+      pool:        @upload_url_pool,
+      notify_url:  "#{configuration.url.to_s.chomp('/')}/api/traces",
+      api_key:     configuration.key,
+      project:     service_name_for_app,
+      environment: rails_env_name
+    )
+
+    @trace_span_processor = FilteringSpanProcessor.new(
+      exporter: @trace_exporter,
+      marker: @marker,
+      max_queue: configuration.tracing_max_queue
+    )
+    OpenTelemetry.tracer_provider.add_span_processor(@trace_span_processor)
+
+    @trace_health_reporter = TraceHealthReporter.new(
+      processor: @trace_span_processor,
+      pool: @upload_url_pool,
+      exporter: @trace_exporter
+    )
+
+    # Path 2 trace marking. Rails-only -- in non-Rails contexts the
+    # subscriber would never fire but creating it is harmless.
+    if defined?(ActiveSupport::Notifications)
+      @web_marker_subscriber = WebMarkerSubscriber.new(sampler: @sampler, marker: @marker).start
+    end
+
+    log "Tracing enabled (poll=#{configuration.tracing_poll_interval}s)"
+  end
+
   # Start the metrics flusher. Called from config.after_initialize so
   # user configuration (metrics_enabled, flush_interval, etc.) is applied.
   def start_metrics_flusher
@@ -223,7 +348,8 @@ module Flare
       @metric_flusher = MetricFlusher.new(
         storage: @metric_storage,
         submitter: submitter,
-        interval: configuration.metrics_flush_interval
+        interval: configuration.metrics_flush_interval,
+        health_reporters: @trace_health_reporter ? [@trace_health_reporter] : []
       )
       @metric_flusher.start
       log "Metrics flusher started (interval=#{configuration.metrics_flush_interval}s)"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: flare
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.3.0
 platform: ruby
 authors:
 - John Nunemaker
@@ -198,18 +198,28 @@ files:
 - lib/flare/cli/status_command.rb
 - lib/flare/configuration.rb
 - lib/flare/engine.rb
+- lib/flare/filtering_span_processor.rb
 - lib/flare/http_metrics_config.rb
+- lib/flare/http_transport.rb
+- lib/flare/marker.rb
 - lib/flare/metric_counter.rb
 - lib/flare/metric_flusher.rb
 - lib/flare/metric_key.rb
 - lib/flare/metric_span_processor.rb
 - lib/flare/metric_storage.rb
 - lib/flare/metric_submitter.rb
+- lib/flare/rule_manager.rb
+- lib/flare/sampler.rb
 - lib/flare/source_location.rb
 - lib/flare/sqlite_exporter.rb
 - lib/flare/storage.rb
 - lib/flare/storage/sqlite.rb
+- lib/flare/trace_blob.rb
+- lib/flare/trace_exporter.rb
+- lib/flare/trace_health_reporter.rb
+- lib/flare/upload_url_pool.rb
 - lib/flare/version.rb
+- lib/flare/web_marker_subscriber.rb
 - public/flare-assets/flare.css
 - public/flare-assets/images/flipper.png
 homepage: https://github.com/jnunemaker/flare