flare 0.2.0 → 0.3.0

data/lib/flare/rule_manager.rb

@@ -0,0 +1,140 @@
+# frozen_string_literal: true
+
+require "json"
+require "logger"
+require "concurrent/timer_task"
+require "concurrent/atomic/atomic_fixnum"
+
+require_relative "http_transport"
+
+module Flare
+  # The SDK's only poll. Every interval seconds (default 30) it does a
+  # GET /api/rules; the 200 response carries the active TraceRules (with
+  # server-computed sample_rate) plus a bag of presigned R2 PUT URLs.
+  # We hand the rules to Sampler#update_rules and the URLs to
+  # UploadUrlPool#replace, and sweep the Marker so stuck rack-span
+  # entries don't linger.
+  #
+  # ETag-guarded: subsequent polls send If-None-Match. A 304 still gets
+  # us a Marker.sweep but doesn't touch sampler or pool. 401/403 stops
+  # the poller (misconfigured token shouldn't beat down the server).
+  # 5xx and exceptions are logged + counted; the timer just tries again
+  # on the next tick.
+  #
+  # Fork-safe: after_fork clears the pool and restarts the timer in the
+  # child process so each child polls independently.
+  class RuleManager
+    DEFAULT_INTERVAL = 30
+
+    attr_reader :poll_count, :etag, :stopped_due_to_auth, :last_error_count
+
+    def initialize(sampler:, marker:, pool:, base_url:, api_key:, project:, environment:,
+                   interval: DEFAULT_INTERVAL, transport: nil, logger: nil)
+      @sampler     = sampler
+      @marker      = marker
+      @pool        = pool
+      @rules_url   = "#{base_url.to_s.chomp('/')}/api/rules"
+      @api_key     = api_key
+      @project     = project
+      @environment = environment
+      @interval    = interval
+      @transport   = transport || HttpTransport.new
+      @logger      = logger || Logger.new($stderr, level: Logger::WARN)
+
+      @etag = nil
+      @poll_count          = Concurrent::AtomicFixnum.new(0)
+      @last_error_count    = Concurrent::AtomicFixnum.new(0)
+      @stopped_due_to_auth = false
+      @pid                 = $$
+    end
+
+    def start
+      return self if @timer || @stopped_due_to_auth
+
+      @timer = Concurrent::TimerTask.execute(
+        execution_interval: @interval,
+        run_now:            true,
+        name:               "flare-rule-manager-timer"
+      ) { poll_safely }
+      self
+    end
+
+    def stop
+      if @timer
+        @timer.shutdown
+        @timer.wait_for_termination(1)
+        @timer.kill unless @timer.shutdown?
+        @timer = nil
+      end
+      self
+    end
+
+    def running?
+      @timer ? @timer.running? : false
+    end
+
+    def after_fork
+      @pid = $$
+      @pool.after_fork
+      stop
+      start
+    end
+
+    # Public so callers can force a poll (tests + integration tests).
+    def poll_now
+      poll_safely
+    end
+
+    private
+
+    def poll_safely
+      poll
+    rescue StandardError => e
+      @last_error_count.increment
+      @logger.warn("[Flare::RuleManager] poll exception: #{e.class}: #{e.message}")
+    end
+
+    def poll
+      return if @stopped_due_to_auth
+
+      response = @transport.get(@rules_url, request_headers)
+      @poll_count.increment
+
+      case response.code
+      when "304"
+        @marker.sweep
+      when "200"
+        @etag = response.header("ETag")
+        apply(JSON.parse(response.body))
+        @marker.sweep
+      when "401", "403"
+        @stopped_due_to_auth = true
+        @logger.warn("[Flare::RuleManager] auth failed (#{response.code}); stopping poll")
+        stop
+      else
+        @last_error_count.increment
+        @logger.warn("[Flare::RuleManager] unexpected #{response.code}")
+      end
+    end
+
+    def request_headers
+      headers = {
+        "Authorization"     => "Bearer #{@api_key}",
+        "Flare-Project"     => @project,
+        "Flare-Environment" => @environment
+      }
+      headers["If-None-Match"] = @etag if @etag
+      headers
+    end
+
+    # Server payload shape (see tirana-v2 Api::RulesController):
+    #   { "trace_rules": [{ "id", "match_attributes", "rate", ..., "urls": [...] }] }
+    def apply(payload)
+      rules = payload["trace_rules"] || []
+      @sampler.update_rules(rules)
+
+      url_entries = rules.flat_map { |r| Array(r["urls"]) }
+      @pool.replace(url_entries)
+    end
+  end
+end

data/lib/flare/sampler.rb

@@ -0,0 +1,130 @@
+# frozen_string_literal: true
+
+require "concurrent/atomic/atomic_reference"
+require "opentelemetry/sdk"
+
+module Flare
+  # Path 1 trace sampler. At span start, iterates active rules; returns
+  # RECORD_AND_SAMPLE when one matches and the deterministic trace_id_ratio
+  # falls under the rule's rate. Otherwise RECORD_ONLY -- the span still
+  # records so MetricSpanProcessor sees it; the trace export decision for
+  # web spans is deferred to Path 2 via Flare::Marker.
+  #
+  # Used as the `root` sampler inside an OTel ParentBased sampler so root
+  # spans go through this logic but child spans inherit upstream decisions.
+  # The `local_parent_not_sampled` slot of the ParentBased should point at
+  # Flare::ALWAYS_RECORD_ONLY -- the default ALWAYS_OFF would drop children
+  # of an unsampled local parent, making them NoOp spans the processors
+  # never see.
+  #
+  # Rules are pushed in via update_rules from Flare::RuleManager; the swap
+  # is atomic, and malformed rule entries are dropped with a counter so a
+  # bad server payload can't crash the tracing path.
+  class Sampler
+    Decision = OpenTelemetry::SDK::Trace::Samplers::Decision
+    Result   = OpenTelemetry::SDK::Trace::Samplers::Result
+
+    RULE_ID_ATTRIBUTE = "flare.rule_id"
+
+    Rule = Struct.new(:id, :match_attributes, :rate, keyword_init: true)
+
+    attr_reader :dropped_rule_count
+
+    def initialize
+      @rules_ref = Concurrent::AtomicReference.new([].freeze)
+      @dropped_rule_count = Concurrent::AtomicFixnum.new(0)
+    end
+
+    # new_rules: an array of rule hashes from GET /api/rules, e.g.
+    #   [{ "id" => 1, "match_attributes" => {...}, "rate" => 0.5 }, ...]
+    # Entries that don't validate are skipped (counted in dropped_rule_count).
+    def update_rules(new_rules)
+      validated = (new_rules || []).filter_map { |r| validate(r) }
+      @dropped_rule_count.increment((new_rules || []).length - validated.length)
+      @rules_ref.set(validated.freeze)
+    end
+
+    def rules
+      @rules_ref.get
+    end
+
+    def should_sample?(trace_id:, parent_context:, links:, name:, kind:, attributes:)
+      tracestate = tracestate_from(parent_context)
+
+      rules.each do |rule|
+        next unless matches?(rule, attributes)
+        next unless trace_id_ratio(trace_id) < rule.rate
+
+        merged = (attributes || {}).merge(RULE_ID_ATTRIBUTE => rule.id)
+        return Result.new(decision: Decision::RECORD_AND_SAMPLE, attributes: merged, tracestate: tracestate)
+      end
+
+      Result.new(decision: Decision::RECORD_ONLY, tracestate: tracestate)
+    end
+
+    def description
+      "Flare::Sampler"
+    end
+
+    # Cross-language formula: last 8 bytes of the 16-byte raw trace_id as
+    # uint64-big-endian, divided by 2^64. Same in every Flare SDK so the
+    # server can reproduce the decision if it ever needs to.
+    def trace_id_ratio(trace_id)
+      bytes = trace_id.is_a?(String) ? trace_id.bytes : Array(trace_id)
+      tail = bytes.last(8)
+      n = 0
+      tail.each { |b| n = (n << 8) | b }
+      n.to_f / (1 << 64)
+    end
+
+    private
+
+    def tracestate_from(parent_context)
+      OpenTelemetry::Trace.current_span(parent_context).context.tracestate ||
+        OpenTelemetry::Trace::Tracestate::DEFAULT
+    end
+
+    def matches?(rule, attributes)
+      return false if attributes.nil?
+      rule.match_attributes.all? { |k, v| attributes[k] == v }
+    end
+
+    def validate(raw)
+      return nil unless raw.is_a?(Hash)
+
+      id    = raw["id"] || raw[:id]
+      match = raw["match_attributes"] || raw[:match_attributes]
+      rate  = raw["rate"] || raw[:rate]
+
+      return nil if id.nil?
+      return nil unless match.is_a?(Hash) && match.any?
+      return nil unless match.all? { |k, v| k.is_a?(String) && v.is_a?(String) && !v.empty? }
+      return nil unless rate.is_a?(Numeric) && rate > 0.0 && rate <= 1.0
+
+      Rule.new(id: id, match_attributes: match, rate: rate.to_f)
+    rescue StandardError
+      nil
+    end
+  end
+
+  # Tiny sampler whose should_sample? returns RECORD_ONLY for every span.
+  # Slot this into the ParentBased local_parent_not_sampled position so
+  # children of an unsampled local parent stay recording (the default
+  # ALWAYS_OFF turns them into NoOp spans no processor ever sees).
+  class AlwaysRecordOnly
+    Decision = OpenTelemetry::SDK::Trace::Samplers::Decision
+    Result   = OpenTelemetry::SDK::Trace::Samplers::Result
+
+    def should_sample?(parent_context: nil, **)
+      tracestate = OpenTelemetry::Trace.current_span(parent_context).context.tracestate ||
+        OpenTelemetry::Trace::Tracestate::DEFAULT
+      Result.new(decision: Decision::RECORD_ONLY, tracestate: tracestate)
+    end
+
+    def description
+      "Flare::AlwaysRecordOnly"
+    end
+  end
+
+  ALWAYS_RECORD_ONLY = AlwaysRecordOnly.new
+end

data/lib/flare/trace_blob.rb

@@ -0,0 +1,116 @@
+# frozen_string_literal: true
+
+require "time"
+
+module Flare
+  # Value object that turns a group of OTel span_data for a single trace
+  # into the Flare-JSON wire format the server expects:
+  #
+  #   {
+  #     "trace_id":       "<hex>",
+  #     "trace_rule_id":  <int|nil>,
+  #     "root_name":      "<string>",
+  #     "started_at":     "<iso8601>",
+  #     "duration_ms":    <int>,
+  #     "spans": [
+  #       { "id", "parent_id", "name", "started_at",
+  #         "duration_ms", "attributes" }
+  #     ]
+  #   }
+  #
+  # The trace_rule_id is read from any span carrying the
+  # `flare.rule_id` attribute (Path 1 sets it on the sampled root, Path 2
+  # sets it on the rack owner span via WebMarkerSubscriber).
+  class TraceBlob
+    ZERO_SPAN_ID = ("\x00".b * 8).freeze
+    ROOT_NAME_LIMIT = 255
+
+    def self.build(trace_id:, spans:)
+      return nil if spans.nil? || spans.empty?
+      new(trace_id: trace_id, spans: spans)
+    end
+
+    def initialize(trace_id:, spans:)
+      @trace_id = trace_id
+      @spans    = spans
+    end
+
+    def to_h
+      root = find_root
+      {
+        "trace_id"      => hexify(@trace_id),
+        "trace_rule_id" => rule_id_from_spans,
+        "root_name"     => root_name(root),
+        "started_at"    => iso(root&.start_timestamp),
+        "duration_ms"   => duration_ms(root),
+        "spans"         => @spans.map { |s| span_to_h(s) }
+      }
+    end
+
+    private
+
+    def find_root
+      @spans.find { |s| entry_span?(s) && rule_id_attribute(s) } ||
+        @spans.find { |s| root?(s) } ||
+        @spans.find { |s| entry_span?(s) } ||
+        @spans.find { |s| rule_id_attribute(s) } ||
+        @spans.first
+    end
+
+    def root?(span)
+      pid = span.parent_span_id
+      pid.nil? || pid.empty? || pid == ZERO_SPAN_ID
+    end
+
+    def entry_span?(span)
+      return false unless span.respond_to?(:kind)
+
+      span.kind == :server || span.kind == :consumer
+    end
+
+    def rule_id_from_spans
+      @spans.each do |s|
+        value = rule_id_attribute(s)
+        return value if value
+      end
+      nil
+    end
+
+    def rule_id_attribute(span)
+      attrs = span.attributes
+      return nil unless attrs
+
+      attrs[Sampler::RULE_ID_ATTRIBUTE] || attrs[Sampler::RULE_ID_ATTRIBUTE.to_sym]
+    end
+
+    def root_name(root)
+      root&.name&.to_s&.slice(0, ROOT_NAME_LIMIT)
+    end
+
+    def span_to_h(span)
+      {
+        "id"          => hexify(span.span_id),
+        "parent_id"   => root?(span) ? nil : hexify(span.parent_span_id),
+        "name"        => span.name,
+        "started_at"  => iso(span.start_timestamp),
+        "duration_ms" => duration_ms(span),
+        "attributes"  => span.attributes || {}
+      }
+    end
+
+    def hexify(bytes)
+      return nil if bytes.nil?
+      bytes.unpack1("H*")
+    end
+
+    def iso(nanos)
+      return nil if nanos.nil?
+      Time.at(nanos / 1_000_000_000.0).utc.iso8601(6)
+    end
+
+    def duration_ms(span)
+      return nil unless span && span.start_timestamp && span.end_timestamp
+      ((span.end_timestamp - span.start_timestamp) / 1_000_000).to_i
+    end
+  end
+end

data/lib/flare/trace_exporter.rb

@@ -0,0 +1,143 @@
+# frozen_string_literal: true
+
+require "json"
+require "zlib"
+require "stringio"
+require "logger"
+require "uri"
+require "concurrent/atomic/atomic_fixnum"
+require "opentelemetry/sdk"
+
+require_relative "sampler"
+require_relative "trace_blob"
+require_relative "http_transport"
+
+module Flare
+  # Custom OTel exporter. For each batch FilteringSpanProcessor hands over:
+  #
+  #   1. Group spans by trace_id.
+  #   2. For each trace, build a Flare::TraceBlob and gzip-JSON-encode it.
+  #   3. Check out a presigned R2 PUT URL from UploadUrlPool.
+  #   4. PUT the gzipped body straight to R2 -- Flare's server is NOT in
+  #      the trace-bytes path.
+  #   5. After R2 returns 200, POST /api/traces { key } using the
+  #      customer's push token + Flare-Project / Flare-Environment headers.
+  #      That's the self-notify hop the design swapped in for the CF Worker.
+  #
+  # 403 from R2 means the presigned URL expired between issue and use;
+  # discard, check out the next URL, retry once. Pool empty -> FAILURE.
+  # Notify-POST failure is logged + counted but doesn't fail the export
+  # (the blob is in R2, just won't be processed; incoming/* lifecycle
+  # cleans it up in 1hr).
+  class TraceExporter
+    SUCCESS = OpenTelemetry::SDK::Trace::Export::SUCCESS
+    FAILURE = OpenTelemetry::SDK::Trace::Export::FAILURE
+
+    PUT_HEADERS = {
+      "Content-Type"     => "application/json",
+      "Content-Encoding" => "gzip"
+    }.freeze
+
+    attr_reader :put_failure_count, :notify_failure_count, :pool_empty_count, :exception_count
+
+    def initialize(pool:, notify_url:, api_key:, project:, environment:,
+                   transport: nil, logger: nil)
+      @pool         = pool
+      @notify_url   = notify_url.to_s
+      @api_key      = api_key
+      @project      = project
+      @environment  = environment
+      @transport    = transport || HttpTransport.new
+      @logger       = logger || Logger.new($stderr, level: Logger::WARN)
+
+      @put_failure_count    = Concurrent::AtomicFixnum.new(0)
+      @notify_failure_count = Concurrent::AtomicFixnum.new(0)
+      @pool_empty_count     = Concurrent::AtomicFixnum.new(0)
+      @exception_count      = Concurrent::AtomicFixnum.new(0)
+    end
+
+    def export(spans, timeout: nil)
+      grouped = spans.group_by(&:trace_id)
+      return SUCCESS if grouped.empty?
+
+      overall = SUCCESS
+      grouped.each do |trace_id, group|
+        result = ship(TraceBlob.build(trace_id: trace_id, spans: group))
+        overall = FAILURE if result == FAILURE
+      end
+      overall
+    rescue StandardError => e
+      @exception_count.increment
+      @logger.warn("[Flare::TraceExporter] export raised: #{e.class}: #{e.message}")
+      FAILURE
+    end
+
+    def force_flush(timeout: nil)
+      SUCCESS
+    end
+
+    def shutdown(timeout: nil)
+      SUCCESS
+    end
+
+    private
+
+    def ship(blob, retried: false)
+      return FAILURE if blob.nil?
+
+      entry = @pool.checkout
+      if entry.nil?
+        @pool_empty_count.increment
+        return FAILURE
+      end
+
+      body = gzip(JSON.generate(blob.to_h))
+      response = @transport.put(entry[:put_url], body, PUT_HEADERS)
+
+      case response.code
+      when "200", "204"
+        notify(entry[:key])
+        SUCCESS
+      when "403"
+        # Presigned URL probably expired; try once more with the next one.
+        retried ? record_put_failure(response) : ship(blob, retried: true)
+      else
+        record_put_failure(response)
+      end
+    end
+
+    def notify(key)
+      response = @transport.post(@notify_url, JSON.generate(key: key), notify_headers)
+      return if response.code == "202"
+
+      @notify_failure_count.increment
+      @logger.warn("[Flare::TraceExporter] notify failed: HTTP #{response.code}")
+    rescue StandardError => e
+      @notify_failure_count.increment
+      @logger.warn("[Flare::TraceExporter] notify exception: #{e.class}: #{e.message}")
+    end
+
+    def notify_headers
+      {
+        "Content-Type"      => "application/json",
+        "Authorization"     => "Bearer #{@api_key}",
+        "Flare-Project"     => @project,
+        "Flare-Environment" => @environment
+      }
+    end
+
+    def record_put_failure(response)
+      @put_failure_count.increment
+      @logger.warn("[Flare::TraceExporter] PUT failed: HTTP #{response.code}")
+      FAILURE
+    end
+
+    def gzip(body)
+      io = StringIO.new
+      gz = Zlib::GzipWriter.new(io)
+      gz.write(body)
+      gz.close
+      io.string
+    end
+  end
+end

data/lib/flare/trace_health_reporter.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+require_relative "metric_key"
+
+module Flare
+  # Records client-side tracing health into MetricStorage so flare-web can
+  # warn when local buffering, URL exhaustion, or export errors reduce trace
+  # fidelity.
+  class TraceHealthReporter
+    NAMESPACE = "sdk"
+    SERVICE = "flare-ruby"
+    TARGET = "tracing"
+
+    def initialize(processor:, pool:, exporter:)
+      @processor = processor
+      @pool = pool
+      @exporter = exporter
+      @last = {}
+      @mutex = Mutex.new
+    end
+
+    def record(storage, bucket: Time.now.utc)
+      @mutex.synchronize do
+        record_counter(storage, bucket, "dropped_spans", @processor.dropped_count.value)
+        record_counter(storage, bucket, "export_failures", @processor.failed_export_count.value)
+        record_counter(storage, bucket, "processor_exceptions", @processor.exception_count.value)
+
+        record_counter(storage, bucket, "upload_url_pool_empty", @pool.empty_count.value)
+        record_counter(storage, bucket, "upload_url_expired", @pool.expired_count.value)
+
+        record_counter(storage, bucket, "r2_put_failures", @exporter.put_failure_count.value)
+        record_counter(storage, bucket, "notify_failures", @exporter.notify_failure_count.value)
+        record_counter(storage, bucket, "trace_pool_empty", @exporter.pool_empty_count.value)
+        record_counter(storage, bucket, "trace_export_exceptions", @exporter.exception_count.value)
+
+        buffer_size = @processor.buffer_size
+        buffer_high_watermark = @processor.buffer_high_watermark.value
+        record_gauge(storage, bucket, "buffer_size", buffer_size)
+        record_gauge(storage, bucket, "buffer_high_watermark", buffer_high_watermark)
+        record_gauge(storage, bucket, "buffer_limit", @processor.max_queue) if buffer_size.positive? || buffer_high_watermark.positive?
+        @processor.reset_buffer_high_watermark
+      end
+    end
+
+    private
+
+    def record_counter(storage, bucket, operation, current)
+      previous = @last.fetch(operation, 0)
+      @last[operation] = current
+      delta = current - previous
+      return unless delta.positive?
+
+      storage.add(key(bucket, operation), count: delta, sum_ms: 0, error_count: 0)
+    end
+
+    def record_gauge(storage, bucket, operation, value)
+      storage.add(key(bucket, operation), count: 1, sum_ms: value, error_count: 0)
+    end
+
+    def key(bucket, operation)
+      MetricKey.new(
+        bucket: bucket_time(bucket),
+        namespace: NAMESPACE,
+        service: SERVICE,
+        target: TARGET,
+        operation: operation
+      )
+    end
+
+    def bucket_time(time)
+      Time.utc(time.year, time.month, time.day, time.hour, time.min, 0)
+    end
+  end
+end