flare 0.4.2 → 0.5.0

data/lib/flare/store_statement.rb

@@ -0,0 +1,136 @@
+# frozen_string_literal: true
+
+require "json"
+require "set"
+
+require_relative "sql_mask"
+require_relative "utf8"
+
+module Flare
+  # The shape of a Redis or MongoDB statement with its values taken out. Keeps what
+  # groups calls together: the command, a key's first segment, a document's
+  # structure. Anything it can't read comes out as a placeholder.
+  module StoreStatement
+    PLACEHOLDER = SqlMask::PLACEHOLDER
+    REDIS_SYSTEM = /redis|valkey/i
+    MORE = "..."
+    MAX_REDIS_LINES = 10
+    MAX_REDIS_BYTES = 16_384
+    MAX_MONGO_BYTES = 65_536
+    MAX_MONGO_DEPTH = 16
+    MAX_MONGO_NODES = 5_000
+
+    KEY_NAMESPACE = /\A[A-Za-z][A-Za-z_-]{0,40}[\/:]/
+
+    # Commands whose first argument is a key.
+    KEY_COMMANDS = %w[
+      APPEND DECR DECRBY DEL EXISTS EXPIRE EXPIREAT GET GETDEL GETEX GETRANGE GETSET HDEL HEXISTS HGET HGETALL
+      HINCRBY HINCRBYFLOAT HKEYS HLEN HMGET HMSET HSCAN HSET HSETNX HSTRLEN HVALS INCR INCRBY INCRBYFLOAT LINDEX
+      LINSERT LLEN LMOVE LPOP LPOS LPUSH LPUSHX LRANGE LREM LSET LTRIM MGET MSET MSETNX PERSIST PEXPIRE PEXPIREAT
+      PFADD PFCOUNT PSETEX PTTL RENAME RENAMENX RPOP RPUSH RPUSHX SADD SCARD SDIFF SET SETEX SETNX SETRANGE SINTER
+      SISMEMBER SMEMBERS SMISMEMBER SPOP SRANDMEMBER SREM SSCAN STRLEN SUBSTR SUNION TOUCH TTL TYPE UNLINK XACK XADD
+      XDEL XLEN XRANGE XREVRANGE XTRIM ZADD ZCARD ZCOUNT ZINCRBY ZLEXCOUNT ZMSCORE ZPOPMAX ZPOPMIN ZRANGE
+      ZRANGEBYLEX ZRANGEBYSCORE ZRANK ZREM ZREMRANGEBYRANK ZREMRANGEBYSCORE ZREVRANGE ZREVRANGEBYSCORE ZREVRANK
+      ZSCAN ZSCORE
+    ].to_set.freeze
+
+    # Commands whose arguments aren't kept.
+    OTHER_COMMANDS = %w[
+      ACL ASKING AUTH BGSAVE BITCOUNT BLMOVE BLPOP BRPOP BZPOPMAX BZPOPMIN CLIENT CLUSTER COMMAND CONFIG COPY DBSIZE
+      DISCARD DUMP ECHO EVAL EVALSHA EVALSHA_RO EVAL_RO EXEC FCALL FLUSHALL FLUSHDB FUNCTION GEOADD GEODIST GEOHASH
+      GEOPOS GEOSEARCH GETBIT HELLO INFO KEYS LASTSAVE MEMORY MIGRATE MODULE MONITOR MOVE MULTI OBJECT PING PSUBSCRIBE
+      PUBLISH PUBSUB PUNSUBSCRIBE QUIT RANDOMKEY READONLY READWRITE RESET RESTORE ROLE SAVE SCAN SCRIPT SELECT SETBIT
+      SHUTDOWN SLAVEOF SLOWLOG SORT SPUBLISH SSUBSCRIBE SUBSCRIBE SUNSUBSCRIBE SWAPDB TIME UNSUBSCRIBE UNWATCH WAIT
+      WATCH XAUTOCLAIM XCLAIM XGROUP XINFO XPENDING XREAD XREADGROUP
+    ].to_set.freeze
+    MODULE_COMMAND = /\A[A-Z]+\.[A-Z]+\z/
+
+    COLLECTION_COMMANDS = %w[
+      find count distinct aggregate insert update delete findAndModify findandmodify getMore create drop listIndexes
+      createIndexes dropIndexes
+    ].to_set.freeze
+    COLLECTION_NAME = /\A[\w.-]{1,64}\z/
+    MONGO_KEY = /\A\$?[A-Za-z_][\w-]{0,63}\z/
+
+    class TooLarge < StandardError; end
+
+    module_function
+
+    def redis_system?(system)
+      REDIS_SYSTEM.match?(system.to_s)
+    end
+
+    # A key's leading word and separator, like views/? or session:?.
+    def masked_key(key)
+      "#{key[KEY_NAMESPACE]}#{PLACEHOLDER}"
+    end
+
+    # The instrumentation joins a pipeline's commands with line breaks and a
+    # command's arguments with spaces, so a line break can also sit inside a value.
+    # Several lines therefore keep only command names, and only lines that start
+    # with one count. A script can't be told from its arguments, so it's left out.
+    def redis(statement, pipelined: true)
+      text = UTF8.scrub(statement.to_s.byteslice(0, MAX_REDIS_BYTES))
+      lines = pipelined ? text.split("\n") : [text.split("\n", 2).first.to_s]
+
+      keep_key = lines.size == 1
+      shapes = lines.filter_map { |line| redis_line(line, keep_key) }.chunk_while { |a, b| a == b }.map(&:first)
+      return PLACEHOLDER if shapes.empty?
+
+      kept = shapes.first(MAX_REDIS_LINES)
+      kept << MORE if lines.include?(MORE) || shapes.size > MAX_REDIS_LINES
+      kept.join("\n")
+    end
+
+    def redis_line(line, keep_key)
+      command, key = line.split(" ", 3)
+      return unless command
+
+      command = command.upcase
+      key_command = KEY_COMMANDS.include?(command)
+      return unless key_command || OTHER_COMMANDS.include?(command) || MODULE_COMMAND.match?(command)
+      return command unless key
+
+      "#{command} #{keep_key && key_command ? masked_key(key) : PLACEHOLDER}"
+    end
+
+    # Keeps the structure of a command's JSON and masks every value. Only a leading
+    # command keeps its value, the collection name.
+    def mongo(statement)
+      text = statement.to_s
+      return PLACEHOLDER if text.empty? || text.bytesize > MAX_MONGO_BYTES
+
+      document = JSON.parse(text, max_nesting: MAX_MONGO_DEPTH)
+      return PLACEHOLDER unless document.is_a?(Hash)
+
+      budget = [MAX_MONGO_NODES]
+      masked = mongo_value(document, budget)
+      command, collection = document.first
+      if COLLECTION_COMMANDS.include?(command) && collection.is_a?(String) && COLLECTION_NAME.match?(collection)
+        masked[mongo_key(command)] = collection
+      end
+      JSON.generate(masked)
+    rescue JSON::ParserError, EncodingError, TooLarge, ArgumentError
+      PLACEHOLDER
+    end
+
+    def mongo_value(value, budget)
+      raise TooLarge if (budget[0] -= 1).negative?
+
+      case value
+      when Hash then value.each_with_object({}) { |(key, item), out| out[mongo_key(key)] = mongo_value(item, budget) }
+      when Array then value.map { |item| mongo_value(item, budget) }.uniq
+      else PLACEHOLDER
+      end
+    end
+
+    def mongo_key(key)
+      head, dot, = key.partition(".")
+      return PLACEHOLDER unless MONGO_KEY.match?(head)
+
+      dot.empty? ? head : "#{head}.#{PLACEHOLDER}"
+    end
+
+    private_class_method :redis_line, :mongo_value, :mongo_key
+  end
+end

data/lib/flare/trace_blob.rb

@@ -2,6 +2,9 @@
 
 require "time"
 
+require_relative "export_filter"
+require_relative "utf8"
+
 module Flare
   # Value object that turns a group of OTel span_data for a single trace
   # into the Flare-JSON wire format the server expects:
@@ -33,18 +36,19 @@ module Flare
     def initialize(trace_id:, spans:)
       @trace_id = trace_id
       @spans    = spans
+      @filter   = ExportFilter.current
     end
 
     def to_h
       root = find_root
-      {
+      UTF8.scrub({
         "trace_id"      => hexify(@trace_id),
         "trace_rule_id" => rule_id_from_spans,
         "root_name"     => root_name(root),
         "started_at"    => iso(root&.start_timestamp),
         "duration_ms"   => duration_ms(root),
         "spans"         => @spans.map { |s| span_to_h(s) }
-      }
+      })
     end
 
     private
@@ -84,17 +88,17 @@ module Flare
     end
 
     def root_name(root)
-      root&.name&.to_s&.slice(0, ROOT_NAME_LIMIT)
+      @filter.name(root&.name&.to_s)&.slice(0, ROOT_NAME_LIMIT)
     end
 
     def span_to_h(span)
       {
         "id"          => hexify(span.span_id),
         "parent_id"   => root?(span) ? nil : hexify(span.parent_span_id),
-        "name"        => span.name,
+        "name"        => @filter.name(span.name),
         "started_at"  => iso(span.start_timestamp),
         "duration_ms" => duration_ms(span),
-        "attributes"  => span.attributes || {}
+        "attributes"  => @filter.attributes(span.attributes)
       }
     end
 

data/lib/flare/trace_exporter.rb

@@ -91,17 +91,22 @@ module Flare
 
     private
 
-    def ship(blob, deadline:, retried: false)
+    # Errors are contained per trace so one bad trace or one transport
+    # error doesn't drop the rest of the batch.
+    def ship(blob, deadline:, body: nil, retried: false)
       return FAILURE if blob.nil?
       return TIMEOUT if deadline.expired?
 
+      # Encode before checkout so a trace that can't be encoded doesn't use
+      # up a URL, but skip the work when there's no URL to upload to.
+      body ||= encode(blob) unless @pool.empty?
       entry = @pool.checkout
       if entry.nil?
         @pool_empty_count.increment
         return FAILURE
       end
 
-      body = gzip(JSON.generate(blob.to_h))
+      body ||= encode(blob)
       response = transport_request(:put, entry[:put_url], body, PUT_HEADERS, timeout: deadline.remaining)
       return TIMEOUT if deadline.expired?
 
@@ -110,10 +115,16 @@ module Flare
         notify(entry[:key], deadline: deadline)
       when "403"
         # Presigned URL probably expired; try once more with the next one.
-        retried ? record_put_failure(response) : ship(blob, deadline: deadline, retried: true)
+        retried ? record_put_failure(response) : ship(blob, deadline: deadline, body: body, retried: true)
       else
         record_put_failure(response)
       end
+    rescue StandardError => e
+      return TIMEOUT if deadline.expired? || e.is_a?(HttpTransport::DeadlineExceeded)
+
+      @exception_count.increment
+      @logger.warn("[Flare::TraceExporter] trace export raised: #{e.class}: #{e.message}")
+      FAILURE
     end
 
     def notify(key, deadline:)
@@ -171,6 +182,10 @@ module Flare
       FAILURE
     end
 
+    def encode(blob)
+      gzip(JSON.generate(blob.to_h))
+    end
+
     def gzip(body)
       io = StringIO.new
       gz = Zlib::GzipWriter.new(io)

data/lib/flare/upload_url_pool.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require "concurrent/atomic/atomic_fixnum"
 require "concurrent/atomic/atomic_reference"
 
 module Flare
@@ -15,11 +16,10 @@ module Flare
   # Fork-safe: after_fork clears the pool so child processes don't reuse
   # parent URLs (each child polls its own copy from /api/rules anyway).
   class UploadUrlPool
-    attr_reader :checkouts, :empty_count, :expired_count
+    attr_reader :empty_count, :expired_count
 
     def initialize
       @entries_ref   = Concurrent::AtomicReference.new([].freeze)
-      @checkouts     = Concurrent::AtomicFixnum.new(0)
       @empty_count   = Concurrent::AtomicFixnum.new(0)
       @expired_count = Concurrent::AtomicFixnum.new(0)
     end
@@ -47,7 +47,6 @@ module Flare
           next # try the next one
         end
 
-        @checkouts.increment
         return candidate
       end
     end
@@ -64,18 +63,6 @@ module Flare
       @entries_ref.set([].freeze)
     end
 
-    # Drop URLs that have already passed their expires_at. Cheap; safe to
-    # call from RuleManager's scheduler in between polls.
-    def sweep
-      now = Time.now
-      current = @entries_ref.get
-      live    = current.reject { |e| expired?(e, now) }
-      return 0 if live.length == current.length
-
-      @entries_ref.set(live.freeze)
-      current.length - live.length
-    end
-
     # Call from Flare.after_fork. Parent's URLs aren't usable from the
     # child's point of view (each child should get its own from a fresh
     # /api/rules poll), so just drop them.

data/lib/flare/utf8.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Flare
+  # Span attributes carry raw request bytes (User-Agent, paths) and SQL, which
+  # can hold invalid UTF-8. JSON.generate raises on those, failing the whole
+  # export batch, so replace the bad bytes before serializing.
+  module UTF8
+    def self.scrub(value)
+      case value
+      when String
+        return value if value.ascii_only? || (value.valid_encoding? && value.encoding != Encoding::BINARY)
+
+        value.dup.force_encoding(Encoding::UTF_8).scrub
+      when Array then value.map { |v| scrub(v) }
+      when Hash then value.to_h { |k, v| [scrub(k), scrub(v)] }
+      else value
+      end
+    end
+  end
+end

data/lib/flare/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Flare
-  VERSION = "0.4.2"
+  VERSION = "0.5.0"
 end

data/lib/flare.rb

@@ -10,6 +10,7 @@ require_relative "flare/source_location"
 require_relative "flare/metric_key"
 require_relative "flare/metric_storage"
 require_relative "flare/metric_span_processor"
+require_relative "flare/redis_statement_processor"
 require_relative "flare/metric_flusher"
 require_relative "flare/backoff_policy"
 require_relative "flare/metric_submitter"
@@ -181,7 +182,11 @@ module Flare
     require "opentelemetry-instrumentation-rack"
     require "opentelemetry-instrumentation-net_http"
     require "opentelemetry-instrumentation-active_support"
-    require "opentelemetry/instrumentation/active_support/span_subscriber"
+    begin
+      # Internal OTel path; subscribe_to_notifications handles it moving.
+      require "opentelemetry/instrumentation/active_support/span_subscriber"
+    rescue LoadError
+    end
     require "opentelemetry-instrumentation-action_pack" if defined?(ActionController)
     require "opentelemetry-instrumentation-action_view" if defined?(ActionView)
     require "opentelemetry-instrumentation-active_job" if defined?(ActiveJob)
@@ -195,6 +200,9 @@ module Flare
     OpenTelemetry::SDK.configure do |c|
       c.service_name = service_name
 
+      # Reads Redis statements as spans start, so raw arguments don't reach a finished span.
+      c.add_span_processor(RedisStatementProcessor.new)
+
       # Spans: detailed trace data stored in SQLite
       if configuration.spans_enabled && exporter
         c.add_span_processor(span_processor)
@@ -208,11 +216,18 @@ module Flare
         "OpenTelemetry::Instrumentation::Rack" => {
           untraced_requests: ->(env) {
             request = Rack::Request.new(env)
-            return true if request.path.start_with?("/flare")
+            # Skip Flare's own dashboard and assets, not app routes like /flares.
+            path = request.path
+            return true if path == "/flare" || path.start_with?("/flare/", "/flare-assets/")
 
             configuration.ignore_request.call(request)
           }
         },
+        # The key is what groups Redis calls. RedisStatementProcessor reduces the
+        # statement to the command and the key's first segment.
+        "OpenTelemetry::Instrumentation::Redis" => {
+          db_statement: :include,
+        },
         # Name Sidekiq job spans after the worker class (e.g. "MyWorker
         # process") instead of the upstream default of the queue name
         # ("default process"), matching how ActiveJob spans are named.
@@ -410,6 +425,10 @@ module Flare
   }.freeze
 
   def subscribe_to_notifications
+    unless OpenTelemetry::Instrumentation::ActiveSupport.respond_to?(:subscribe)
+      return warn "[Flare] OpenTelemetry::Instrumentation::ActiveSupport.subscribe not found. SQL, cache, mailer and custom notification spans are disabled. Check your opentelemetry-instrumentation-active_support version."
+    end
+
     NOTIFICATION_TRANSFORMERS.each do |pattern, transformer|
       OpenTelemetry::Instrumentation::ActiveSupport.subscribe(tracer, pattern, transformer)
     rescue

data/public/flare-assets/flare.css

@@ -190,21 +190,6 @@ code, pre {
   opacity: 1;
 }
 
-.nav-item-count {
-  margin-left: auto;
-  font-size: 0.6875rem;
-  font-weight: 500;
-  color: var(--cb-text-muted);
-  background: var(--cb-bg-muted);
-  padding: 0.125rem 0.5rem;
-  border-radius: 10px;
-}
-
-.nav-item.active .nav-item-count {
-  background: var(--cb-text);
-  color: white;
-}
-
 /* Sidebar Footer */
 .sidebar-footer {
   padding: 0.75rem 0.625rem;
@@ -332,21 +317,6 @@ code, pre {
   box-shadow: none;
 }
 
-.card-header {
-  padding: 0.75rem 1rem;
-  border-bottom: 1px solid var(--cb-border);
-  display: flex;
-  align-items: center;
-  justify-content: space-between;
-  background: transparent;
-}
-
-.card-title {
-  font-size: 0.8125rem;
-  font-weight: 600;
-  color: var(--cb-text);
-}
-
 .card-body {
   padding: 0;
 }
@@ -369,6 +339,15 @@ code, pre {
   border-bottom: 1px solid var(--cb-border);
 }
 
+.sort-link {
+  color: inherit;
+}
+
+.sort-link:hover,
+.sort-link.active {
+  color: var(--cb-text);
+}
+
 .data-table td {
   padding: 0.5rem 0.75rem;
   border-bottom: 1px solid var(--cb-bg-muted);
@@ -400,11 +379,6 @@ code, pre {
   letter-spacing: 0.02em;
 }
 
-.badge-verb {
-  background: var(--cb-bg-muted);
-  color: var(--cb-text-secondary);
-}
-
 .badge-get { background: rgba(107,138,154,0.15); color: #4a7a8a; }
 .badge-post { background: rgba(43,122,62,0.15); color: #1d6b2f; }
 .badge-put, .badge-patch { background: rgba(196,163,90,0.15); color: #9a7d2e; }
@@ -419,15 +393,8 @@ code, pre {
 .badge-error { background: rgba(197,48,48,0.15); color: #a53030; }
 .badge-warning { background: rgba(196,163,90,0.15); color: #9a7d2e; }
 
-.badge-type {
-  font-family: var(--cb-font);
-  font-weight: 500;
-}
-
-.badge-request { background: rgba(107,138,154,0.15); color: #4a7a8a; }
 .badge-job { background: rgba(138,154,107,0.15); color: #5a7a3d; }
 
-.badge-clue { font-family: var(--cb-font-mono); font-size: 0.6875rem; }
 .badge-sql { background: rgba(107,138,154,0.15); color: #4a7a8a; }
 .badge-cache { background: rgba(43,122,62,0.15); color: #1d6b2f; }
 .badge-view { background: rgba(196,163,90,0.15); color: #9a7d2e; }
@@ -436,7 +403,6 @@ code, pre {
 .badge-exception { background: rgba(197,48,48,0.15); color: #a53030; }
 .badge-other { background: var(--cb-bg-muted); color: var(--cb-text-secondary); }
 .badge-redis { background: rgba(197,48,48,0.12); color: #a53030; }
-.badge-query { background: rgba(107,138,154,0.15); color: #4a7a8a; }
 
 /* --- Search / Filters --- */
 .filters {
@@ -1037,15 +1003,6 @@ code, pre {
 }
 
 /* --- Code Blocks --- */
-.code-path {
-  font-family: var(--cb-font-mono);
-  font-size: 0.8125rem;
-  color: var(--cb-text-secondary);
-  background: var(--cb-bg-subtle);
-  padding: 0.125rem 0.375rem;
-  border-radius: var(--cb-radius-sm);
-  display: inline-block;
-}
 
 pre {
   background: var(--cb-text);
@@ -1168,14 +1125,7 @@ pre {
   border: 1px solid rgba(197,48,48,0.2);
 }
 
-.alert-success {
-  background: rgba(43,122,62,0.1);
-  color: var(--cb-success);
-  border: 1px solid rgba(43,122,62,0.2);
-}
-
 /* --- Muted --- */
-.muted { color: var(--cb-text-muted); }
 
 /* --- Responsive --- */
 @media (max-width: 1024px) {
@@ -1186,7 +1136,7 @@ pre {
     padding: 1rem 0.75rem;
     justify-content: center;
   }
-  .logo-text, .nav-section-title, .nav-item span, .nav-item-count, .btn-clear span {
+  .logo-text, .nav-section-title, .nav-item span, .btn-clear span {
     display: none;
   }
   .nav-item {