sixty 0.1.0

data/lib/sixty/plans.rb

@@ -0,0 +1,258 @@
+# frozen_string_literal: true
+
+require 'json'
+
+require_relative 'sql'
+
+module Sixty
+  # A query plan, reduced to its shape.
+  #
+  # ── Why this is the signal worth having ───────────────────────────────────
+  #
+  # Everything else this agent measures is a symptom: rows went up, latency went
+  # up, a method makes more queries than it did. A plan change is the *cause* —
+  # the moment a query stops using an index is the moment it becomes the
+  # incident, and every other number only reflects it afterwards, usually days
+  # afterwards, once the table is large enough for the difference to show.
+  #
+  # It is also the finding a person can act on without understanding any of
+  # this. "This got slower" invites a shrug; "this stopped using
+  # index_orders_on_user_id" names the fix.
+  #
+  # ── The rules that make this safe to run in production ────────────────────
+  #
+  # 1. EXPLAIN, never EXPLAIN ANALYZE. ANALYZE *executes* the statement to
+  #    measure it — doubling the load of every explained query and, for anything
+  #    that writes, performing the write a second time.
+  # 2. GENERIC_PLAN, so no parameter is ever bound. Postgres 16 added it exactly
+  #    for this. It is what keeps the privacy boundary intact by construction
+  #    rather than by a promise to strip values afterwards.
+  # 3. Read-only statements only, refused before they reach the database.
+  # 4. Off the request path entirely. The Node agent issues its EXPLAIN after
+  #    the caller's promise has settled; Ruby's notification subscriber runs
+  #    *inside* the caller's stack with the connection still checked out, so
+  #    doing the same here would put a second round trip in front of a user.
+  #    Instead the statement is queued and the flush thread explains it later on
+  #    a connection of its own. The plan lands one window late, which is nothing
+  #    — a plan is a property of the statement and the schema, not of the call.
+  module Plans
+    MAX_PLANS = 500
+    MAX_PENDING = 100
+
+    # Only statements that read. Anything else is refused here rather than
+    # relying on GENERIC_PLAN being side-effect free.
+    READ_ONLY = /\A\s*(select|with)\b/i.freeze
+    # A statement already carrying its own EXPLAIN, or several statements at
+    # once, is not something to wrap in another EXPLAIN.
+    UNSAFE = /\A\s*explain\b|;\s*\S/i.freeze
+
+    STRUCTURAL = [
+      'Node Type', 'Join Type', 'Strategy', 'Relation Name',
+      'Index Name', 'Scan Direction', 'Parent Relationship'
+    ].freeze
+
+    MAX_DEPTH = 12
+    MAX_NODES = 120
+
+    class << self
+      def mutex
+        @mutex ||= Mutex.new
+      end
+
+      def plans
+        @plans ||= {}
+      end
+
+      def pending
+        @pending ||= {}
+      end
+
+      def get(key)
+        mutex.synchronize { plans[key] }
+      end
+
+      def set(key, plan)
+        mutex.synchronize do
+          # Bounded like every other per-operation cache here. A process that
+          # has seen five hundred distinct statements will not learn much from
+          # the next one.
+          return if plans.size >= MAX_PLANS || plans.key?(key)
+
+          plans[key] = plan
+        end
+      end
+
+      # Remember a statement to explain on the next flush.
+      #
+      # `attempted` is marked at enqueue rather than at success, so a statement
+      # the planner refuses — an older Postgres with no GENERIC_PLAN, an
+      # untypable parameter, a temp table that no longer exists — is tried once
+      # and then left alone instead of retried on every execution forever.
+      #
+      # ── Two refusals, both about values ──────────────────────────────────
+      #
+      # **MySQL gets no plans at all.** It has no equivalent of GENERIC_PLAN: a
+      # statement can only be explained with its parameters bound, so capturing
+      # a plan would mean *retaining* somebody's query values in order to
+      # compose a command out of them. The signal is worth a great deal — an
+      # index that stopped being used names a cause rather than a symptom — and
+      # it is still not worth holding customer data to get. `@sixty-sh/node`
+      # refuses this for the same reason, and MongoDB is refused a third time on
+      # the same grounds.
+      #
+      # **A statement that arrived with literals in it is refused too**, even on
+      # Postgres. `where id = $1` carries no data and is safe to hand back;
+      # `where id = 42` is what an application with prepared statements turned
+      # off produces, and explaining it would mean keeping the values in memory
+      # until the next flush. The lexer decides which is which, because the
+      # question is exactly "would normalization have removed anything".
+      def enqueue(text, key, dialect: :postgres)
+        return unless dialect == :postgres
+        return unless text.is_a?(String) && READ_ONLY.match?(text) && !UNSAFE.match?(text)
+        return unless Sql.value_free?(text)
+
+        mutex.synchronize do
+          return if attempted.key?(key) || pending.size >= MAX_PENDING || plans.size >= MAX_PLANS
+
+          attempted[key] = true
+          pending[key] = text
+        end
+      end
+
+      # Drain the queue, explaining each statement with the caller's runner.
+      #
+      # @param runner [#call] takes a SQL string, returns the parsed `QUERY PLAN`
+      #   value (an Array or Hash), or raises. Supplied by the ActiveRecord
+      #   instrumentation so this file never has to know what a connection is.
+      def capture_pending(runner)
+        queued = mutex.synchronize do
+          taken = pending
+          @pending = {}
+          taken
+        end
+
+        queued.each do |key, text|
+          explained = runner.call("explain (generic_plan, format json) #{text}")
+          shaped = shape(explained)
+          next unless shaped
+
+          set(key, shaped.merge(key: plan_key(shaped[:shape])))
+        rescue StandardError
+          # Swallowed, deliberately and completely. None of the ordinary reasons
+          # this fails is the application's problem, and an observability agent
+          # that turns a planning quirk into a runtime error has done far more
+          # harm than the signal is worth.
+          next
+        end
+      end
+
+      # Reduce an EXPLAIN (FORMAT JSON) result to a stable, comparable shape.
+      #
+      # A plan as Postgres emits it is mostly numbers — startup cost, total
+      # cost, row estimate, width, loops — and every one of those moves whenever
+      # the statistics move, which is after every autovacuum. A detector
+      # comparing plans literally would fire constantly and mean nothing. What
+      # matters is structural and changes rarely: this query used to use an
+      # index and now scans the table.
+      def shape(explain)
+        explain = JSON.parse(explain) if explain.is_a?(String)
+        root = explain.is_a?(Array) ? explain.dig(0, 'Plan') : explain&.dig('Plan')
+        return nil unless root
+
+        nodes = 0
+        scans = []
+
+        walk = lambda do |node, depth|
+          return nil if node.nil? || depth > MAX_DEPTH || nodes >= MAX_NODES
+
+          nodes += 1
+          out = {}
+          STRUCTURAL.each { |field| out[field] = node[field] unless node[field].nil? }
+
+          if node['Node Type'].is_a?(String) && node['Node Type'].include?('Scan')
+            relation = node['Relation Name']
+            index = node['Index Name']
+            if relation
+              scans << (index ? "#{node['Node Type']} #{relation} using #{index}" : "#{node['Node Type']} #{relation}")
+            end
+          end
+
+          # The presence of a filter is structural; its contents are not.
+          out['Filtered'] = true if node['Filter']
+          out['IndexCond'] = true if node['Index Cond']
+          out['HashCond'] = true if node['Hash Cond']
+
+          children = Array(node['Plans']).map { |child| walk.call(child, depth + 1) }.compact
+          out['Plans'] = children unless children.empty?
+          out
+        end
+
+        shaped = walk.call(root, 0)
+        return nil unless shaped
+
+        # Two lists, because they answer different questions. `scans` is every
+        # scan node, which is what a person reading a summary wants. `seqScans`
+        # is only the sequential ones — the reads with no index — which is what
+        # the detector reports as a defect.
+        { shape: shaped, summary: summarize(shaped), scans: scans, seqScans: sequential_scans(shaped) }
+      end
+
+      # A stable identity for a shape, so "did the plan change" is one string
+      # comparison. Key order is normalised on the way in: Postgres emits fields
+      # consistently today, but a shape whose identity depended on that would
+      # report a change the first time a minor release reordered them.
+      def plan_key(shape)
+        JSON.generate(canonical(shape))
+      end
+
+      def sequential_scans(shape)
+        found = []
+        walk = lambda do |node|
+          found << node['Relation Name'] if node['Node Type'] == 'Seq Scan' && node['Relation Name']
+          Array(node['Plans']).each { |child| walk.call(child) }
+        end
+        walk.call(shape) if shape
+        found
+      end
+
+      def reset!
+        mutex.synchronize do
+          @plans = {}
+          @pending = {}
+          @attempted = {}
+        end
+      end
+
+      private
+
+      def attempted
+        @attempted ||= {}
+      end
+
+      # A one-line description, for a feed card with no room for a tree. It
+      # reads outside-in — the top node is what the query *is*, the scans are
+      # what it costs — because "Aggregate over Seq Scan on orders" is the
+      # sentence a person would say out loud.
+      def summarize(shape)
+        top = shape['Node Type'] || 'Plan'
+        scans = []
+        collect = lambda do |node|
+          if node['Node Type'].is_a?(String) && node['Node Type'].include?('Scan') && node['Relation Name']
+            scans << "#{node['Node Type'].sub(' Scan', '')} on #{node['Relation Name']}"
+          end
+          Array(node['Plans']).each { |child| collect.call(child) }
+        end
+        collect.call(shape)
+        scans.empty? ? top : "#{top} over #{scans.uniq.join(', ')}"
+      end
+
+      def canonical(node)
+        case node
+        when Array then node.map { |n| canonical(n) }
+        when Hash then node.keys.sort.each_with_object({}) { |k, out| out[k] = canonical(node[k]) }
+        else node
+        end
+      end
+    end
+  end
+end

data/lib/sixty/railtie.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+require 'rails/railtie'
+require_relative 'instrument/rack'
+require_relative 'instrument/active_record'
+require_relative 'instrument/action_controller'
+
+module Sixty
+  # Rails wiring.
+  #
+  # The whole install is `gem 'sixty'` plus a key in the environment. Everything
+  # below happens on boot, in the order the framework allows it: the middleware
+  # goes on the stack, the query subscriber attaches once ActiveRecord is
+  # loaded, and controllers are wrapped through `on_load` so this file never
+  # forces a constant into existence that the application had not asked for.
+  #
+  # ── Where the middleware sits, and why it matters ─────────────────────────
+  #
+  # Second from the top, after `ActionDispatch::RequestId`, so the span covers
+  # essentially the whole request — including the time other middleware spends,
+  # which is where a surprising amount of a slow request lives (a session store
+  # doing a database read, a rate limiter calling Redis). Inserting it lower
+  # would make every one of those invisible and quietly attribute their time to
+  # the controller.
+  class Railtie < ::Rails::Railtie
+    config.sixty = ActiveSupport::OrderedOptions.new
+
+    initializer 'sixty.init', before: :load_config_initializers do |app|
+      options = app.config.sixty.to_h
+      state = Sixty.init(options)
+
+      if state
+        app.middleware.insert_after(::ActionDispatch::RequestId, Sixty::Instrument::Rack)
+        Sixty::Instrument::ActionController.install
+      end
+    end
+
+    # ActiveRecord is subscribed to on `:active_record` load rather than in the
+    # initializer above, because in a Rails app that has not touched a model yet
+    # ActiveRecord::Base is not loaded — and referencing it to attach a
+    # subscriber would eagerly load the ORM (and open a connection) in processes
+    # that never needed one.
+    initializer 'sixty.active_record' do
+      ActiveSupport.on_load(:active_record) do
+        Sixty::Instrument::ActiveRecord.install(Sixty.config) if Sixty.enabled?
+      end
+    end
+  end
+end

data/lib/sixty/shape.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+module Sixty
+  # Argument shape: what a query *is*, with none of what it is about.
+  #
+  # SECURITY BOUNDARY, and the document-database counterpart to Sixty::Sql.
+  #
+  # For SQL, identity is the statement with its literals stripped — a string
+  # with things taken out of it. That approach has no analogue here. A Mongo
+  # filter is a tree in which the values sit *beside* the keys, arbitrarily
+  # deep, so there is no text to lex and nothing to remove: the values are
+  # structurally interleaved with the only part we are allowed to keep.
+  #
+  # So this walk never removes anything. It emits keys, and a value has no path
+  # to the output at all — not a redaction that could miss a case, but a
+  # construction in which the unsafe half is unreachable. If you are tempted to
+  # add "just the value, when it is a small integer, for readability": don't.
+  # That is the change that turns a structural guarantee back into a filter with
+  # exceptions.
+  #
+  # The other job is bounding cardinality, exactly as SQL normalization does.
+  # Identity that moved with the data would make every query a new operation,
+  # and an operation with no history has nothing to drift against.
+  module Shape
+    # Bounds on the walk, so a pathological argument cannot produce an unbounded
+    # identity. Both sit far above anything a real query reaches, and the depth
+    # cap is also what makes a cyclic structure terminate.
+    MAX_DEPTH = 6
+    MAX_KEYS = 24
+
+    # How many stages of an ordered structure — an aggregation pipeline — count.
+    MAX_STAGES = 12
+
+    module_function
+
+    # The shape of one argument tree: `filter{createdAt{$gte},status}`.
+    #
+    # Keys are sorted, because two call sites writing the same query with the
+    # keys in a different order are the same query, and insertion order would
+    # make them two operations that never accumulate a shared history.
+    #
+    # Only a Hash or an Array is structure. Everything else is a *value*,
+    # however many attributes it happens to carry — and that is not a nicety:
+    # a BSON::ObjectId, a Time, a BSON::Regexp or any application object has
+    # internals that are not the query's structure, and walking them would put
+    # meaningless keys into the identity of every query that filters by `_id`,
+    # which is most of them.
+    def shape_of(value, depth = 0)
+      return '' if depth > MAX_DEPTH
+
+      case value
+      when Array
+        # An array contributes the shape of its first element, never its length.
+        # `{_id: {'$in' => [...]}}` with three ids and with three thousand is one
+        # operation — the same bargain SQL normalization strikes when it folds
+        # `in (?, ?, ?)` down to `in (?)`.
+        value.empty? ? '' : shape_of(value.first, depth + 1)
+      when Hash
+        keys = value.keys.sort_by(&:to_s).first(MAX_KEYS)
+        return '' if keys.empty?
+
+        keys.map do |key|
+          inner = shape_of(value[key], depth + 1)
+          inner.empty? ? key.to_s : "#{key}{#{inner}}"
+        end.join(',')
+      else
+        ''
+      end
+    end
+
+    # The shape of an *ordered* structure, where every element matters.
+    #
+    # An aggregation pipeline is a list whose entries are deliberately different
+    # from one another — `[{$match}, {$group}, {$sort}]` — so it is structure,
+    # not data, and taking only the first element the way `shape_of` does would
+    # collapse every pipeline in the application to the shape of whatever it
+    # happened to start with. `$match → $group` and `$match → $lookup → $unwind`
+    # would be one operation, which is the difference between a working query
+    # and the N+1 that replaced it.
+    def shape_of_sequence(values)
+      return '' unless values.is_a?(Array) && !values.empty?
+
+      stages = values.first(MAX_STAGES).map do |stage|
+        inner = shape_of(stage, 1)
+        inner.empty? ? '[]' : "[#{inner}]"
+      end
+      stages << '[…]' if values.length > MAX_STAGES
+      stages.join
+    end
+  end
+end

data/lib/sixty/sketch.rb

@@ -0,0 +1,211 @@
+# frozen_string_literal: true
+
+require 'base64'
+
+module Sixty
+  # DDSketch — quantile sketch with *relative* error guarantees.
+  #
+  # A byte-for-byte port of packages/core/src/sketch.js. The format is the
+  # contract between every agent and the collector: the collector decodes these
+  # blobs with the JavaScript implementation, merges them across releases, and
+  # computes the percentiles a finding is made of. A Ruby sketch that decoded
+  # into slightly different buckets would not error anywhere — it would produce
+  # a p95 that quietly disagrees with the one a Node service in the same org
+  # reports, which is the kind of wrong this project exists to avoid.
+  #
+  # So the header layout, the varint encoding and the bucket indexing are
+  # reproduced exactly rather than reimplemented idiomatically, and
+  # test/sketch_test.rb checks the bytes against fixtures generated by the
+  # JavaScript writer.
+  #
+  # Why a sketch at all: latency is heavy-tailed and multimodal. A mean moves
+  # too little to detect a real p95 regression, and merges of DDSketches are
+  # lossless — which is what lets the collector store one sketch per 5-minute
+  # bucket and union arbitrary windows without the numbers drifting.
+  class Sketch
+    MAGIC = 0xd5
+    VERSION = 1
+    MIN_VALUE = 1e-9
+    # 1 magic + 1 version + 5 doubles + 1 flag.
+    HEADER_BYTES = 43
+
+    attr_reader :alpha, :count, :sum, :zero_count, :buckets
+
+    # @param alpha [Float] relative accuracy, e.g. 0.01 for 1%
+    def initialize(alpha = 0.01)
+      @alpha = alpha
+      @gamma = (1 + alpha) / (1 - alpha)
+      @log_gamma = Math.log(@gamma)
+      @buckets = {}
+      @zero_count = 0
+      @count = 0
+      @sum = 0.0
+      @min = Float::INFINITY
+      @max = -Float::INFINITY
+    end
+
+    def add(value, count = 1)
+      value = value.to_f
+      return self if value.nan? || value.infinite? || count <= 0
+
+      value = 0.0 if value.negative?
+
+      @count += count
+      @sum += value * count
+      @min = value if value < @min
+      @max = value if value > @max
+
+      if value < MIN_VALUE
+        @zero_count += count
+        return self
+      end
+
+      i = index(value)
+      @buckets[i] = (@buckets[i] || 0) + count
+      self
+    end
+
+    # Lossless when both sketches share alpha. Mutates self.
+    def merge(other)
+      return self if other.nil? || other.count.zero?
+      raise ArgumentError, "cannot merge sketches with different alpha" if (other.alpha - @alpha).abs > 1e-12
+
+      other.buckets.each { |i, c| @buckets[i] = (@buckets[i] || 0) + c }
+      @zero_count += other.zero_count
+      @count += other.count
+      @sum += other.sum
+      @min = other.min if other.min < @min
+      @max = other.max if other.max > @max
+      self
+    end
+
+    def quantile(q)
+      return nil if @count.zero?
+      return @min if q <= 0
+      return @max if q >= 1
+
+      rank = q * (@count - 1)
+      return 0.0 if rank < @zero_count
+
+      acc = @zero_count
+      @buckets.keys.sort.each do |i|
+        acc += @buckets[i]
+        # Clamped to observed bounds: a bucket's nominal value can fall slightly
+        # outside [min, max], and a p99 above any observed value reads as a bug.
+        return [[value_at(i), @min].max, @max].min if acc > rank
+      end
+      @max
+    end
+
+    def mean
+      @count.zero? ? nil : @sum / @count
+    end
+
+    # Compact binary form: header + zigzag-varint delta-encoded buckets.
+    def to_binary
+      indices = @buckets.keys.sort
+      body = +''.b
+      prev = 0
+      indices.each do |i|
+        write_varint(body, zigzag(i - prev))
+        write_varint(body, @buckets[i].round)
+        prev = i
+      end
+
+      header = [MAGIC, VERSION].pack('C2')
+      # 'E' is an IEEE-754 double, little-endian — the same layout DataView
+      # writes with littleEndian = true.
+      header += [@alpha, @zero_count.to_f, @sum.to_f,
+                 @count.zero? ? 0.0 : @min.to_f,
+                 @count.zero? ? 0.0 : @max.to_f].pack('E5')
+      header += [indices.empty? ? 0 : 1].pack('C')
+      header + body
+    end
+
+    # Base64 of the binary form, which is how the agent ships it over JSON.
+    def to_base64
+      Base64.strict_encode64(to_binary)
+    end
+
+    # Only the tests decode; the agent is write-only. Kept here anyway because a
+    # codec whose two halves live in different languages is a codec nobody can
+    # check locally.
+    def self.from_binary(bytes)
+      bytes = bytes.b
+      return new if bytes.bytesize < HEADER_BYTES
+      raise ArgumentError, 'not a DDSketch buffer' unless bytes.getbyte(0) == MAGIC
+
+      version = bytes.getbyte(1)
+      raise ArgumentError, "unsupported sketch version #{version}" unless version == VERSION
+
+      alpha, zero_count, sum, min, max = bytes.byteslice(2, 40).unpack('E5')
+      sketch = new(alpha)
+      sketch.instance_variable_set(:@zero_count, zero_count)
+      sketch.instance_variable_set(:@sum, sum)
+
+      offset = HEADER_BYTES
+      total = zero_count
+      prev = 0
+      while offset < bytes.bytesize
+        delta, offset = read_varint(bytes, offset)
+        count, offset = read_varint(bytes, offset)
+        i = prev + unzigzag(delta)
+        sketch.buckets[i] = (sketch.buckets[i] || 0) + count
+        total += count
+        prev = i
+      end
+
+      sketch.instance_variable_set(:@count, total)
+      sketch.instance_variable_set(:@min, total.positive? ? min : Float::INFINITY)
+      sketch.instance_variable_set(:@max, total.positive? ? max : -Float::INFINITY)
+      sketch
+    end
+
+    protected
+
+    attr_reader :min, :max
+
+    private
+
+    def index(value)
+      (Math.log(value) / @log_gamma).ceil
+    end
+
+    def value_at(i)
+      (2 * (@gamma**i)) / (@gamma + 1)
+    end
+
+    def zigzag(n)
+      n >= 0 ? n * 2 : -n * 2 - 1
+    end
+
+    def write_varint(out, n)
+      n = [0, n.round].max
+      while n >= 0x80
+        out << ((n & 0x7f) | 0x80).chr
+        n /= 128
+      end
+      out << n.chr
+    end
+
+    def self.unzigzag(n)
+      n.even? ? n / 2 : -(n + 1) / 2
+    end
+
+    def self.read_varint(bytes, offset)
+      result = 0
+      shift = 1
+      loop do
+        b = bytes.getbyte(offset)
+        offset += 1
+        result += (b & 0x7f) * shift
+        break if (b & 0x80).zero?
+
+        shift *= 128
+      end
+      [result, offset]
+    end
+
+    private_class_method :unzigzag, :read_varint
+  end
+end