sixty 0.1.0

data/lib/sixty/sql.rb

@@ -0,0 +1,441 @@
+# frozen_string_literal: true
+
+require 'set'
+
+module Sixty
+  # SQL and route normalization.
+  #
+  # SECURITY BOUNDARY. Everything in this file runs inside the customer's
+  # process, before any byte leaves their network. Raw SQL text contains PII in
+  # literals ('alice@example.com', SSNs, tokens). We never transmit raw SQL —
+  # only the literal-free shape. If you are tempted to add a "send the original
+  # for debugging" option: don't.
+  #
+  # Normalization also bounds cardinality, which is the other way systems like
+  # this die. `where id = 1` and `where id = 99` must collapse to one operation
+  # or the operations table grows with traffic instead of with the number of
+  # queries the application contains.
+  #
+  # Ported from packages/core/src/sql.js and kept deliberately close to it. A
+  # Rails app and a Node service in the same organization must reduce the same
+  # statement to the same shape, because the shape is what the collector hashes
+  # into an operation's identity — two spellings of the same query would split
+  # one operation's history in half at the exact moment somebody rewrites a
+  # service in the other language.
+  module Sql
+    module_function
+
+    # ActiveRecord issues the same statement text thousands of times per minute.
+    # Normalization is a per-character lexer, so the result is memoized against
+    # the raw text — bounded, because a caller with unbounded distinct SQL is
+    # exactly the case that would otherwise grow this hash forever.
+    MAX_CACHED = 1000
+    @cache = {}
+    @cache_mutex = Mutex.new
+
+    # @param sql [String]
+    # @param dialect [Symbol] :postgres or :mysql
+    def normalize_sql(sql, dialect = :postgres)
+      return '' unless sql.is_a?(String)
+
+      analyze(sql, dialect).first
+    end
+
+    # Did this statement arrive with its values already out of it?
+    #
+    # A statement written with bind parameters (`where id = $1`) carries no data;
+    # one written with literals (`where id = 42`) carries all of it. Only the
+    # first kind may be handed back to the database in an `EXPLAIN` — see
+    # plans.rb — and this is what decides which it is.
+    #
+    # It is answered by the lexer rather than by a second pattern, because the
+    # question is exactly "would normalization have removed anything", and the
+    # only code that can answer that without disagreeing with itself is the code
+    # that does the removing.
+    def value_free?(sql, dialect = :postgres)
+      return false unless sql.is_a?(String)
+
+      analyze(sql, dialect).last.zero?
+    end
+
+    # @return [Array(String, Integer)] the normalized text and how many literals
+    #   were taken out of it
+    def analyze(sql, dialect)
+      key = dialect == :mysql ? "mysql\x00#{sql}" : sql
+      cached = @cache[key]
+      return cached if cached
+
+      result = lex_with_stats(sql, dialect)
+      @cache_mutex.synchronize do
+        @cache.clear if @cache.size >= MAX_CACHED
+        @cache[key] = result
+      end
+      result
+    end
+
+    def lex(sql, dialect = :postgres)
+      lex_with_stats(sql, dialect).first
+    end
+
+    # Strip literals, collapse whitespace, fold IN-lists.
+    #
+    # Deliberately a lexer, not a parser: it must never raise on a dialect
+    # quirk, must be fast enough to run on every query, and its only job is
+    # removing things — an unparseable statement still gets its literals
+    # stripped, which is the property that matters.
+    #
+    # ── Why the dialect is a parameter and not a union of both rule sets ──────
+    #
+    # The two dialects disagree about a character rather than merely differing.
+    # `"alice@example.com"` is a *quoted identifier* in Postgres — schema, safe
+    # to keep — and in MySQL's default sql_mode the same bytes are a *string
+    # literal*, which is exactly the PII this file exists to remove. Reading
+    # MySQL with the Postgres rules would transmit it. Backticks are the mirror
+    # image: MySQL's identifier quote, and not a quote at all in Postgres.
+    #
+    # ActiveRecord knows which one it is talking to, so the adapter name decides
+    # (see instrument/active_record.rb) rather than a guess from the text.
+    def lex_with_stats(sql, dialect = :postgres)
+      mysql = dialect == :mysql
+      src = sql.chars
+      n = src.length
+      out = +''
+      i = 0
+      # Every literal removed, counted. A bind parameter is not a literal: it
+      # was already a placeholder when it arrived.
+      literals = 0
+
+      while i < n
+        c = src[i]
+
+        # --- line comment
+        if c == '-' && src[i + 1] == '-'
+          i += 1 while i < n && src[i] != "\n"
+          next
+        end
+
+        # --- MySQL line comment. `#` is not a comment introducer in Postgres.
+        if mysql && c == '#'
+          i += 1 while i < n && src[i] != "\n"
+          next
+        end
+
+        # --- block comment
+        if c == '/' && src[i + 1] == '*'
+          i += 2
+          i += 1 while i < n && !(src[i] == '*' && src[i + 1] == '/')
+          i += 2
+          next
+        end
+
+        # --- single-quoted string (SQL escape is '')
+        if c == "'"
+          i += 1
+          while i < n
+            # MySQL also honours backslash escapes by default, so `'it\'s'` does
+            # not end where a Postgres lexer thinks it does. Mis-finding the
+            # closing quote resumes lexing *inside* a literal, and the tail of
+            # somebody's data is then emitted as if it were SQL.
+            if mysql && src[i] == '\\'
+              i += 2
+              next
+            end
+            if src[i] == "'" && src[i + 1] == "'"
+              i += 2
+              next
+            end
+            if src[i] == "'"
+              i += 1
+              break
+            end
+            i += 1
+          end
+          out << '?'
+          literals += 1
+          next
+        end
+
+        # --- MySQL double-quoted string. Postgres reads these as identifiers
+        #     and keeps them; here they are data and must not survive. Checked
+        #     before the identifier branch below, which is the Postgres reading.
+        if mysql && c == '"'
+          i += 1
+          while i < n
+            if src[i] == '\\'
+              i += 2
+              next
+            end
+            if src[i] == '"' && src[i + 1] == '"'
+              i += 2
+              next
+            end
+            if src[i] == '"'
+              i += 1
+              break
+            end
+            i += 1
+          end
+          out << '?'
+          literals += 1
+          next
+        end
+
+        # --- MySQL backtick identifier: preserved, it is schema, not data
+        if mysql && c == '`'
+          out << c
+          i += 1
+          while i < n
+            out << src[i]
+            if src[i] == '`' && src[i + 1] != '`'
+              i += 1
+              break
+            end
+            if src[i] == '`' && src[i + 1] == '`'
+              out << src[i + 1]
+              i += 2
+              next
+            end
+            i += 1
+          end
+          next
+        end
+
+        # --- dollar-quoted string ($tag$ ... $tag$). Postgres only: `$` is a
+        #     legal identifier character in MySQL, where `a$b$c` is one name and
+        #     reading it as a quoted string would swallow the rest of the
+        #     statement.
+        if !mysql && c == '$'
+          rest = sql[i..]
+          if (m = /\A\$([A-Za-z_]\w*)?\$/.match(rest))
+            tag = m[0]
+            found = sql.index(tag, i + tag.length)
+            i = found.nil? ? n : found + tag.length
+            out << '?'
+            literals += 1
+            next
+          end
+          # $1, $2 — already placeholders. Normalized to one symbol so a
+          # difference in parameter *count* does not fragment the identity.
+          if (p = /\A\$\d+/.match(rest))
+            out << '?'
+            i += p[0].length
+            next
+          end
+        end
+
+        # --- double-quoted identifier: preserved, it is schema, not data
+        if c == '"'
+          out << c
+          i += 1
+          while i < n
+            out << src[i]
+            if src[i] == '"' && src[i + 1] != '"'
+              i += 1
+              break
+            end
+            if src[i] == '"' && src[i + 1] == '"'
+              out << src[i + 1]
+              i += 2
+              next
+            end
+            i += 1
+          end
+          next
+        end
+
+        # --- numeric literal (not part of an identifier like col2)
+        #
+        # `previous` is spelled out rather than written as `src[i - 1]`: at i = 0
+        # Ruby's negative index would hand back the *last* character of the
+        # statement, so a query starting with a digit would be judged by its own
+        # final byte.
+        previous = i.zero? ? ' ' : (src[i - 1] || ' ')
+        if c =~ /[0-9]/ && previous !~ /[A-Za-z_$."]/
+          while i < n && src[i] =~ /[0-9.eE+\-xa-fA-F]/
+            # stop at an operator that merely follows the number
+            break if src[i] =~ /[+\-]/ && src[i - 1] !~ /[eE]/
+
+            i += 1
+          end
+          out << '?'
+          literals += 1
+          next
+        end
+
+        # --- whitespace run
+        if c =~ /\s/
+          out << ' '
+          i += 1 while i < n && src[i] =~ /\s/
+          next
+        end
+
+        out << c
+        i += 1
+      end
+
+      normalized = out
+                   # fold IN (?, ?, ?) -> IN (?) so batch size does not
+                   # fragment identity
+                   .gsub(/\b(?:in|IN|In)\s*\(\s*\?(?:\s*,\s*\?)+\s*\)/) { |m| m[0, m.index('(')] + '(?)' }
+                   # fold multi-row VALUES (?),(?) -> VALUES (?)
+                   .gsub(/\bvalues\s*(\(\s*\?(?:\s*,\s*\?)*\s*\))(?:\s*,\s*\(\s*\?(?:\s*,\s*\?)*\s*\))+/i, 'values \1')
+                   .gsub(/\s+/, ' ')
+                   .gsub(/\s*([(),;])\s*/, '\1')
+                   .strip
+
+      [normalized, literals]
+    end
+
+    SQL_VERB = /\A\s*(select|insert|update|delete|with|begin|commit|rollback|create|alter|drop|truncate|copy|explain|set|show|savepoint|release|listen|notify)\b/i.freeze
+
+    @names = {}
+    @names_mutex = Mutex.new
+
+    # Short display name for a SQL operation: "select:orders", "insert:users".
+    # The collector derives its own from the normalized text — this is what the
+    # span carries so a trace is readable before it ever leaves the process.
+    #
+    # Memoized against the normalized statement, and that is not a
+    # micro-optimization: naming walks the statement looking for the relation it
+    # touches, which measured at fifteen microseconds *per query* — four times
+    # the cost of everything else the agent does per query put together. A name
+    # is a pure function of the text, and the same few hundred statements repeat
+    # forever, so it is computed once each.
+    def sql_operation_name(normalized)
+      cached = @names[normalized]
+      return cached if cached
+
+      name = compute_operation_name(normalized)
+      @names_mutex.synchronize do
+        @names.clear if @names.size >= MAX_CACHED
+        @names[normalized] = name
+      end
+      name
+    end
+
+    def compute_operation_name(normalized)
+      body, names = split_ctes(normalized)
+      verb_match = SQL_VERB.match(body) || SQL_VERB.match(normalized)
+      verb = verb_match ? verb_match[1].downcase : 'query'
+
+      # Resolved against the *main* statement, not the first common table
+      # expression: otherwise every `with d as (select ... from unnest(...))`
+      # collapses to the same label, several unrelated statements all reading
+      # `with:unnest` and indistinguishable in a list.
+      table = relation_for(verb, body, names)
+      table ||= relation_for(verb, normalized, names) || first_relation(normalized, names)
+
+      table ? "#{verb}:#{table}" : verb
+    end
+
+    # Which keyword introduces the relation that names the statement. INSERT is
+    # named by its target, not by the SELECT that feeds it.
+    BY_VERB = {
+      'insert' => [/\binto\s+/i],
+      'update' => [/\bupdate\s+/i],
+      'delete' => [/\bdelete\s+from\s+/i, /\bfrom\s+/i],
+      'select' => [/\bfrom\s+/i, /\bjoin\s+/i],
+      'with' => [/\bfrom\s+/i, /\bjoin\s+/i]
+    }.freeze
+    # The quote is optional and may be either dialect's. A MySQL statement
+    # reaches here as ``insert into `orders` (a) values(?)``, and a pattern that
+    # only knows the Postgres quote finds no relation in it — so every
+    # backticked statement collapses to the bare verb `insert`, and the feed
+    # becomes a list of verbs. The captured name excludes the quotes, so
+    # `orders` and `` `orders` `` produce one label rather than two spellings.
+    IDENT = /(["`]?[A-Za-z_][\w$]*["`]?\.)?["`]?([A-Za-z_][\w$]*)["`]?/.freeze
+
+    # Compiled once. `Regexp.new(lead.source + IDENT.source)` per call meant
+    # building and discarding a regular expression on every query in the
+    # application.
+    RELATION_PATTERNS = BY_VERB.transform_values do |leads|
+      leads.map { |lead| Regexp.new(lead.source + IDENT.source, Regexp::IGNORECASE) }
+    end.freeze
+
+    def relation_for(verb, sql, cte_names)
+      (RELATION_PATTERNS[verb] || RELATION_PATTERNS['select']).each do |re|
+        sql.scan(re) do
+          name = Regexp.last_match(2)
+          # A CTE alias names nothing the reader can go and look at.
+          return name if name && !cte_names.include?(name.downcase)
+        end
+      end
+      nil
+    end
+
+    # Any real relation the statement touches, CTE aliases excluded.
+    def first_relation(sql, cte_names)
+      sql.scan(/\b(?:from|into|update|join)\s+(["`]?[A-Za-z_][\w$]*["`]?\.)?["`]?([A-Za-z_][\w$]*)["`]?/i) do
+        name = Regexp.last_match(2)
+        return name if name && !cte_names.include?(name.downcase)
+      end
+      nil
+    end
+
+    # Split a statement into its CTE names and the statement that follows them.
+    # Paren-aware: a CTE body contains commas and parentheses, and a regex that
+    # ignores nesting stops in the wrong place.
+    def split_ctes(sql)
+      names = Set.new
+      return [sql, names] unless sql =~ /\A\s*with\b/i
+
+      i = (sql =~ /\bwith\b/i) + 4
+      loop do
+        name_match = /\A\s*(?:recursive\s+)?["`]?([A-Za-z_][\w$]*)["`]?/i.match(sql[i..] || '')
+        open = sql.index('(', i)
+        return [sql, names] if open.nil?
+
+        names << name_match[1].downcase if name_match
+
+        depth = 0
+        j = open
+        while j < sql.length
+          if sql[j] == '('
+            depth += 1
+          elsif sql[j] == ')'
+            depth -= 1
+            if depth.zero?
+              j += 1
+              break
+            end
+          end
+          j += 1
+        end
+        return [sql, names] unless depth.zero?
+
+        rest = sql[j..] || ''
+        comma = /\A\s*,/.match(rest)
+        return [rest.sub(/\A\s+/, ''), names] unless comma
+
+        i = j + comma[0].length
+      end
+    end
+
+    UUID = /\A[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\z/i.freeze
+
+    # Template an HTTP path so /users/42 and /users/43 are one operation.
+    # Used only when the framework does not hand us a route pattern; a real
+    # Rails route always wins over this heuristic.
+    def template_path(pathname)
+      return '/' unless pathname.is_a?(String)
+
+      pathname = pathname.split('?', 2).first.to_s
+
+      templated = pathname.split('/', -1).map do |seg|
+        next seg if seg.empty?
+        next ':id' if seg =~ /\A\d+\z/
+        next ':uuid' if seg =~ UUID
+        next ':hash' if seg =~ /\A[0-9a-f]{24,}\z/i
+        # A colon inside a path segment is almost never part of a route: routes
+        # are named with words. It is, reliably, a delimiter inside an id.
+        next ':id' if seg.include?(':') && !seg.start_with?(':')
+        # long, high-entropy, mixed-case segments are almost always ids/slugs
+        next ':id' if seg.length > 24 && seg =~ /\d/ && seg =~ /[A-Za-z]/
+
+        seg
+      end.join('/')
+
+      templated.empty? ? '/' : templated
+    end
+  end
+end

data/lib/sixty/stack.rb

@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+module Sixty
+  # Where in your code did this operation come from.
+  #
+  # A finding you cannot locate in your own repository is much less actionable —
+  # "select:orders returns 30,000 rows" is only useful once you know which file
+  # writes that query. In a Rails app that file is almost never the one that
+  # executes the statement: ActiveRecord does, through six frames of adapter,
+  # relation and log subscriber. So the frames kept here are the application's
+  # own, and the ORM's are skipped.
+  #
+  # ── Why this is affordable ────────────────────────────────────────────────
+  #
+  # Capturing a backtrace costs microseconds, which is unacceptable per query.
+  # But a call site is a property of the *operation*, not of the call: the same
+  # query is issued from the same place every time. So the stack is captured on
+  # the first sighting of a normalized statement and never again. The cost
+  # amortises to nothing over a process lifetime and is bounded by operation
+  # cardinality, which is already capped.
+  #
+  # ── Why several frames, not one ───────────────────────────────────────────
+  #
+  # Applications funnel queries through helpers — a scope, a query object, a
+  # concern. The top application frame is therefore that helper for every query,
+  # which is useless. Rather than guess which frame is "the real caller" with a
+  # heuristic that will often be wrong, the top few application frames are kept
+  # and shown. The developer recognises their own code immediately, and no guess
+  # has to be correct.
+  module Stack
+    MAX_FRAMES = 4
+    MAX_TRACKED = 2000 # matches the aggregator's operation cap
+
+    # Frames from Ruby itself, from installed gems, and from this agent are
+    # never the answer — the caller wants their own code. `<internal:` covers
+    # the frames Ruby's own prelude contributes.
+    NOT_USER_CODE = %r{
+      /gems/ | /ruby/\d | /rubygems/ | ^<internal: | /bundler/ | /packages/ruby/lib/sixty/
+    }x.freeze
+
+    class << self
+      attr_accessor :root
+
+      def seen
+        @seen ||= {}
+      end
+
+      def mutex
+        @mutex ||= Mutex.new
+      end
+
+      # @param key [String] the operation's normalized identity
+      # @return [Array<Hash>, nil] frames, once per key, nil every time after
+      def capture(key)
+        return nil if key.nil? || key.empty?
+
+        mutex.synchronize do
+          return nil if seen.key?(key) || seen.size >= MAX_TRACKED
+
+          seen[key] = true
+        end
+
+        frames = []
+        # Deep enough to get past ActiveRecord's adapter stack, not so deep that
+        # building the array becomes the expensive part.
+        caller_locations(2, 40).each do |location|
+          path = location.absolute_path || location.path
+          next if path.nil? || NOT_USER_CODE.match?(path)
+
+          frames << { file: relativise(path), line: location.lineno, fn: location.label.to_s[0, 80] }
+          break if frames.length >= MAX_FRAMES
+        end
+
+        frames.empty? ? nil : frames
+      end
+
+      # Repository-relative paths, so a frame is comparable across machines and
+      # can be turned into a link into the commit that produced it. An absolute
+      # deploy path is meaningless to a reader and discloses the layout of the
+      # host it was built on.
+      def relativise(path)
+        prefix = root
+        return path if prefix.nil? || prefix.empty?
+
+        prefix = "#{prefix}/" unless prefix.end_with?('/')
+        path.start_with?(prefix) ? path[prefix.length..] : path
+      end
+
+      # Exported for tests: forget what has been seen.
+      def reset!
+        mutex.synchronize { @seen = {} }
+      end
+    end
+  end
+end