sitegoose 0.3.3

data/lib/sitegoose/queries.rb

@@ -0,0 +1,399 @@
+# frozen_string_literal: true
+
+# Database queries (sdk/protocols/queries.md), part of Server speed: per
+# minute, request method and route, database and query, how many ran, how
+# many failed, their total and slowest time and a count per time bucket, sent
+# to POST /events/server-queries. Every value in a query is replaced by "?"
+# before it is counted (QueryText); bound parameters are never read.
+#
+# Caught through each library's public hook, one layer per query:
+#   Rails (Active Record): the sql.active_record notification, automatic.
+#     SCHEMA and cached queries are left out.
+#   Sequel: SiteGoose.watch_database(DB), one line. A Sequel query
+#     run on Active Record's connection is counted once, as Sequel's.
+#   Mongo (the mongo gem, Mongoid): a global command subscriber, automatic.
+#
+# A query is added to the request (or job) it ran in, and the request's
+# queries go into the minute's rows when it finishes, under the method and
+# route Server speed records for it. A query outside any request or job is
+# "JOB" "(background)". Rides on the performance module: with Server speed
+# off nothing is recorded; queries: false keeps Server speed without them.
+
+module SiteGoose
+  class Queries
+    FLUSH_EVERY = Performance::FLUSH_EVERY
+    MAX_ROWS = 500
+    # Distinct statements one request keeps; past it new ones are "(other)".
+    MAX_PER_REQUEST = 50
+    # Keys in a minute: past MAX_KEYS new statements are "(other)", past
+    # HARD_MAX_KEYS nothing new is kept.
+    MAX_KEYS = 200
+    HARD_MAX_KEYS = 300
+    # Minutes kept unsent (a platform that can't send flushes by hand).
+    MAX_MINUTES = 5
+    OTHER = QueryText::OTHER_STATEMENT
+    BACKGROUND = ["JOB", "(background)"].freeze
+    # The request's own queries, fiber-local beside Context's fields: Context.with
+    # starts each request or job with none.
+    REQUEST = Context::QUERIES
+    # Set while an outer layer times a query, so an inner one leaves it.
+    TIMING = :sitegoose_query_timing
+    CACHE_MAX = 2000
+    CACHE_TEXT = 4000
+
+    @open = []
+    @lock = Mutex.new
+    @cache = {}
+    @hooked = {}
+
+    class << self
+      # The platform's Queries that hooks report to: the newest still open.
+      def active
+        @lock.synchronize { @open.last }
+      end
+
+      def opened(queries)
+        @lock.synchronize { @open << queries }
+      end
+
+      def closed(queries)
+        @lock.synchronize { @open.delete(queries) }
+      end
+
+      # The statement as stored ("" to leave it out); the same query text
+      # comes round again and again, so the answer is kept.
+      def statement(text, db)
+        return QueryText.normalize_statement(text, db) if !text.is_a?(String) || text.size > CACHE_TEXT
+
+        key = [db, text]
+        found = @lock.synchronize { @cache[key] }
+        return found if found
+
+        shaped = QueryText.normalize_statement(text, db).freeze
+        @lock.synchronize do
+          @cache.clear if @cache.size >= CACHE_MAX
+          @cache[key.map { |part| part.dup.freeze }] = shaped
+        end
+        shaped
+      end
+
+      def timing?
+        Thread.current[TIMING] ? true : false
+      end
+
+      # Runs the block with this thread's queries marked as timed by an outer layer.
+      def timing
+        saved = Thread.current[TIMING]
+        Thread.current[TIMING] = true
+        yield
+      ensure
+        Thread.current[TIMING] = saved
+      end
+
+      # One finished query, from any hook. db: one of QueryText::DATABASES.
+      def record(db, text, ms, failed: false)
+        active&.record(db, text, ms, failed: failed)
+      rescue StandardError
+        nil
+      end
+
+      def blank
+        { count: 0, errors: 0, sum_ms: 0.0, max_ms: 0.0, buckets: Array.new(Performance::BUCKET_EDGES_MS.size + 1, 0) }
+      end
+
+      def merge(into, from)
+        into[:count] += from[:count]
+        into[:errors] += from[:errors]
+        into[:sum_ms] += from[:sum_ms]
+        into[:max_ms] = from[:max_ms] if from[:max_ms] > into[:max_ms]
+        from[:buckets].each_with_index { |n, i| into[:buckets][i] += n }
+        into
+      end
+
+      def add(into, ms, failed)
+        into[:count] += 1
+        into[:errors] += 1 if failed
+        into[:sum_ms] += ms
+        into[:max_ms] = ms if ms > into[:max_ms]
+        into[:buckets][Performance.bucket_of(ms)] += 1
+        into
+      end
+
+      # ---- Hooks: installed once per process, for every platform ----
+
+      def install
+        hook_active_record
+        hook_mongo
+      end
+
+      # Active Record's sql.active_record notification (Rails, or Active
+      # Record on its own). Monotonic times where Rails has them.
+      def hook_active_record
+        return if @hooked[:active_record] || !defined?(::ActiveSupport::Notifications)
+
+        @hooked[:active_record] = true
+        handler = lambda do |_name, started, finished, _id, payload|
+          Queries.active_record(started, finished, payload)
+        end
+        if ::ActiveSupport::Notifications.respond_to?(:monotonic_subscribe)
+          ::ActiveSupport::Notifications.monotonic_subscribe("sql.active_record", &handler)
+        else
+          ::ActiveSupport::Notifications.subscribe("sql.active_record", &handler)
+        end
+      rescue StandardError
+        nil
+      end
+
+      def active_record(started, finished, payload)
+        return if !payload.is_a?(Hash) || timing? || active.nil?
+
+        name = payload[:name].to_s
+        return if name == "SCHEMA" || name == "CACHE" || payload[:cached]
+
+        failed = payload[:exception] || payload[:exception_object] ? true : false
+        record(active_record_db(payload[:connection]), payload[:sql], (finished - started).to_f * 1000, failed: failed)
+      rescue StandardError
+        nil
+      end
+
+      # The connection's adapter, as a database name.
+      def active_record_db(connection)
+        name = connection.respond_to?(:adapter_name) ? connection.adapter_name.to_s.downcase : ""
+        case name
+        when /postg/ then "postgresql"
+        when "mysql", "mysql2", "trilogy" then "mysql"
+        when /sqlite/ then "sqlite"
+        when /sqlserver|mssql/ then "mssql"
+        when /oracle/ then "oracle"
+        else "other"
+        end
+      end
+
+      # The mongo gem's global command monitoring (Mongoid uses it too).
+      def hook_mongo
+        return if @hooked[:mongo] || !defined?(::Mongo::Monitoring::Global)
+
+        @hooked[:mongo] = true
+        ::Mongo::Monitoring::Global.subscribe(::Mongo::Monitoring::COMMAND, MongoSubscriber.new)
+      rescue StandardError
+        nil
+      end
+
+      def sequel_db(database)
+        case database.respond_to?(:database_type) ? database.database_type.to_s : ""
+        when "postgres" then "postgresql"
+        when "mysql" then "mysql"
+        when "sqlite" then "sqlite"
+        when "mssql" then "mssql"
+        when "oracle" then "oracle"
+        else "other"
+        end
+      end
+
+      # Times every query a Sequel::Database runs.
+      def watch_sequel(database)
+        database.instance_variable_set(:@__sitegoose_db, sequel_db(database))
+        database.extend(SequelHook) unless database.singleton_class.include?(SequelHook)
+        database
+      end
+    end
+
+    # Extends one Sequel::Database (Sequel's own extensions wrap the same
+    # public method); nothing else of Sequel is changed.
+    module SequelHook
+      def log_connection_yield(sql, conn, args = nil)
+        return super if Queries.timing? || Queries.active.nil?
+
+        started = SiteGoose.clock
+        failed = false
+        begin
+          Queries.timing { super }
+        rescue StandardError
+          failed = true
+          raise
+        ensure
+          Queries.record(@__sitegoose_db, sql, (SiteGoose.clock - started) * 1000, failed: failed)
+        end
+      end
+    end
+
+    # Mongo::Monitoring's command subscriber: the statement is built when a
+    # command starts, recorded when it ends.
+    class MongoSubscriber
+      CHATTER = %w[hello isMaster ismaster ping buildInfo saslStart saslContinue getMore endSessions killCursors
+                   getLastError].freeze
+      FILTER = { "find" => "filter", "aggregate" => "pipeline", "count" => "query", "distinct" => "query",
+                 "findAndModify" => "query" }.freeze
+      LISTS = { "delete" => "deletes", "update" => "updates" }.freeze
+      WAITING_MAX = 1000
+
+      def initialize
+        @waiting = {}
+        @lock = Mutex.new
+      end
+
+      # "find users {...}" for a command (a hash).
+      def self.text(name, command)
+        collection = command.is_a?(Hash) ? command[name] : nil
+        return name if collection.nil?
+
+        filter = if FILTER.key?(name) then command[FILTER[name]]
+                 elsif LISTS.key?(name)
+                   first = command[LISTS[name]]
+                   first = first.first if first.is_a?(Array)
+                   first.is_a?(Hash) ? (first["q"] || first[:q]) : nil
+                 end
+        text = "#{name} #{collection}"
+        filter.nil? ? text : "#{text} #{JSON.generate(QueryText.mongo_shape(filter))}"
+      rescue StandardError
+        name
+      end
+
+      def started(event)
+        name = event.command_name.to_s
+        return if CHATTER.include?(name) || Queries.active.nil?
+
+        text = MongoSubscriber.text(name, event.command)
+        @lock.synchronize do
+          @waiting.delete(@waiting.keys.first) while @waiting.size >= WAITING_MAX
+          @waiting[event.request_id] = text
+        end
+      rescue StandardError
+        nil
+      end
+
+      def succeeded(event)
+        finished(event, false)
+      end
+
+      def failed(event)
+        finished(event, true)
+      end
+
+      def finished(event, failed)
+        text = @lock.synchronize { @waiting.delete(event.request_id) }
+        Queries.record("mongodb", text, event.duration.to_f * 1000, failed: failed) if text
+      rescue StandardError
+        nil
+      end
+    end
+
+    attr_reader :rows
+
+    def initialize(core, server_name: nil)
+      @core = core
+      @server_name = SiteGoose.text(server_name || SiteGoose.hostname, 100)
+      @rows = {}
+      @lock = Mutex.new
+      @job = core.valid? ? Periodic.new(FLUSH_EVERY) { flush(everything: false); FLUSH_EVERY } : nil
+      Forking.register(self)
+      Queries.opened(self)
+      Queries.install
+    end
+
+    # In a forked worker the parent's rows are the parent's to send.
+    def after_fork_in_child
+      @lock = Mutex.new
+      @rows = {}
+    end
+
+    def on?
+      @core.on?("performance")
+    end
+
+    # One finished query: into its request's (or job's) queries, or, with
+    # none around it, straight into the minute's rows as background.
+    def record(db, text, ms, failed: false)
+      return unless on?
+
+      db = QueryText.database_of(db)
+      statement = Queries.statement(text, db)
+      return if statement.empty?
+
+      ms = ms.to_f
+      ms = 0.0 if ms.negative? || ms.nan?
+      if Context.request
+        mine = (Thread.current[REQUEST] ||= {})
+        key = [db, statement]
+        key = [db, OTHER] if !mine.key?(key) && mine.size >= MAX_PER_REQUEST
+        Queries.add(mine[key] ||= Queries.blank, ms, failed)
+      else
+        one = Queries.add(Queries.blank, ms, failed)
+        @lock.synchronize { add_row(Queries.minute, *BACKGROUND, db, statement, one) }
+      end
+    end
+
+    def self.minute
+      (Time.now.to_i / 60) * 60
+    end
+
+    # The request (or job) finished, and Server speed recorded it under
+    # method and route: its queries go into this minute's rows.
+    def finish(method, route)
+      mine = Thread.current[REQUEST]
+      Thread.current[REQUEST] = nil
+      return if mine.nil? || mine.empty? || !on?
+
+      minute = Queries.minute
+      @lock.synchronize do
+        mine.each { |(db, statement), agg| add_row(minute, method, route, db, statement, agg) }
+      end
+    end
+
+    # Under @lock.
+    def add_row(minute, method, route, db, statement, agg)
+      keys = @rows[minute]
+      unless keys
+        @rows.delete(@rows.keys.min) while @rows.size >= MAX_MINUTES
+        keys = @rows[minute] = {}
+      end
+      key = [method, route, db, statement]
+      key = [method, route, db, OTHER] if !keys.key?(key) && keys.size >= MAX_KEYS
+      return if !keys.key?(key) && keys.size >= HARD_MAX_KEYS
+
+      Queries.merge(keys[key] ||= Queries.blank, agg)
+    end
+    private :add_row
+
+    def flush(everything: true)
+      current = Queries.minute
+      ready = @lock.synchronize do
+        taken = @rows.select { |minute, _| everything || minute < current }
+        taken.each_key { |minute| @rows.delete(minute) }
+        taken
+      end
+      rows = ready.flat_map do |minute, keys|
+        at = Time.at(minute).utc.iso8601(3)
+        keys.map do |(method, route, db, statement), v|
+          { "minute" => at, "method" => method, "route" => route, "db" => db, "statement" => statement,
+            "count" => v[:count], "errors" => v[:errors], "sum_ms" => v[:sum_ms].round(1),
+            "max_ms" => v[:max_ms].round(1), "buckets" => v[:buckets] }
+        end
+      end
+      rows.each_slice(MAX_ROWS) do |batch|
+        @core.request("queries", "/events/server-queries", auth: :bearer,
+                      body: { "site_id" => @core.site_id, "server_name" => @server_name,
+                              "runtime" => RUNTIME, "rows" => batch })
+      end
+    end
+
+    def close
+      @job&.stop
+      Forking.unregister(self)
+      Queries.closed(self)
+      flush
+    end
+  end
+
+  # Times every query this Sequel database runs (Health -> Server speed ->
+  # Queries): SiteGoose.watch_database(DB). Rails' Active Record and
+  # the mongo gem are caught on their own.
+  def self.watch_database(database)
+    if defined?(::Sequel::Database) && database.is_a?(::Sequel::Database)
+      Queries.watch_sequel(database)
+    else
+      warn "[analytics-platform] watch_database: #{database.class} is not a Sequel::Database; Active Record and Mongo are timed on their own"
+    end
+    database
+  end
+end

data/lib/sitegoose/query_text.rb

@@ -0,0 +1,215 @@
+# frozen_string_literal: true
+
+require "json"
+
+# Database queries with their values taken out (sdk/protocols/queries.md):
+# the one rule every server package applies before a query leaves the
+# customer's server. A port of expresscollect/Analytics/queryText.js, which
+# is the reference; the tests run sdk/protocols/query-cases.json against it.
+# Lengths are code points, and "whitespace" is what JavaScript's \s and
+# trim() mean, so the two give the same text character for character.
+#
+#   SELECT * FROM users WHERE email = 'a@b.c' AND id IN (1, 2, 3)
+#   -> SELECT * FROM users WHERE email = ? AND id IN (?)
+
+module SiteGoose
+  module QueryText
+    MAX_STATEMENT = 1000
+    OTHER_STATEMENT = "(other)"
+    # The databases a row may name; anything else is "other".
+    DATABASES = %w[postgresql mysql mariadb sqlite mssql oracle mongodb other].freeze
+
+    # JavaScript's \s (and what its trim() removes).
+    SPACE = "\\t\\n\\v\\f\\r \\u00a0\\u1680\\u2000-\\u200a\\u2028\\u2029\\u202f\\u205f\\u3000\\ufeff"
+    SPACE_CHAR = /\A[#{SPACE}]\z/.freeze
+    TRIM = /\A[#{SPACE}]+|[#{SPACE}]+\z/.freeze
+    # A digit right after one of these is part of a name or a placeholder
+    # (table2, $1, @p1, :1), not a value. ASCII only, in every package.
+    WORD = /\A[A-Za-z0-9_$@:]\z/.freeze
+    DIGIT = /\A[0-9]\z/.freeze
+    PREFIX_LETTER = /\A[EeNnXxBb]\z/.freeze
+    NAME_CHAR = /\A[A-Za-z0-9_]\z/.freeze
+    NUMBER = /\A(?:0[xX][0-9a-fA-F]+|(?:[0-9]+\.?[0-9]*|\.[0-9]+)(?:[eE][+-]?[0-9]+)?)/.freeze
+    # The only characters NUMBER can match.
+    NUMBER_CHAR = /\A[0-9a-fA-FxX.eE+-]\z/.freeze
+    DOLLAR_TAG = /\A\$(?:[A-Za-z_][A-Za-z0-9_]*)?\$/.freeze
+    PLACEHOLDER = "(?:\\?|\\$[0-9]+|:[A-Za-z0-9_]+|@[A-Za-z0-9_]+)"
+    LIST = /\([#{SPACE}]*#{PLACEHOLDER}(?:[#{SPACE}]*,[#{SPACE}]*#{PLACEHOLDER})*[#{SPACE}]*\)/.freeze
+    ROWS = /\(\?\)(?:[#{SPACE}]*,[#{SPACE}]*\(\?\))+/.freeze
+    MONGO = /\A([^#{SPACE}]+)(?: ([^#{SPACE}]+))?(?: (.*))?\z/m.freeze
+    # A JavaScript object lists keys that are array indexes first, in order.
+    INDEX_KEY = /\A(?:0|[1-9][0-9]*)\z/.freeze
+
+    module_function
+
+    def trim(text)
+      text.gsub(TRIM, "")
+    end
+
+    def database_of(db)
+      name = db.nil? ? "" : SiteGoose.text(db).downcase
+      DATABASES.include?(name) ? name : "other"
+    end
+
+    # The statement to store, or "" when nothing is left of it.
+    def normalize_statement(text, db)
+      return "" unless text.is_a?(String)
+
+      text = SiteGoose.text(text)
+      return text if text == OTHER_STATEMENT
+
+      db = database_of(db)
+      cut(db == "mongodb" ? mongo_statement(text) : sql_shape(text, db))
+    end
+
+    def cut(text)
+      text.length > MAX_STATEMENT ? trim(text[0, MAX_STATEMENT]) : text
+    end
+
+    # The first index at or after from where chars holds seq (an array of
+    # characters), or nil.
+    def index_of(chars, seq, from)
+      last = chars.size - seq.size
+      i = from
+      while i <= last
+        return i if chars[i, seq.size] == seq
+
+        i += 1
+      end
+      nil
+    end
+
+    # The end of a quoted run starting at start (the opening quote): the
+    # index after the closing quote, or the text's length. A doubled quote
+    # and a backslash both escape.
+    def quote_end(chars, start, quote)
+      i = start + 1
+      while i < chars.size
+        c = chars[i]
+        if c == "\\"
+          i += 2
+        elsif c == quote
+          return i + 1 unless chars[i + 1] == quote
+
+          i += 2
+        else
+          i += 1
+        end
+      end
+      chars.size
+    end
+
+    NEWLINE = ["\n"].freeze
+    COMMENT_END = ["*", "/"].freeze
+
+    def sql_shape(sql, db)
+      mysql = %w[mysql mariadb].include?(db)
+      chars = sql.chars
+      out = []
+      space = -> { out << " " if !out.empty? && out[-1] != " " }
+      i = 0
+      while i < chars.size
+        c = chars[i]
+        nxt = chars[i + 1]
+        prev = i.positive? ? chars[i - 1] : ""
+        if (c == "-" && nxt == "-") || (c == "#" && mysql)
+          i = index_of(chars, NEWLINE, i) || chars.size
+          space.call
+        elsif c == "/" && nxt == "*"
+          ends = index_of(chars, COMMENT_END, i + 2)
+          i = ends ? ends + 2 : chars.size
+          space.call
+        elsif c == "'" || (c == '"' && mysql)
+          out.pop if out[-1]&.match?(PREFIX_LETTER) && (out.size == 1 || !out[-2].match?(NAME_CHAR))
+          i = quote_end(chars, i, c)
+          out << "?"
+        elsif c == '"' || c == "`"
+          ends = quote_end(chars, i, c)
+          out.concat(chars[i...ends])
+          i = ends
+        elsif c == "$" && !prev.match?(WORD) && (tag = chars[i, 64].join[DOLLAR_TAG])
+          tag_chars = tag.chars
+          ends = index_of(chars, tag_chars, i + tag_chars.size)
+          i = ends ? ends + tag_chars.size : chars.size
+          out << "?"
+        elsif (c.match?(DIGIT) || (c == "." && nxt.to_s.match?(DIGIT))) && !prev.match?(WORD)
+          run = +""
+          j = i
+          while j < chars.size && j < i + 400 && chars[j].match?(NUMBER_CHAR)
+            run << chars[j]
+            j += 1
+          end
+          i += run[NUMBER].length
+          out << "?"
+        elsif c.match?(SPACE_CHAR)
+          i += 1
+          space.call
+        else
+          out << c
+          i += 1
+        end
+      end
+      trim(out.join.gsub(LIST, "(?)").gsub(ROWS, "(?)"))
+    end
+
+    # A Mongo filter or pipeline's shape: keys kept, every value "?", and an
+    # array whose items all have the same shape kept as one.
+    def mongo_shape(value, depth = 0)
+      return "?" if depth >= 6
+
+      case value
+      when Array
+        items = value.map { |item| mongo_shape(item, depth + 1) }
+        items.map { |item| JSON.generate(item) }.uniq.size == 1 ? [items[0]] : items
+      when Hash
+        pairs = value.map { |key, item| [key.to_s, item] }
+        index, named = pairs.partition { |key, _| key.match?(INDEX_KEY) && key.to_i < 4_294_967_295 }
+        (index.sort_by { |key, _| key.to_i } + named).each_with_object({}) do |(key, item), out|
+          out[key] = mongo_shape(item, depth + 1)
+        end
+      else "?"
+      end
+    end
+
+    # Ruby's parser takes comments and raw control characters that
+    # JavaScript's JSON.parse refuses: text with either is not JSON here.
+    def strict_json?(text)
+      quoted = false
+      escaped = false
+      text.each_char do |c|
+        if quoted
+          if escaped then escaped = false
+          elsif c == "\\" then escaped = true
+          elsif c == '"' then quoted = false
+          elsif c.ord < 0x20 then return false
+          end
+        elsif c == '"' then quoted = true
+        elsif c == "/" then return false
+        end
+      end
+      true
+    end
+
+    # "find users {...}": the command, the collection, and the filter's shape.
+    def mongo_statement(text)
+      match = MONGO.match(trim(text))
+      return "" unless match
+
+      command, collection, rest = match.captures
+      # A "collection" with a brace, bracket or quote in it is the filter's start.
+      if collection&.match?(/[{}\[\]"']/)
+        rest = rest.nil? ? collection : "#{collection} #{rest}"
+        collection = nil
+      end
+      shape = ""
+      if rest && strict_json?(rest)
+        begin
+          shape = JSON.generate(mongo_shape(JSON.parse(rest, max_nesting: false)))
+        rescue JSON::ParserError, JSON::GeneratorError, EncodingError, TypeError
+          shape = ""
+        end
+      end
+      [command, collection, shape].reject { |part| part.nil? || part.empty? }.join(" ")
+    end
+  end
+end