sixty 0.1.0

data/lib/sixty/config.rb

@@ -0,0 +1,155 @@
+# frozen_string_literal: true
+
+module Sixty
+  # Configuration, under both names.
+  #
+  # The product is called sixty, so its variables are `SIXTY_*`. They used to be
+  # `DRIFT_*`, and that name is not ours to retire: it is set in other people's
+  # deployments — a Dockerfile, a Fly secret, a CI job someone configured once
+  # and has not thought about since. Renaming without reading the old name would
+  # not produce an error anybody could act on. It would produce an agent that
+  # starts cleanly, finds no key, and reports nothing, in exactly the silence
+  # this product exists to eliminate.
+  #
+  # Nothing here warns when the old name is used. A warning would fire on every
+  # boot of a correctly configured service that simply predates the rename,
+  # which teaches people to filter our log lines out.
+  class Config
+    ATTRIBUTES = %i[
+      api_key endpoint service environment release repo_url
+      flush_interval sample_rate slow_trace_ms capture_plans
+      instrument ignore_paths debug on_warn logger
+    ].freeze
+
+    attr_accessor(*ATTRIBUTES)
+
+    # Most PaaS providers set one of these without the user doing anything, and
+    # release attribution is what makes deploy-anchored detection possible — so
+    # the agent tries hard to find one before giving up. Without a release, two
+    # deploys are one undifferentiated stream and nothing can be compared
+    # against anything.
+    RELEASE_VARS = %w[
+      HEROKU_SLUG_COMMIT RENDER_GIT_COMMIT RAILWAY_GIT_COMMIT_SHA
+      GITHUB_SHA FLY_MACHINE_VERSION SOURCE_VERSION VERCEL_GIT_COMMIT_SHA
+    ].freeze
+
+    def initialize(options = {})
+      @api_key = options[:api_key] || env('API_KEY')
+      @endpoint = options[:endpoint] || env('ENDPOINT') || 'http://localhost:4319'
+      @service = options[:service] || env('SERVICE') || infer_service
+      @environment = options[:environment] || env('ENV') || rails_env || 'development'
+      @release = options[:release] || env('RELEASE') || detect_release || ''
+      @repo_url = options[:repo_url] || env('REPO_URL') || github_repo_url || ''
+      # Seconds when passed in code, milliseconds in the environment — because
+      # `SIXTY_FLUSH_MS` is the name every other agent in this repository reads,
+      # and a Ruby app and a Node app sharing a compose file must not need two
+      # spellings of the same knob.
+      @flush_interval = options[:flush_interval] || (env_number('FLUSH_MS', 15_000) / 1000.0)
+      @sample_rate = options[:sample_rate] || env_number('SAMPLE_RATE', 0.05)
+      @slow_trace_ms = options[:slow_trace_ms] || env_number('SLOW_TRACE_MS', 1000)
+      @capture_plans = options.fetch(:capture_plans, env('CAPTURE_PLANS') != '0')
+      # Which database clients to measure. nil means "decide from what is in the
+      # process" — see Sixty.install_database_instrumentation, which is where
+      # the ActiveRecord-versus-raw-driver choice is made and explained.
+      @instrument = options[:instrument] || env_list('INSTRUMENT')
+      # Health checks and asset requests are noise: they are not the
+      # application, and left in they dominate the operation list of every
+      # service behind a load balancer.
+      @ignore_paths = options[:ignore_paths] || [
+        %r{\A/assets/}, %r{\A/packs/}, %r{\A/rails/active_storage/},
+        %r{\A/health}, %r{\A/up\z}, /favicon\.ico\z/
+      ]
+      @debug = options.fetch(:debug, env('DEBUG') == '1')
+      @logger = options[:logger] || rails_logger
+      @on_warn = options[:on_warn] || default_warn
+    end
+
+    def active?
+      !api_key.to_s.empty?
+    end
+
+    def ignore?(path)
+      ignore_paths.any? { |pattern| pattern.match?(path) }
+    end
+
+    private
+
+    # Emptiness is checked per name rather than after choosing one. `SIXTY_X=`
+    # in a compose file is someone who added the variable and has not filled it
+    # in yet — if that shadowed a `DRIFT_X` that is actually set, adding the new
+    # name would break a working deployment, which is the one outcome this
+    # lookup exists to prevent.
+    def env(name)
+      ["SIXTY_#{name}", "DRIFT_#{name}"].each do |key|
+        value = ENV.fetch(key, nil)
+        return value if value && !value.empty?
+      end
+      nil
+    end
+
+    def env_number(name, fallback)
+      raw = env(name)
+      return fallback if raw.nil?
+
+      value = Float(raw, exception: false)
+      value.nil? ? fallback : value
+    end
+
+    # SIXTY_INSTRUMENT=pg,mongo — for the application that knows better than the
+    # default, usually because it uses ActiveRecord *and* a driver directly.
+    def env_list(name)
+      raw = env(name)
+      return nil if raw.nil?
+
+      raw.split(',').map { |part| part.strip.downcase.to_sym }.reject { |part| part.to_s.empty? }
+    end
+
+    def detect_release
+      RELEASE_VARS.each do |key|
+        value = ENV.fetch(key, nil)
+        return value if value && !value.empty?
+      end
+      nil
+    end
+
+    def github_repo_url
+      repo = ENV.fetch('GITHUB_REPOSITORY', nil)
+      repo && !repo.empty? ? "https://github.com/#{repo}" : nil
+    end
+
+    def infer_service
+      return rails_application_name if rails_application_name
+
+      File.basename(Dir.pwd)
+    rescue StandardError
+      'unknown-service'
+    end
+
+    def rails_application_name
+      return nil unless defined?(::Rails) && ::Rails.respond_to?(:application) && ::Rails.application
+
+      ::Rails.application.class.module_parent_name.to_s.gsub(/([a-z])([A-Z])/, '\1-\2').downcase
+    rescue StandardError
+      nil
+    end
+
+    def rails_env
+      defined?(::Rails) && ::Rails.respond_to?(:env) ? ::Rails.env.to_s : nil
+    end
+
+    def rails_logger
+      defined?(::Rails) && ::Rails.respond_to?(:logger) ? ::Rails.logger : nil
+    end
+
+    def default_warn
+      log = @logger
+      lambda do |message|
+        if log
+          log.warn(message)
+        else
+          warn(message)
+        end
+      end
+    end
+  end
+end

data/lib/sixty/exporter.rb

@@ -0,0 +1,144 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'net/http'
+require 'uri'
+
+module Sixty
+  # Transport. Ships aggregate windows and exemplar traces to the collector.
+  #
+  # Rules this obeys, because an observability agent that harms the host is
+  # worse than no agent at all:
+  #   - never block the request path (the flush runs on the agent's own thread)
+  #   - never grow without bound (the exemplar queue is capped and drops oldest)
+  #   - never keep the process alive (the thread is not joined at exit; the
+  #     final flush is bounded by its own timeout)
+  #   - never raise into user code (every failure is swallowed and counted)
+  class Exporter
+    MAX_QUEUED_EXEMPLARS = 200
+    TIMEOUT_SECONDS = 10
+
+    attr_reader :failures
+
+    def initialize(endpoint:, api_key:, service:, environment:, release:, repo_url: '',
+                   path: '/v1/ingest', on_warn: ->(_msg) {})
+      @endpoint = endpoint.to_s.sub(%r{/\z}, '')
+      # The collector's receive path, appended to the endpoint — and overridable
+      # because not every endpoint is the collector. A service reporting through
+      # a proxy mounted on its own origin is given the exact URL that proxy
+      # lives at, and appending anything to it produces a route the application
+      # never registered. That failure is unusually quiet: the request 404s, the
+      # agent swallows it like any other transport error, and the only symptom
+      # is an absence of data.
+      @path = path
+      @api_key = api_key
+      @resource = {
+        service: service,
+        environment: environment,
+        release: release,
+        repoUrl: repo_url.to_s
+      }
+      @on_warn = on_warn
+      @exemplars = []
+      @mutex = Mutex.new
+      @failures = 0
+      @warned_at = 0.0
+      @last_attempt = nil
+    end
+
+    def queue_exemplar(trace)
+      @mutex.synchronize do
+        @exemplars.shift if @exemplars.length >= MAX_QUEUED_EXEMPLARS
+        @exemplars << trace
+      end
+    end
+
+    # @param metrics [Hash, nil] a drained aggregator window
+    # @param sections [Hash] payload sections this exporter knows nothing about,
+    #   so a future adapter can add one without teaching the transport about it
+    # @param timeout [Numeric] socket timeout; shortened for the exit flush,
+    #   where the process is trying to stop and a dead collector must not be
+    #   able to hold it open
+    def flush(metrics, sections: {}, timeout: TIMEOUT_SECONDS)
+      exemplars = @mutex.synchronize do
+        taken = @exemplars
+        @exemplars = []
+        taken
+      end
+
+      has_sections = sections.any? { |_k, v| !v.nil? }
+      return if metrics.nil? && exemplars.empty? && !has_sections
+
+      # A collector that is down stops being contacted for a while, rather than
+      # being dialled on every interval for as long as the outage lasts. The
+      # window that would have been sent is dropped here, deliberately: an agent
+      # that retained data through an outage would be an agent whose memory
+      # footprint is decided by somebody else's uptime.
+      return if backing_off?
+
+      body = JSON.generate(
+        {
+          resource: @resource,
+          metrics: metrics,
+          exemplars: exemplars,
+          sentAt: (Time.now.to_f * 1000).round
+        }.merge(sections.reject { |_k, v| v.nil? })
+      )
+
+      post(body, timeout)
+    end
+
+    private
+
+    # Exponential, capped, and counted from the first failure — so a collector
+    # that is briefly unreachable is retried on the next interval, and one that
+    # has been down for an hour is dialled every five minutes instead of every
+    # fifteen seconds.
+    MAX_BACKOFF_SECONDS = 300
+
+    def backing_off?
+      return false if @failures < 2 || @last_attempt.nil?
+
+      delay = [2**[@failures, 8].min, MAX_BACKOFF_SECONDS].min
+      (Process.clock_gettime(Process::CLOCK_MONOTONIC) - @last_attempt) < delay
+    end
+
+    def post(body, timeout)
+      @last_attempt = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      uri = URI.parse("#{@endpoint}#{@path}")
+      http = Net::HTTP.new(uri.host, uri.port)
+      http.use_ssl = uri.scheme == 'https'
+      http.open_timeout = timeout
+      http.read_timeout = timeout
+      http.write_timeout = timeout if http.respond_to?(:write_timeout=)
+
+      request = Net::HTTP::Post.new(uri.request_uri)
+      request['content-type'] = 'application/json'
+      # Omitted rather than sent empty when there is no key, so a
+      # misconfiguration reads as "no credential" at the collector rather than
+      # as an empty one it has to decide what to do with.
+      request['authorization'] = "Bearer #{@api_key}" if @api_key && !@api_key.empty?
+      request.body = body
+
+      response = http.request(request)
+      if response.code.to_i >= 300
+        warn_once("ingest rejected payload: #{response.code} #{response.body.to_s[0, 200]}")
+      else
+        @failures = 0
+      end
+    rescue StandardError => e
+      warn_once("ingest unreachable: #{e.message}")
+    end
+
+    def warn_once(message)
+      @failures += 1
+      # The first failure is logged, then at most one a minute. An agent that
+      # spams stderr through a collector outage is its own incident.
+      now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      return unless @failures == 1 || now - @warned_at > 60
+
+      @warned_at = now
+      @on_warn.call("sixty: #{message}#{@failures > 1 ? " (#{@failures} failures)" : ''}")
+    end
+  end
+end

data/lib/sixty/instrument/action_controller.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+require_relative '../tracer'
+
+module Sixty
+  module Instrument
+    # Controller actions, as operations of their own.
+    #
+    # The Rack span says a request took 900ms. The query spans say the database
+    # took 40ms. Without a span in between, the remaining 860ms lands on the
+    # request as self time — true, but not actionable: "GET /orders got slower"
+    # names an endpoint, not a change. `OrdersController#index` names the method
+    # somebody edited.
+    #
+    # It is also the layer that makes fanout readable. `db_calls` is credited to
+    # every ancestor, so an N+1 introduced in a controller shows up here as the
+    # call count changing, which is the sentence a person can act on: "this
+    # action went from 3 queries to 41".
+    #
+    # `process_action` is the hook rather than an `around_action` for one
+    # reason: an around_action can be skipped, reordered, or short-circuited by
+    # another filter — including a `before_action` that renders and halts, which
+    # is exactly the request whose measurement matters most.
+    module ActionController
+      def process_action(*args)
+        return super unless Sixty.enabled?
+
+        Sixty.trace("#{self.class.name}##{action_name}") { super }
+      end
+      ruby2_keywords :process_action if respond_to?(:ruby2_keywords, true)
+
+      def self.install
+        return false unless defined?(::ActiveSupport)
+
+        ::ActiveSupport.on_load(:action_controller) do
+          # `self` here is ActionController::Base or ::API, whichever the
+          # application loaded — both, in an app that has each.
+          prepend Sixty::Instrument::ActionController
+        end
+        true
+      end
+    end
+  end
+end

data/lib/sixty/instrument/active_record.rb

@@ -0,0 +1,139 @@
+# frozen_string_literal: true
+
+require_relative '../tracer'
+require_relative '../sql'
+require_relative '../stack'
+require_relative '../plans'
+
+module Sixty
+  module Instrument
+    # ActiveRecord.
+    #
+    # What is captured beyond timing is the point of the product:
+    #   rows — how many the statement returned. The 30 -> 30,000 signal, and the
+    #          one an ORM makes easy to lose: `.where(...)` without a limit is
+    #          one character away from `.all`, and on a development database
+    #          both take four milliseconds.
+    #   calls — every query, credited to every enclosing method and to the
+    #          request. One query becoming twelve is an N+1 being born.
+    #
+    # ── Why a notification subscriber and not a patched adapter ───────────────
+    #
+    # `sql.active_record` is a public, stable interface that every adapter emits
+    # and every Rails version has. Patching `exec_query` would work on one
+    # adapter and one version and break quietly on the next, and "quietly" is
+    # the failure mode this project refuses. The cost is that the event arrives
+    # *after* the query — so the span is assembled from the event's own start
+    # and finish times rather than measured around it, which is exactly what
+    # `Tracer.record` exists for.
+    module ActiveRecord
+      # Statements Rails issues about itself rather than on the application's
+      # behalf. A schema load at boot is not an operation anyone can act on, and
+      # a cached query never reached the database at all — recording it would
+      # report database work that did not happen.
+      IGNORED_NAMES = ['SCHEMA', 'TRANSACTION', 'EXPLAIN'].freeze
+
+      class << self
+        def install(config)
+          return false if @installed
+          return false unless defined?(::ActiveSupport::Notifications)
+
+          @config = config
+          @subscriber = ::ActiveSupport::Notifications.monotonic_subscribe('sql.active_record') do |_name, started, finished, _id, payload|
+            record(payload, (finished - started) * 1000.0)
+          end
+
+          Sixty.before_flush { capture_plans } if config.capture_plans
+          @installed = true
+        end
+
+        def installed?
+          @installed == true
+        end
+
+        # Which SQL dialect this application speaks.
+        #
+        # It decides whether `"alice@example.com"` is an identifier to keep or a
+        # string literal to strip, so guessing from the text is not an option —
+        # a wrong guess in one direction transmits a customer's data. Asked of
+        # the adapter, which knows, and cached because the answer cannot change
+        # for the life of a process.
+        #
+        # `connection_db_config` rather than a live connection: reading this
+        # must never be a reason to open one.
+        def dialect
+          return @dialect if defined?(@dialect) && @dialect
+
+          @dialect = begin
+            name = ::ActiveRecord::Base.connection_db_config.adapter.to_s
+            name.match?(/mysql|trilogy|maria/i) ? :mysql : :postgres
+          rescue StandardError
+            :postgres
+          end
+        end
+
+        private
+
+        def record(payload, duration_ms)
+          return if Thread.current[:sixty_in_agent_query]
+          return if payload[:cached]
+          return if IGNORED_NAMES.include?(payload[:name].to_s)
+
+          sql = payload[:sql]
+          return unless sql.is_a?(String) && !sql.empty?
+
+          normalized = Sql.normalize_sql(sql, dialect)
+          attrs = { normalized_sql: normalized }
+
+          # `row_count` is what a modern Rails reports; on anything older the
+          # metric is simply absent rather than guessed at. A wrong row count is
+          # worse than no row count — it is the number the whole detector leans
+          # on.
+          attrs[:rows] = payload[:row_count] if payload[:row_count].is_a?(Numeric)
+
+          # Once per distinct statement, never again: the call site is a
+          # property of the query, not of the call.
+          frames = Stack.capture(normalized)
+          attrs[:frames] = frames if frames
+          if frames&.first
+            attrs[:file] = frames.first[:file]
+            attrs[:line] = frames.first[:line]
+          end
+
+          Tracer.record(
+            kind: Tracer::KIND_DB,
+            name: Sql.sql_operation_name(normalized),
+            duration_ms: duration_ms,
+            attrs: attrs
+          )
+
+          Plans.enqueue(sql, normalized, dialect: dialect) if @config&.capture_plans
+        rescue StandardError => e
+          @config&.on_warn&.call("sixty: error recording query: #{e.message}")
+        end
+
+        # Runs on the agent's flush thread, never on a request.
+        #
+        # `with_connection` checks a connection out of the pool and returns it,
+        # so this cannot leak one, and the thread-local flag keeps the EXPLAIN
+        # from being recorded as an operation of its own — a query that measures
+        # queries would count toward the fanout of nothing and appear in the
+        # feed as an operation the application does not contain.
+        def capture_plans
+          return unless defined?(::ActiveRecord::Base)
+
+          Thread.current[:sixty_in_agent_query] = true
+          ::ActiveRecord::Base.connection_pool.with_connection do |connection|
+            Plans.capture_pending(lambda { |sql|
+              result = connection.exec_query(sql)
+              row = result.rows.first
+              row && row.first
+            })
+          end
+        ensure
+          Thread.current[:sixty_in_agent_query] = false
+        end
+      end
+    end
+  end
+end