goodsorcery 0.1.1 → 0.2.0

data/lib/goodsorcery/performance/trace_buffer.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+module GoodSorcery
+  module Performance
+    # Holds the slowest trace per transaction per minute (DESIGN.md 8.5).
+    # +candidate?+ is the cheap check made for every unit of work; only a
+    # candidate's trace is built (outside the lock), then +offer+ swaps it in
+    # if it is still the slowest. At most MAX_TRACES_PER_MINUTE traces per
+    # minute: when full, a new transaction's trace replaces the fastest one
+    # held, and only if it is slower.
+    class TraceBuffer
+      MAX_TRACES_PER_MINUTE = 30
+      MAX_MINUTES = Aggregator::MAX_PENDING_MINUTES
+
+      Entry = Struct.new(:duration_ms, :trace)
+
+      def initialize
+        @lock = Mutex.new
+        @minutes = {}
+      end
+
+      def candidate?(minute, key, duration_ms)
+        @lock.synchronize { slower?(@minutes[minute], key, duration_ms) }
+      end
+
+      # => true when the trace was kept.
+      def offer(minute, key, duration_ms, trace)
+        @lock.synchronize do
+          entries = minute_entries(minute)
+          next false unless slower?(entries, key, duration_ms)
+
+          evict_fastest(entries) if !entries.key?(key) && entries.size >= MAX_TRACES_PER_MINUTE
+          entries[key] = Entry.new(duration_ms, trace)
+          true
+        end
+      end
+
+      # Removes and returns the traces of the minutes up to +last_minute+
+      # (all when nil), slowest first.
+      def take(last_minute = nil)
+        @lock.synchronize do
+          minutes = @minutes.keys.select { |minute| last_minute.nil? || minute <= last_minute }
+          entries = minutes.flat_map { |minute| @minutes.delete(minute).values }
+          entries.sort_by { |entry| -entry.duration_ms }.map(&:trace)
+        end
+      end
+
+      def size
+        @lock.synchronize { @minutes.values.sum(&:size) }
+      end
+
+      private
+
+      def slower?(entries, key, duration_ms)
+        return true if entries.nil?
+
+        held = entries[key]
+        return duration_ms > held.duration_ms if held
+        return true if entries.size < MAX_TRACES_PER_MINUTE
+
+        duration_ms > entries.values.min_by(&:duration_ms).duration_ms
+      end
+
+      def evict_fastest(entries)
+        key, = entries.min_by { |_key, entry| entry.duration_ms }
+        entries.delete(key)
+      end
+
+      def minute_entries(minute)
+        entries = @minutes[minute]
+        return entries if entries
+
+        @minutes.delete(@minutes.keys.min) while @minutes.size >= MAX_MINUTES
+        @minutes[minute] = {}
+      end
+    end
+  end
+end

data/lib/goodsorcery/performance/trace_builder.rb

@@ -0,0 +1,121 @@
+# frozen_string_literal: true
+
+module GoodSorcery
+  module Performance
+    # What every trace in this process shares: the SQL normaliser, app paths,
+    # the source line cache (only with send_code_context), host and runtime.
+    class TraceContext
+      attr_reader :normalizer, :paths, :host, :runtime
+
+      def initialize(config)
+        @normalizer = SqlNormalizer.new
+        @paths = Backtrace::Paths.new(config.root)
+        @line_cache = config.send_code_context ? Backtrace::LineCache.new(@paths, max_files: 32) : nil
+        @host = hostname
+        @runtime = { "ruby" => RUBY_VERSION, "rails" => rails_version }.compact
+      end
+
+      def call_site_hash(call_site)
+        call_site.to_h(@paths, line_cache: @line_cache)
+      end
+
+      private
+
+      def hostname
+        Text.limit(Socket.gethostname, 255)
+      rescue StandardError
+        nil
+      end
+
+      def rails_version
+        ::Rails.version.to_s if defined?(::Rails) && ::Rails.respond_to?(:version)
+      end
+    end
+
+    # Builds one trace payload (INGEST_API.md › Traces payload) from a
+    # finished recorder. Only the slowest unit of work so far in its minute
+    # gets here, so this may take a little time; it never runs inside a lock.
+    class TraceBuilder
+      MAX_TRACE_BYTES = 65_536
+      DECIMALS = 3
+
+      def initialize(recorder, minute, context)
+        @recorder = recorder
+        @minute = minute
+        @context = context
+      end
+
+      def build
+        spans = SpanList.new(@recorder, @context)
+        trace = identity.merge(timing, outcome, details, counts(spans.to_a, spans.dropped))
+        fit(trace)
+      end
+
+      private
+
+      def identity
+        hash = { "id" => UuidV7.generate(Time.at(@recorder.started_wall)), "kind" => @recorder.kind.to_s,
+                 "name" => Text.limit(@recorder.name, 255) }
+        hash.merge(Labels.for(@recorder).except("source"))
+      end
+
+      def timing
+        { "minute" => Performance.minute_iso(@minute), "started_at" => Time.at(@recorder.started_wall).utc.iso8601(3),
+          "duration_ms" => round(@recorder.duration_ms), "db_ms" => round(@recorder.db_ms),
+          "http_ms" => round(@recorder.http_ms), "view_ms" => round(@recorder.view_ms),
+          "ruby_ms" => round(@recorder.ruby_ms) }
+      end
+
+      def outcome
+        hash = { "failed" => @recorder.failed?, "error_event_id" => @recorder.error_event_id }
+        hash["status"] = @recorder.status if @recorder.kind == :web && @recorder.status
+        hash
+      end
+
+      def details
+        section = @recorder.kind == :web ? { "request" => request } : { "job" => job }
+        gc = @recorder.gc || {}
+        section.merge("host" => @context.host, "process" => ProcessLabel.current, "runtime" => @context.runtime,
+                      "gc" => gc.slice("time_ms", "count"), "allocations" => gc["allocations"]).compact
+      end
+
+      def request
+        env = @recorder.env || {}
+        path = "#{env["SCRIPT_NAME"]}#{env["PATH_INFO"]}"
+        { "method" => @recorder.http_method && Text.limit(@recorder.http_method, 16),
+          "path" => Text.limit(path.empty? ? "/" : path, SpanList::MAX_PATH_LENGTH),
+          "request_id" => env["action_dispatch.request_id"] && Text.limit(env["action_dispatch.request_id"], 128) }
+          .compact
+      end
+
+      def job
+        { "adapter" => @recorder.adapter, "queue" => @recorder.queue,
+          "queue_wait_ms" => @recorder.queue_wait_ms && round(@recorder.queue_wait_ms),
+          "executions" => @recorder.job_details&.executions }.compact
+      end
+
+      def counts(spans, dropped)
+        { "sql_count" => @recorder.sql_count, "http_count" => @recorder.http_count,
+          "spans_dropped" => @recorder.spans_dropped + dropped, "spans" => spans }
+      end
+
+      # Keeps a trace under MAX_TRACE_BYTES of JSON by dropping its shortest
+      # spans, half at a time.
+      def fit(trace)
+        halve_spans(trace) while JSON.generate(trace).bytesize > MAX_TRACE_BYTES && !trace["spans"].empty?
+        trace
+      end
+
+      def halve_spans(trace)
+        spans = trace["spans"]
+        kept = spans.max_by(spans.size / 2) { |span| span["duration_ms"] }.sort_by { |span| span["offset_ms"] }
+        trace["spans_dropped"] += (spans - kept).sum { |span| span.fetch("count", 1) }
+        trace["spans"] = kept
+      end
+
+      def round(value)
+        value.to_f.round(DECIMALS)
+      end
+    end
+  end
+end

data/lib/goodsorcery/performance/transaction_stats.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+module GoodSorcery
+  module Performance
+    # What one finished unit of work contributes to the metrics.
+    Sample = Struct.new(:kind, :name, :labels, :duration_ms, :failed, :db_ms, :http_ms, :view_ms, :ruby_ms,
+                        :queue_wait_ms) do
+      def self.from(recorder)
+        new(recorder.kind, recorder.name, Labels.for(recorder), recorder.duration_ms, recorder.failed?,
+            recorder.db_ms, recorder.http_ms, recorder.view_ms, recorder.ruby_ms, recorder.queue_wait_ms)
+      end
+    end
+
+    # The labels sent with a transaction: method and route for web requests,
+    # adapter and queue for jobs, and the source file of either.
+    module Labels
+      LIMITS = { "method" => 16, "route" => 255, "adapter" => 32, "queue" => 128, "source" => 500 }.freeze
+
+      def self.for(recorder)
+        labels = if recorder.kind == :web
+                   { "method" => recorder.http_method, "route" => recorder.route }
+                 else
+                   { "adapter" => recorder.adapter, "queue" => recorder.queue }
+                 end
+        labels["source"] = recorder.source
+        labels.each_with_object({}) do |(key, value), limited|
+          limited[key] = Text.limit(value, LIMITS.fetch(key)) unless value.nil?
+        end
+      end
+    end
+
+    # One transaction's numbers for one minute (DESIGN.md 8.4): count,
+    # failed, the summed duration and, per histogram bucket that has
+    # requests, [count, db_ms, http_ms, view_ms, ruby_ms]. Jobs add the
+    # summed queue wait and its bucket counts.
+    class TransactionStats
+      DECIMALS = 3
+
+      attr_reader :count, :failed, :duration_ms, :buckets, :queue_wait_ms, :queue_wait, :labels
+
+      def initialize
+        @count = @failed = 0
+        @duration_ms = 0.0
+        @buckets = {}
+        @queue_wait_ms = nil
+        @queue_wait = {}
+        @labels = {}
+      end
+
+      # +keep_labels+ is false for the "(other)" row, which has none.
+      def add(sample, keep_labels: true)
+        @labels = sample.labels if keep_labels
+        @count += 1
+        @failed += 1 if sample.failed
+        @duration_ms += sample.duration_ms
+        add_bucket(sample)
+        add_queue_wait(sample.queue_wait_ms) if sample.queue_wait_ms
+      end
+
+      def to_h(kind, name)
+        hash = { "kind" => kind.to_s, "name" => name }.merge(@labels)
+        hash.merge!("count" => @count, "failed" => @failed, "duration_ms" => @duration_ms.round(DECIMALS),
+                    "buckets" => bucket_rows)
+        return hash unless @queue_wait_ms
+
+        hash.merge("queue_wait_ms" => @queue_wait_ms.round(DECIMALS), "queue_wait" => @queue_wait.sort)
+      end
+
+      private
+
+      def add_bucket(sample)
+        row = (@buckets[Histogram.index(sample.duration_ms)] ||= [0, 0.0, 0.0, 0.0, 0.0])
+        row[0] += 1
+        row[1] += sample.db_ms
+        row[2] += sample.http_ms
+        row[3] += sample.view_ms
+        row[4] += sample.ruby_ms
+      end
+
+      def add_queue_wait(wait_ms)
+        @queue_wait_ms = (@queue_wait_ms || 0.0) + wait_ms
+        index = Histogram.index(wait_ms)
+        @queue_wait[index] = @queue_wait.fetch(index, 0) + 1
+      end
+
+      def bucket_rows
+        @buckets.sort.map do |index, (count, *layers)|
+          [index, count, *layers.map { |ms| ms.round(DECIMALS) }]
+        end
+      end
+    end
+  end
+end

data/lib/goodsorcery/performance/uuid_v7.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module GoodSorcery
+  module Performance
+    # Time-ordered UUIDs (RFC 9562 version 7): the first 48 bits are the
+    # given time in Unix milliseconds, the rest random. Trace ids carry the
+    # request's start time this way, which SecureRandom.uuid_v7 (always "now")
+    # cannot do.
+    module UuidV7
+      VARIANT_NIBBLES = %w[8 9 a b].freeze
+
+      def self.generate(time = Time.now)
+        hex = hex_digits(time)
+        "#{hex[0, 8]}-#{hex[8, 4]}-#{hex[12, 4]}-#{hex[16, 4]}-#{hex[20, 12]}"
+      end
+
+      # 32 hex digits: 48 bits of time, the version, the variant, randomness.
+      def self.hex_digits(time)
+        milliseconds = (time.to_r * 1000).to_i & 0xFFFF_FFFF_FFFF
+        hex = format("%012x", milliseconds) + SecureRandom.random_bytes(10).unpack1("H*")
+        hex[12] = "7"
+        hex[16] = VARIANT_NIBBLES[hex[16].to_i(16) & 3]
+        hex
+      end
+    end
+  end
+end

data/lib/goodsorcery/performance.rb

@@ -0,0 +1,158 @@
+# frozen_string_literal: true
+
+require_relative "performance/histogram"
+require_relative "performance/uuid_v7"
+require_relative "performance/process_label"
+require_relative "performance/sql_normalizer"
+require_relative "performance/call_site"
+require_relative "performance/recorder"
+require_relative "performance/transaction_stats"
+require_relative "performance/aggregator"
+require_relative "performance/trace_buffer"
+require_relative "performance/span_list"
+require_relative "performance/trace_builder"
+require_relative "performance/payloads"
+require_relative "performance/flusher"
+require_relative "performance/job_timing"
+require_relative "performance/subscribers"
+require_relative "performance/net_http"
+require_relative "performance/agent"
+require_relative "performance/measurement"
+
+module GoodSorcery
+  # Performance monitoring (DESIGN.md 8): metrics for every request and job,
+  # and traces of the slowest ones, aggregated in process and sent once a
+  # minute. Off unless config.performance_monitoring is true; then
+  # Performance.install (the Railtie calls it after the app initialized)
+  # subscribes to Rails' notifications and instruments Net::HTTP. Nothing
+  # here ever raises into the host app.
+  module Performance
+    RECORDER_KEY = :__goodsorcery_recorder
+    SUPPRESS_KEY = :__goodsorcery_suppress_performance
+    LOCK = Mutex.new
+
+    @active = false
+    @traces = true
+    @ignore = IgnoreMatcher.new([])
+
+    extend Measurement
+    extend TestPayloads
+
+    class << self
+      # Starts recording when performance_monitoring is on and the
+      # environment reports (or the delivery is :test). => true when on.
+      def install(config = GoodSorcery.configuration)
+        return false unless config.performance_monitoring && GoodSorcery.client.performance_allowed?
+
+        configure(config)
+        Subscribers.subscribe
+        NetHttp.install
+        @active = true
+      rescue StandardError => e
+        log.warn("could not start performance monitoring (#{e.class})")
+        false
+      end
+
+      # The one check integrations make per request or job.
+      def active? = @active
+
+      # Stops recording in this process (the server has no APM endpoints).
+      def disable!
+        @active = false
+        @agent&.flusher&.stop!
+      end
+
+      def recording? = !Thread.current[RECORDER_KEY].nil?
+      def current_recorder = Thread.current[RECORDER_KEY]
+      def suppressed? = Thread.current[SUPPRESS_KEY] ? true : false
+      def monotonic = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+
+      # Nothing is measured inside the block (the gem's own HTTP requests).
+      def suppress
+        previous = Thread.current[SUPPRESS_KEY]
+        Thread.current[SUPPRESS_KEY] = true
+        yield
+      ensure
+        Thread.current[SUPPRESS_KEY] = previous
+      end
+
+      # => a new recorder for this fiber, or nil when off or already recording
+      # (a unit of work inside another is part of the outer one).
+      def start(kind)
+        return nil unless @active && Thread.current[RECORDER_KEY].nil?
+
+        Thread.current[RECORDER_KEY] = Recorder.new(kind, traces: @traces)
+      rescue StandardError
+        nil
+      end
+
+      def finish(recorder)
+        Thread.current[RECORDER_KEY] = nil if Thread.current[RECORDER_KEY].equal?(recorder)
+        recorder.stop
+        agent.record(recorder, background: GoodSorcery.configuration.delivery != :test) if recordable?(recorder)
+      rescue StandardError => e
+        log.warn_every(:apm_finish_error, 60, "could not record performance data (#{e.class})")
+      end
+
+      # Sends (or, with the :test delivery, stores) what was recorded.
+      def flush_performance(include_open_minute: true)
+        current = @agent
+        return true unless current&.current?
+
+        current.flusher.flush(include_open: include_open_minute)
+      end
+
+      def shutdown(timeout)
+        current = @agent
+        current.flusher.shutdown(timeout) if current&.current?
+        true
+      rescue StandardError
+        false
+      end
+
+      def agent
+        current = @agent
+        return current if current&.current?
+
+        LOCK.synchronize do
+          @agent = Agent.new(GoodSorcery.configuration, log) unless @agent&.current?
+          @agent
+        end
+      end
+
+      def paths
+        @paths ||= Backtrace::Paths.new(GoodSorcery.configuration.root)
+      end
+
+      def minute_iso(minute)
+        Time.at(minute * 60).utc.strftime("%Y-%m-%dT%H:%M:00Z")
+      end
+
+      # Forgets everything recorded and turns recording off. For tests.
+      def reset!
+        @active = false
+        @agent&.flusher&.stop!
+        @agent = nil
+        @paths = nil
+        Thread.current[RECORDER_KEY] = nil
+        SourceLocation.clear
+        clear_test_payloads
+      end
+
+      private
+
+      def configure(config)
+        @traces = config.performance_traces
+        @ignore = IgnoreMatcher.new(config.performance_ignore_transactions)
+        @paths = Backtrace::Paths.new(config.root)
+        SourceLocation.clear
+      end
+
+      def log = GoodSorcery.log
+
+      def recordable?(recorder)
+        @active && !recorder.broken? && !recorder.name.nil? && !@ignore.ignored?(recorder.name)
+      end
+    end
+  end
+end

data/lib/goodsorcery/rack/middleware.rb

@@ -10,7 +10,8 @@ module GoodSorcery
     # "METHOD /path" breadcrumb and reports exceptions that propagate, then
     # re-raises them. Exceptions an inner layer turned into a 5xx response
     # (env["action_dispatch.exception"], env["sinatra.error"]) are reported
-    # too. The request itself is only read if an error is reported.
+    # too. The request itself is only read if an error is reported. With
+    # performance monitoring on, it also times the request (DESIGN.md 8.2).
     #
     #   use GoodSorcery::Rack::Middleware
     class Middleware
@@ -22,6 +23,14 @@ module GoodSorcery
       end
 
       def call(env)
+        return call_in_scope(env) unless Performance.active?
+
+        Performance.measure_request(env) { call_in_scope(env) }
+      end
+
+      private
+
+      def call_in_scope(env)
         GoodSorcery.with_scope do |scope|
           open_request(scope, env)
           response = call_app(env)
@@ -30,8 +39,6 @@ module GoodSorcery
         end
       end
 
-      private
-
       def open_request(scope, env)
         scope.request_env = env
         scope.enricher = @enricher.new(env)

data/lib/goodsorcery/transport.rb

@@ -52,6 +52,12 @@ module GoodSorcery
       send_request(Net::HTTP::Get.new(path(VERIFY_PATH), headers))
     end
 
+    # Sends one performance payload (DESIGN.md 8.7). The flusher decides
+    # what to do with the result and does the logging.
+    def post_apm(path_name, json)
+      send_request(post(path_name, json))
+    end
+
     def close
       @connection_lock.synchronize { @connection.close }
     end
@@ -111,8 +117,13 @@ module GoodSorcery
     end
 
     # The persistent connection serves one thread at a time; a concurrent
-    # caller (the :sync delivery in a busy app) gets a short-lived one.
+    # caller (the :sync delivery in a busy app) gets a short-lived one. The
+    # gem's own requests are never measured by performance monitoring.
     def perform(request)
+      Performance.suppress { perform_unmeasured(request) }
+    end
+
+    def perform_unmeasured(request)
       return Connection.new(@uri, **@timeouts).request_once(request) unless @connection_lock.try_lock
 
       begin

data/lib/goodsorcery/version.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
 module GoodSorcery
-  VERSION = "0.1.1"
+  VERSION = "0.2.0"
   NOTIFIER_NAME = "goodsorcery-ruby"
 end

data/lib/goodsorcery.rb

@@ -51,6 +51,7 @@ require_relative "goodsorcery/delivery"
 require_relative "goodsorcery/verification"
 require_relative "goodsorcery/test_event"
 require_relative "goodsorcery/client"
+require_relative "goodsorcery/performance"
 require_relative "goodsorcery/rack/middleware"
 
 # Reports exceptions to a Good Sorcery error tracker. See README.md.
@@ -152,13 +153,29 @@ module GoodSorcery
       client.test_events
     end
 
+    # Also forgets the performance payloads kept by the :test delivery.
     def clear_test_events
+      Performance.clear_test_payloads
       client.clear_test_events
     end
 
+    # Sends the recorded performance data now (with the :test delivery:
+    # builds it into test_performance_payloads). Never raises.
+    def flush_performance(include_open_minute: true)
+      Performance.flush_performance(include_open_minute: include_open_minute)
+    rescue StandardError
+      false
+    end
+
+    # [{ "path" => "/api/v1/apm/metrics", "body" => Hash }, ...] with the :test delivery.
+    def test_performance_payloads
+      Performance.test_payloads
+    end
+
     # Forgets configuration, client and the current scope. For tests.
     def reset!
       shutdown(0.5)
+      Performance.reset!
       LOCK.synchronize do
         @client = nil
         @configuration = nil

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: goodsorcery
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.0
 platform: ruby
 authors:
 - Philip Q Nguyen
@@ -32,6 +32,7 @@ files:
 - lib/goodsorcery/bundler_plugin.rb
 - lib/goodsorcery/client.rb
 - lib/goodsorcery/configuration.rb
+- lib/goodsorcery/configuration/performance_settings.rb
 - lib/goodsorcery/configuration/validation.rb
 - lib/goodsorcery/defaults.rb
 - lib/goodsorcery/delivery.rb
@@ -82,6 +83,25 @@ files:
 - lib/goodsorcery/key_filter.rb
 - lib/goodsorcery/log.rb
 - lib/goodsorcery/masked_key.rb
+- lib/goodsorcery/performance.rb
+- lib/goodsorcery/performance/agent.rb
+- lib/goodsorcery/performance/aggregator.rb
+- lib/goodsorcery/performance/call_site.rb
+- lib/goodsorcery/performance/flusher.rb
+- lib/goodsorcery/performance/histogram.rb
+- lib/goodsorcery/performance/job_timing.rb
+- lib/goodsorcery/performance/measurement.rb
+- lib/goodsorcery/performance/net_http.rb
+- lib/goodsorcery/performance/payloads.rb
+- lib/goodsorcery/performance/process_label.rb
+- lib/goodsorcery/performance/recorder.rb
+- lib/goodsorcery/performance/span_list.rb
+- lib/goodsorcery/performance/sql_normalizer.rb
+- lib/goodsorcery/performance/subscribers.rb
+- lib/goodsorcery/performance/trace_buffer.rb
+- lib/goodsorcery/performance/trace_builder.rb
+- lib/goodsorcery/performance/transaction_stats.rb
+- lib/goodsorcery/performance/uuid_v7.rb
 - lib/goodsorcery/rack/middleware.rb
 - lib/goodsorcery/rack/request_details.rb
 - lib/goodsorcery/rack/request_enricher.rb