goodsorcery 0.1.1 → 0.3.0

data/lib/goodsorcery/performance/aggregator.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+module GoodSorcery
+  module Performance
+    # Per-process, per-minute metrics (DESIGN.md 8.4). Keyed by the minute in
+    # which a unit of work finished (integer Unix minutes), then kind, then
+    # name. At most MAX_TRANSACTIONS names per kind per minute (the rest are
+    # counted under "(other)") and MAX_PENDING_MINUTES minutes are held.
+    class Aggregator
+      MAX_TRANSACTIONS = 500
+      MAX_PENDING_MINUTES = 5
+      OTHER = "(other)"
+
+      attr_reader :dropped_minutes
+
+      def initialize
+        @lock = Mutex.new
+        @minutes = {}
+        @dropped_minutes = 0
+      end
+
+      def record(minute, sample)
+        @lock.synchronize do
+          names = (minute_table(minute)[sample.kind] ||= {})
+          name = names.key?(sample.name) || names.size < MAX_TRANSACTIONS ? sample.name : OTHER
+          (names[name] ||= TransactionStats.new).add(sample, keep_labels: name != OTHER)
+        end
+      end
+
+      # Removes and returns the minutes up to and including +last_minute+
+      # (all of them when nil), oldest first: [[minute, {kind => {name => stats}}]].
+      def take(last_minute = nil)
+        @lock.synchronize do
+          taken = @minutes.keys.sort.select { |minute| last_minute.nil? || minute <= last_minute }
+          taken.map { |minute| [minute, @minutes.delete(minute)] }
+        end
+      end
+
+      def minutes
+        @lock.synchronize { @minutes.keys.sort }
+      end
+
+      def empty?
+        @lock.synchronize { @minutes.empty? }
+      end
+
+      private
+
+      def minute_table(minute)
+        table = @minutes[minute]
+        return table if table
+
+        drop_oldest while @minutes.size >= MAX_PENDING_MINUTES
+        @minutes[minute] = {}
+      end
+
+      def drop_oldest
+        @minutes.delete(@minutes.keys.min)
+        @dropped_minutes += 1
+      end
+    end
+  end
+end

data/lib/goodsorcery/performance/call_site.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module GoodSorcery
+  module Performance
+    # Where the app ran a repeated query: the first stack frame that is the
+    # app's own code (not a gem, not this library). Captured inside the
+    # sql.active_record callback; the code lines are read only when a trace
+    # is built.
+    class CallSite
+      GEM_LIB = "#{File.expand_path("../..", __dir__)}/".freeze
+      CONTEXT_BEFORE = 2
+      CONTEXT_AFTER = 2
+      MAX_LINE_LENGTH = 200
+      MAX_PATH_LENGTH = 1_024
+      MAX_METHOD_LENGTH = 255
+
+      attr_reader :path, :line, :label
+
+      # => a CallSite, or nil when no app frame is on the stack.
+      def self.capture(paths)
+        Thread.each_caller_location do |location|
+          path = location.absolute_path || location.path
+          next if path.nil? || path.start_with?(GEM_LIB) || !paths.in_app?(path)
+
+          return new(path, location.lineno, location.base_label)
+        end
+        nil
+      end
+
+      def initialize(path, line, label)
+        @path = path
+        @line = line
+        @label = label
+      end
+
+      def to_h(paths, line_cache: nil)
+        hash = { "file" => Text.limit(paths.relative(path), MAX_PATH_LENGTH), "line" => line,
+                 "method" => label && Text.limit(label, MAX_METHOD_LENGTH) }.compact
+        context = line_cache && code_context(line_cache)
+        hash["context"] = context if context
+        hash
+      end
+
+      private
+
+      # Two lines before and after the call (5 lines of up to 200 chars).
+      def code_context(line_cache)
+        lines = line_cache.lines(path)
+        return nil unless lines && line.between?(1, lines.size)
+
+        first = [line - CONTEXT_BEFORE, 1].max
+        last = [line + CONTEXT_AFTER, lines.size].min
+        { "start_line" => first, "lines" => limited(lines[(first - 1)..(last - 1)]) }
+      end
+
+      def limited(lines)
+        lines.map { |text| Text.limit(text, MAX_LINE_LENGTH) }
+      end
+    end
+  end
+end

data/lib/goodsorcery/performance/flusher.rb

@@ -0,0 +1,224 @@
+# frozen_string_literal: true
+
+module GoodSorcery
+  module Performance
+    # Sends the closed minutes once a minute from one background thread per
+    # process ("goodsorcery-apm"), at the minute boundary plus a random
+    # 2–20 s offset that spreads the server's load (DESIGN.md 8.7).
+    class Flusher
+      OFFSET_RANGE = (2.0..20.0)
+
+      attr_reader :offset, :outbox
+
+      def initialize(agent:, log:, clock: nil, offset: nil, transport: nil)
+        @agent = agent
+        @log = log
+        @clock = clock || -> { Process.clock_gettime(Process::CLOCK_REALTIME) }
+        @offset = offset || rand(OFFSET_RANGE)
+        @outbox = Outbox.new(log: log, transport: transport || -> { GoodSorcery.client.transport })
+        @lock = Mutex.new
+        @wake = ConditionVariable.new
+        @flush_lock = Mutex.new
+        @stopped = false
+        @dropped_seen = 0
+      end
+
+      # The next wall-clock time (seconds) to wake at after +now+.
+      def next_wake_at(now)
+        (((now / 60).floor + 1) * 60) + @offset
+      end
+
+      def ensure_started
+        return if @stopped || @thread&.alive?
+
+        @lock.synchronize { start_thread unless @stopped || @thread&.alive? }
+      end
+
+      def running?
+        @thread&.alive? || false
+      end
+
+      # Collects the closed minutes (every minute with +include_open+) and
+      # sends what is pending, stopping at +deadline+ (monotonic seconds).
+      # With the :test delivery the payloads are stored instead.
+      def flush(include_open: false, deadline: nil)
+        @flush_lock.synchronize do
+          collect(include_open ? nil : (@clock.call / 60).floor - 1)
+          test_delivery? ? @outbox.store_for_tests : @outbox.send_pending(deadline)
+        end
+        true
+      rescue StandardError => e
+        @log.warn_every(:apm_flush_error, 60, "performance flush failed (#{e.class})")
+        false
+      end
+
+      # Stops the thread and sends everything, including the open minute.
+      def shutdown(timeout)
+        thread = stop_thread
+        thread&.join([timeout, 1].min) unless thread == Thread.current
+        flush(include_open: true, deadline: Performance.monotonic + timeout)
+      end
+
+      # Performance monitoring was turned off: stop and forget what is pending.
+      def stop!
+        stop_thread
+        @outbox.clear
+      end
+
+      private
+
+      def start_thread
+        @thread = Thread.new do
+          Thread.current.report_on_exception = false
+          run
+        end
+        @thread.name = "goodsorcery-apm"
+        Delivery::ExitHook.install
+      end
+
+      def stop_thread
+        @lock.synchronize do
+          @stopped = true
+          @wake.broadcast
+          @thread
+        end
+      end
+
+      def run
+        until @stopped
+          wait_until(next_wake_at(@clock.call))
+          flush unless @stopped
+        end
+      end
+
+      def wait_until(target)
+        @lock.synchronize do
+          loop do
+            remaining = target - @clock.call
+            break if @stopped || remaining <= 0
+
+            @wake.wait(@lock, remaining)
+          end
+        end
+      end
+
+      def collect(last_minute)
+        minutes = @agent.aggregator.take(last_minute)
+        traces = @agent.trace_buffer.take(last_minute)
+        log_dropped_minutes
+        @outbox.add(Payloads.metrics(minutes) + Payloads.traces(traces))
+      end
+
+      def log_dropped_minutes
+        dropped = @agent.aggregator.dropped_minutes
+        return if dropped == @dropped_seen
+
+        @dropped_seen = dropped
+        @log.warn_every(:apm_minutes_dropped, 60, "dropped #{dropped} minute(s) of performance metrics so far")
+      end
+
+      def test_delivery?
+        GoodSorcery.configuration.delivery == :test
+      end
+    end
+
+    # The payloads waiting to be sent, oldest first, and what to do with each
+    # result (DESIGN.md 8.7): 5xx or a network error keeps the payload (with
+    # its batch_id) for the next wake, up to MAX_ATTEMPTS; 429 pauses sending
+    # until Retry-After; 400/401/413/415/422 drop it; 404 turns performance
+    # monitoring off for the process. At most MAX_PENDING_MINUTES minutes of
+    # metrics and MAX_TRACE_BODIES trace bodies wait; the oldest go first.
+    class Outbox
+      MAX_ATTEMPTS = 3
+      MAX_TRACE_BODIES = 5
+      Pending = Struct.new(:body, :attempts)
+
+      def initialize(log:, transport:)
+        @log = log
+        @transport = transport
+        @pending = []
+        @resume_at = nil
+      end
+
+      def add(bodies)
+        bodies.each { |body| @pending << Pending.new(body, 0) }
+        drop_oldest(:metrics?) while pending_minutes > Aggregator::MAX_PENDING_MINUTES
+        drop_oldest(:traces?) while @pending.count { |item| traces?(item) } > MAX_TRACE_BODIES
+      end
+
+      def size = @pending.size
+      def clear = @pending.clear
+
+      def send_pending(deadline = nil)
+        return if paused?
+
+        while (item = @pending.first)
+          break if deadline && Performance.monotonic > deadline
+          break unless handled?(@transport.call.post_apm(item.body.path, item.body.json), item)
+        end
+      end
+
+      def store_for_tests
+        @pending.each { |item| Performance.store_test_payload(item.body.path, JSON.parse(item.body.json)) }
+        @pending.clear
+      end
+
+      private
+
+      def metrics?(item) = item.body.minutes.positive?
+      def traces?(item) = item.body.minutes.zero?
+      def pending_minutes = @pending.sum { |item| item.body.minutes }
+
+      def drop_oldest(kind)
+        @pending.delete_at(@pending.index { |item| send(kind, item) })
+        @log.warn_every(:apm_dropped, 60, "dropped unsent performance data (the server did not accept it in time)")
+      end
+
+      # => true to go on with the next payload.
+      def handled?(result, item)
+        return sent if result.success?
+        return pause(result.retry_after) if result.rate_limited?
+        return server_without_apm if result.not_found?
+        return failed_attempt(result, item) if result.retryable?
+
+        @pending.shift
+        @log.warn_every(:apm_rejected, 60, "the server rejected performance data (#{result.description}); dropped")
+        true
+      end
+
+      def sent
+        @pending.shift
+        true
+      end
+
+      def pause(seconds)
+        @resume_at = Performance.monotonic + seconds.to_f
+        false
+      end
+
+      def paused?
+        return false unless @resume_at
+        return true if Performance.monotonic < @resume_at
+
+        @resume_at = nil
+        false
+      end
+
+      def failed_attempt(result, item)
+        item.attempts += 1
+        return false if item.attempts < MAX_ATTEMPTS
+
+        @pending.shift
+        @log.warn_every(:apm_failed, 60, "could not send performance data (#{result.description}); dropped")
+        false
+      end
+
+      def server_without_apm
+        @log.warn_once(:apm_not_found, "the server has no performance endpoints; performance monitoring is off")
+        @pending.clear
+        Performance.disable!
+        false
+      end
+    end
+  end
+end

data/lib/goodsorcery/performance/histogram.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module GoodSorcery
+  module Performance
+    # The fixed log-scale duration buckets (scheme "gs-v1", DESIGN.md 8.6),
+    # shared literally with the server. Bucket i holds durations d with
+    # BOUNDS_MS[i - 1] <= d < BOUNDS_MS[i]; bucket 30 holds 48 s and up.
+    module Histogram
+      SCHEME = "gs-v1"
+      BOUNDS_MS = [
+        1, 1.5, 2, 3, 4.5, 6.5, 9.5, 14, 20, 28,
+        40, 60, 85, 125, 180, 260, 380, 550, 800, 1_150,
+        1_700, 2_450, 3_550, 5_100, 7_500, 11_000, 16_000, 23_000, 33_000, 48_000
+      ].freeze
+      LAST_INDEX = BOUNDS_MS.size
+
+      # The bucket index (0..30) for a duration in milliseconds.
+      def self.index(duration_ms)
+        BOUNDS_MS.bsearch_index { |bound| bound > duration_ms } || LAST_INDEX
+      end
+    end
+  end
+end

data/lib/goodsorcery/performance/job_timing.rb

@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+
+module GoodSorcery
+  module Performance
+    # Labels a job recorder: name, adapter, queue, source and queue wait
+    # (start − scheduled_at or enqueued_at, never negative). Subclasses read
+    # them from an Active Job instance or a Sidekiq job hash.
+    class JobTiming
+      def label(recorder)
+        recorder.name = name
+        recorder.adapter = adapter
+        recorder.queue = queue
+        recorder.source = SourceLocation.for(source_class)
+        recorder.queue_wait_ms = wait_ms(recorder.started_wall)
+        recorder.job_details = self
+      end
+
+      private
+
+      def wait_ms(started_wall)
+        since = epoch_seconds(queued_since)
+        since ? [(started_wall - since) * 1000.0, 0.0].max : nil
+      end
+
+      # A Time, epoch seconds (Float), epoch milliseconds (Sidekiq 8) or
+      # an ISO 8601 string.
+      def epoch_seconds(value)
+        case value
+        when Numeric then Event::Fields.epoch_seconds(value).to_f
+        when String then Time.iso8601(value).to_f
+        when nil then nil
+        else value.to_time.to_f
+        end
+      rescue ArgumentError, NoMethodError
+        nil
+      end
+    end
+
+    # An Active Job execution (any adapter).
+    class ActiveJobTiming < JobTiming
+      REQUEUED_AT_KEY = :__goodsorcery_requeued_at
+
+      # While the block runs, jobs wait from +time+: when the adapter queued
+      # this run again without updating the job's own enqueued_at (a Sidekiq
+      # retry).
+      def self.requeued_at(time)
+        previous = Thread.current[REQUEUED_AT_KEY]
+        Thread.current[REQUEUED_AT_KEY] = time
+        yield
+      ensure
+        Thread.current[REQUEUED_AT_KEY] = previous
+      end
+
+      def initialize(job)
+        super()
+        @job = job
+      end
+
+      def name = @job.class.name
+      def queue = @job.queue_name&.to_s
+      def source_class = @job.class.name
+      def executions = @job.executions
+
+      def adapter
+        klass = @job.class
+        name = klass.queue_adapter_name if klass.respond_to?(:queue_adapter_name)
+        name&.to_s
+      end
+
+      private
+
+      def queued_since
+        requeued = Thread.current[REQUEUED_AT_KEY]
+        return requeued if requeued
+
+        scheduled = @job.respond_to?(:scheduled_at) ? @job.scheduled_at : nil
+        scheduled || @job.enqueued_at
+      end
+    end
+
+    # A Sidekiq job not wrapped by Active Job.
+    class SidekiqTiming < JobTiming
+      def initialize(job, queue, worker)
+        super()
+        @job = job
+        @queue = queue || job["queue"]
+        @worker = worker
+      end
+
+      def name = @job["class"].to_s
+      def adapter = "sidekiq"
+      def queue = @queue&.to_s
+      def source_class = @worker&.class&.name || name
+
+      # retry_count is absent on the first run and 0 on the first retry.
+      def executions
+        retries = @job["retry_count"]
+        retries.is_a?(Integer) ? retries + 2 : 1
+      end
+
+      private
+
+      def queued_since = @job["enqueued_at"]
+    end
+  end
+end

data/lib/goodsorcery/performance/measurement.rb

@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+
+module GoodSorcery
+  module Performance
+    # What integrations and subscribers call while a unit of work runs:
+    # timing a request or job, reaching the fiber's recorder, and linking a
+    # reported error. Extended into GoodSorcery::Performance.
+    module Measurement
+      # Wraps a Rack request (the middleware calls this when active?).
+      def measure_request(env)
+        recorder = start(:web)
+        return yield unless recorder
+
+        recorder.env = env
+        recorder.http_method = env["REQUEST_METHOD"]
+        response = yield
+        recorder.response_status = response.first.to_i if response.is_a?(Array)
+        response
+      rescue Exception => e # rubocop:disable Lint/RescueException
+        recorder&.raised!(exception_status(e))
+        raise
+      ensure
+        finish(recorder) if recorder
+      end
+
+      # Wraps a job execution; +timing+ is a JobTiming.
+      def measure_job(timing)
+        recorder = start(:job)
+        return yield unless recorder
+
+        label_job(recorder, timing)
+        yield
+      rescue Exception # rubocop:disable Lint/RescueException
+        recorder&.raised!
+        raise
+      ensure
+        finish(recorder) if recorder
+      end
+
+      # Yields the active recorder of this fiber; a failure stops recording
+      # this unit of work and is logged by class name.
+      def with_recorder
+        recorder = Thread.current[RECORDER_KEY]
+        return nil if recorder.nil? || recorder.broken?
+
+        yield recorder
+      rescue StandardError => e
+        recorder&.broken!
+        log.warn_every(:apm_subscriber_error, 60,
+                       "performance recording failed (#{e.class}); a unit of work is not measured")
+        nil
+      end
+
+      def http_recorder
+        return nil if Thread.current[SUPPRESS_KEY]
+
+        recorder = Thread.current[RECORDER_KEY]
+        recorder unless recorder.nil? || recorder.broken?
+      end
+
+      def record_http(recorder, started, host, request, response)
+        status = response.respond_to?(:code) ? response.code.to_i : nil
+        recorder.add_http([started, monotonic], request.method, host, request.path, status)
+      rescue StandardError => e
+        recorder.broken!
+        log.warn_every(:apm_http_error, 60, "could not record an HTTP call (#{e.class})")
+      end
+
+      # Client#notify links the first reported error to the unit of work.
+      def note_error(event_id)
+        recorder = Thread.current[RECORDER_KEY]
+        recorder.error_event_id ||= event_id if recorder && event_id
+      end
+
+      private
+
+      # The status Rails' ShowExceptions answers an escaping exception with
+      # (404 for ActiveRecord::RecordNotFound, 400 for ParameterMissing…);
+      # nil outside Rails.
+      def exception_status(error)
+        return nil unless defined?(::ActionDispatch::ExceptionWrapper)
+
+        template_error = defined?(::ActionView::Template::Error) && error.is_a?(::ActionView::Template::Error)
+        error = error.cause || error if template_error
+        ::ActionDispatch::ExceptionWrapper.status_code_for_exception(error.class.name)
+      rescue StandardError
+        nil
+      end
+
+      def label_job(recorder, timing)
+        timing.label(recorder)
+      rescue StandardError => e
+        recorder.broken!
+        log.warn_every(:apm_job_label_error, 60, "could not read job details for performance data (#{e.class})")
+      end
+    end
+
+    # The payloads the :test delivery keeps instead of sending them
+    # (GoodSorcery.test_performance_payloads).
+    module TestPayloads
+      PAYLOADS_LOCK = Mutex.new
+      PAYLOADS = [] # rubocop:disable Style/MutableConstant
+
+      def store_test_payload(path, body)
+        PAYLOADS_LOCK.synchronize { PAYLOADS << { "path" => path, "body" => body } }
+      end
+
+      def test_payloads
+        PAYLOADS_LOCK.synchronize { PAYLOADS.dup }
+      end
+
+      def clear_test_payloads
+        PAYLOADS_LOCK.synchronize { PAYLOADS.clear }
+      end
+    end
+  end
+end

data/lib/goodsorcery/performance/net_http.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module GoodSorcery
+  module Performance
+    # Prepended to Net::HTTP: one "http" span per request made while a unit
+    # of work is recorded (DESIGN.md 8.3). It always calls super and never
+    # rescues the app's exceptions. The recursive call Net::HTTP#request
+    # makes through #start is skipped, and nothing is recorded while
+    # Performance.suppressed? (the gem's own requests).
+    module NetHttp
+      ACTIVE_KEY = :__goodsorcery_http_request
+
+      def self.install
+        ::Net::HTTP.prepend(self) unless installed?
+      end
+
+      def self.installed?
+        defined?(::Net::HTTP) ? ::Net::HTTP.ancestors.include?(self) : false
+      end
+
+      def request(req, body = nil, &)
+        recorder = Thread.current[ACTIVE_KEY] ? nil : Performance.http_recorder
+        return super unless recorder
+
+        Thread.current[ACTIVE_KEY] = true
+        started = Performance.monotonic
+        response = nil
+        begin
+          response = super
+        ensure
+          Thread.current[ACTIVE_KEY] = nil
+          Performance.record_http(recorder, started, address, req, response)
+        end
+      end
+    end
+  end
+end