goodsorcery 0.1.1 → 0.2.0

data/lib/goodsorcery/performance/flusher.rb

@@ -0,0 +1,224 @@
+# frozen_string_literal: true
+
+module GoodSorcery
+  module Performance
+    # Sends the closed minutes once a minute from one background thread per
+    # process ("goodsorcery-apm"), at the minute boundary plus a random
+    # 2–20 s offset that spreads the server's load (DESIGN.md 8.7).
+    class Flusher
+      OFFSET_RANGE = (2.0..20.0)
+
+      attr_reader :offset, :outbox
+
+      def initialize(agent:, log:, clock: nil, offset: nil, transport: nil)
+        @agent = agent
+        @log = log
+        @clock = clock || -> { Process.clock_gettime(Process::CLOCK_REALTIME) }
+        @offset = offset || rand(OFFSET_RANGE)
+        @outbox = Outbox.new(log: log, transport: transport || -> { GoodSorcery.client.transport })
+        @lock = Mutex.new
+        @wake = ConditionVariable.new
+        @flush_lock = Mutex.new
+        @stopped = false
+        @dropped_seen = 0
+      end
+
+      # The next wall-clock time (seconds) to wake at after +now+.
+      def next_wake_at(now)
+        (((now / 60).floor + 1) * 60) + @offset
+      end
+
+      def ensure_started
+        return if @stopped || @thread&.alive?
+
+        @lock.synchronize { start_thread unless @stopped || @thread&.alive? }
+      end
+
+      def running?
+        @thread&.alive? || false
+      end
+
+      # Collects the closed minutes (every minute with +include_open+) and
+      # sends what is pending, stopping at +deadline+ (monotonic seconds).
+      # With the :test delivery the payloads are stored instead.
+      def flush(include_open: false, deadline: nil)
+        @flush_lock.synchronize do
+          collect(include_open ? nil : (@clock.call / 60).floor - 1)
+          test_delivery? ? @outbox.store_for_tests : @outbox.send_pending(deadline)
+        end
+        true
+      rescue StandardError => e
+        @log.warn_every(:apm_flush_error, 60, "performance flush failed (#{e.class})")
+        false
+      end
+
+      # Stops the thread and sends everything, including the open minute.
+      def shutdown(timeout)
+        thread = stop_thread
+        thread&.join([timeout, 1].min) unless thread == Thread.current
+        flush(include_open: true, deadline: Performance.monotonic + timeout)
+      end
+
+      # Performance monitoring was turned off: stop and forget what is pending.
+      def stop!
+        stop_thread
+        @outbox.clear
+      end
+
+      private
+
+      def start_thread
+        @thread = Thread.new do
+          Thread.current.report_on_exception = false
+          run
+        end
+        @thread.name = "goodsorcery-apm"
+        Delivery::ExitHook.install
+      end
+
+      def stop_thread
+        @lock.synchronize do
+          @stopped = true
+          @wake.broadcast
+          @thread
+        end
+      end
+
+      def run
+        until @stopped
+          wait_until(next_wake_at(@clock.call))
+          flush unless @stopped
+        end
+      end
+
+      def wait_until(target)
+        @lock.synchronize do
+          loop do
+            remaining = target - @clock.call
+            break if @stopped || remaining <= 0
+
+            @wake.wait(@lock, remaining)
+          end
+        end
+      end
+
+      def collect(last_minute)
+        minutes = @agent.aggregator.take(last_minute)
+        traces = @agent.trace_buffer.take(last_minute)
+        log_dropped_minutes
+        @outbox.add(Payloads.metrics(minutes) + Payloads.traces(traces))
+      end
+
+      def log_dropped_minutes
+        dropped = @agent.aggregator.dropped_minutes
+        return if dropped == @dropped_seen
+
+        @dropped_seen = dropped
+        @log.warn_every(:apm_minutes_dropped, 60, "dropped #{dropped} minute(s) of performance metrics so far")
+      end
+
+      def test_delivery?
+        GoodSorcery.configuration.delivery == :test
+      end
+    end
+
+    # The payloads waiting to be sent, oldest first, and what to do with each
+    # result (DESIGN.md 8.7): 5xx or a network error keeps the payload (with
+    # its batch_id) for the next wake, up to MAX_ATTEMPTS; 429 pauses sending
+    # until Retry-After; 400/401/413/415/422 drop it; 404 turns performance
+    # monitoring off for the process. At most MAX_PENDING_MINUTES minutes of
+    # metrics and MAX_TRACE_BODIES trace bodies wait; the oldest go first.
+    class Outbox
+      MAX_ATTEMPTS = 3
+      MAX_TRACE_BODIES = 5
+      Pending = Struct.new(:body, :attempts)
+
+      def initialize(log:, transport:)
+        @log = log
+        @transport = transport
+        @pending = []
+        @resume_at = nil
+      end
+
+      def add(bodies)
+        bodies.each { |body| @pending << Pending.new(body, 0) }
+        drop_oldest(:metrics?) while pending_minutes > Aggregator::MAX_PENDING_MINUTES
+        drop_oldest(:traces?) while @pending.count { |item| traces?(item) } > MAX_TRACE_BODIES
+      end
+
+      def size = @pending.size
+      def clear = @pending.clear
+
+      def send_pending(deadline = nil)
+        return if paused?
+
+        while (item = @pending.first)
+          break if deadline && Performance.monotonic > deadline
+          break unless handled?(@transport.call.post_apm(item.body.path, item.body.json), item)
+        end
+      end
+
+      def store_for_tests
+        @pending.each { |item| Performance.store_test_payload(item.body.path, JSON.parse(item.body.json)) }
+        @pending.clear
+      end
+
+      private
+
+      def metrics?(item) = item.body.minutes.positive?
+      def traces?(item) = item.body.minutes.zero?
+      def pending_minutes = @pending.sum { |item| item.body.minutes }
+
+      def drop_oldest(kind)
+        @pending.delete_at(@pending.index { |item| send(kind, item) })
+        @log.warn_every(:apm_dropped, 60, "dropped unsent performance data (the server did not accept it in time)")
+      end
+
+      # => true to go on with the next payload.
+      def handled?(result, item)
+        return sent if result.success?
+        return pause(result.retry_after) if result.rate_limited?
+        return server_without_apm if result.not_found?
+        return failed_attempt(result, item) if result.retryable?
+
+        @pending.shift
+        @log.warn_every(:apm_rejected, 60, "the server rejected performance data (#{result.description}); dropped")
+        true
+      end
+
+      def sent
+        @pending.shift
+        true
+      end
+
+      def pause(seconds)
+        @resume_at = Performance.monotonic + seconds.to_f
+        false
+      end
+
+      def paused?
+        return false unless @resume_at
+        return true if Performance.monotonic < @resume_at
+
+        @resume_at = nil
+        false
+      end
+
+      def failed_attempt(result, item)
+        item.attempts += 1
+        return false if item.attempts < MAX_ATTEMPTS
+
+        @pending.shift
+        @log.warn_every(:apm_failed, 60, "could not send performance data (#{result.description}); dropped")
+        false
+      end
+
+      def server_without_apm
+        @log.warn_once(:apm_not_found, "the server has no performance endpoints; performance monitoring is off")
+        @pending.clear
+        Performance.disable!
+        false
+      end
+    end
+  end
+end

data/lib/goodsorcery/performance/histogram.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module GoodSorcery
+  module Performance
+    # The fixed log-scale duration buckets (scheme "gs-v1", DESIGN.md 8.6),
+    # shared literally with the server. Bucket i holds durations d with
+    # BOUNDS_MS[i - 1] <= d < BOUNDS_MS[i]; bucket 30 holds 48 s and up.
+    module Histogram
+      SCHEME = "gs-v1"
+      BOUNDS_MS = [
+        1, 1.5, 2, 3, 4.5, 6.5, 9.5, 14, 20, 28,
+        40, 60, 85, 125, 180, 260, 380, 550, 800, 1_150,
+        1_700, 2_450, 3_550, 5_100, 7_500, 11_000, 16_000, 23_000, 33_000, 48_000
+      ].freeze
+      LAST_INDEX = BOUNDS_MS.size
+
+      # The bucket index (0..30) for a duration in milliseconds.
+      def self.index(duration_ms)
+        BOUNDS_MS.bsearch_index { |bound| bound > duration_ms } || LAST_INDEX
+      end
+    end
+  end
+end

data/lib/goodsorcery/performance/job_timing.rb

@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+
+module GoodSorcery
+  module Performance
+    # Labels a job recorder: name, adapter, queue, source and queue wait
+    # (start − scheduled_at or enqueued_at, never negative). Subclasses read
+    # them from an Active Job instance or a Sidekiq job hash.
+    class JobTiming
+      def label(recorder)
+        recorder.name = name
+        recorder.adapter = adapter
+        recorder.queue = queue
+        recorder.source = SourceLocation.for(source_class)
+        recorder.queue_wait_ms = wait_ms(recorder.started_wall)
+        recorder.job_details = self
+      end
+
+      private
+
+      def wait_ms(started_wall)
+        since = epoch_seconds(queued_since)
+        since ? [(started_wall - since) * 1000.0, 0.0].max : nil
+      end
+
+      # A Time, epoch seconds (Float), epoch milliseconds (Sidekiq 8) or
+      # an ISO 8601 string.
+      def epoch_seconds(value)
+        case value
+        when Numeric then Event::Fields.epoch_seconds(value).to_f
+        when String then Time.iso8601(value).to_f
+        when nil then nil
+        else value.to_time.to_f
+        end
+      rescue ArgumentError, NoMethodError
+        nil
+      end
+    end
+
+    # An Active Job execution (any adapter).
+    class ActiveJobTiming < JobTiming
+      REQUEUED_AT_KEY = :__goodsorcery_requeued_at
+
+      # While the block runs, jobs wait from +time+: when the adapter queued
+      # this run again without updating the job's own enqueued_at (a Sidekiq
+      # retry).
+      def self.requeued_at(time)
+        previous = Thread.current[REQUEUED_AT_KEY]
+        Thread.current[REQUEUED_AT_KEY] = time
+        yield
+      ensure
+        Thread.current[REQUEUED_AT_KEY] = previous
+      end
+
+      def initialize(job)
+        super()
+        @job = job
+      end
+
+      def name = @job.class.name
+      def queue = @job.queue_name&.to_s
+      def source_class = @job.class.name
+      def executions = @job.executions
+
+      def adapter
+        klass = @job.class
+        name = klass.queue_adapter_name if klass.respond_to?(:queue_adapter_name)
+        name&.to_s
+      end
+
+      private
+
+      def queued_since
+        requeued = Thread.current[REQUEUED_AT_KEY]
+        return requeued if requeued
+
+        scheduled = @job.respond_to?(:scheduled_at) ? @job.scheduled_at : nil
+        scheduled || @job.enqueued_at
+      end
+    end
+
+    # A Sidekiq job not wrapped by Active Job.
+    class SidekiqTiming < JobTiming
+      def initialize(job, queue, worker)
+        super()
+        @job = job
+        @queue = queue || job["queue"]
+        @worker = worker
+      end
+
+      def name = @job["class"].to_s
+      def adapter = "sidekiq"
+      def queue = @queue&.to_s
+      def source_class = @worker&.class&.name || name
+
+      # retry_count is absent on the first run and 0 on the first retry.
+      def executions
+        retries = @job["retry_count"]
+        retries.is_a?(Integer) ? retries + 2 : 1
+      end
+
+      private
+
+      def queued_since = @job["enqueued_at"]
+    end
+  end
+end

data/lib/goodsorcery/performance/measurement.rb

@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+
+module GoodSorcery
+  module Performance
+    # What integrations and subscribers call while a unit of work runs:
+    # timing a request or job, reaching the fiber's recorder, and linking a
+    # reported error. Extended into GoodSorcery::Performance.
+    module Measurement
+      # Wraps a Rack request (the middleware calls this when active?).
+      def measure_request(env)
+        recorder = start(:web)
+        return yield unless recorder
+
+        recorder.env = env
+        recorder.http_method = env["REQUEST_METHOD"]
+        response = yield
+        recorder.response_status = response.first.to_i if response.is_a?(Array)
+        response
+      rescue Exception => e # rubocop:disable Lint/RescueException
+        recorder&.raised!(exception_status(e))
+        raise
+      ensure
+        finish(recorder) if recorder
+      end
+
+      # Wraps a job execution; +timing+ is a JobTiming.
+      def measure_job(timing)
+        recorder = start(:job)
+        return yield unless recorder
+
+        label_job(recorder, timing)
+        yield
+      rescue Exception # rubocop:disable Lint/RescueException
+        recorder&.raised!
+        raise
+      ensure
+        finish(recorder) if recorder
+      end
+
+      # Yields the active recorder of this fiber; a failure stops recording
+      # this unit of work and is logged by class name.
+      def with_recorder
+        recorder = Thread.current[RECORDER_KEY]
+        return nil if recorder.nil? || recorder.broken?
+
+        yield recorder
+      rescue StandardError => e
+        recorder&.broken!
+        log.warn_every(:apm_subscriber_error, 60,
+                       "performance recording failed (#{e.class}); a unit of work is not measured")
+        nil
+      end
+
+      def http_recorder
+        return nil if Thread.current[SUPPRESS_KEY]
+
+        recorder = Thread.current[RECORDER_KEY]
+        recorder unless recorder.nil? || recorder.broken?
+      end
+
+      def record_http(recorder, started, host, request, response)
+        status = response.respond_to?(:code) ? response.code.to_i : nil
+        recorder.add_http([started, monotonic], request.method, host, request.path, status)
+      rescue StandardError => e
+        recorder.broken!
+        log.warn_every(:apm_http_error, 60, "could not record an HTTP call (#{e.class})")
+      end
+
+      # Client#notify links the first reported error to the unit of work.
+      def note_error(event_id)
+        recorder = Thread.current[RECORDER_KEY]
+        recorder.error_event_id ||= event_id if recorder && event_id
+      end
+
+      private
+
+      # The status Rails' ShowExceptions answers an escaping exception with
+      # (404 for ActiveRecord::RecordNotFound, 400 for ParameterMissing…);
+      # nil outside Rails.
+      def exception_status(error)
+        return nil unless defined?(::ActionDispatch::ExceptionWrapper)
+
+        template_error = defined?(::ActionView::Template::Error) && error.is_a?(::ActionView::Template::Error)
+        error = error.cause || error if template_error
+        ::ActionDispatch::ExceptionWrapper.status_code_for_exception(error.class.name)
+      rescue StandardError
+        nil
+      end
+
+      def label_job(recorder, timing)
+        timing.label(recorder)
+      rescue StandardError => e
+        recorder.broken!
+        log.warn_every(:apm_job_label_error, 60, "could not read job details for performance data (#{e.class})")
+      end
+    end
+
+    # The payloads the :test delivery keeps instead of sending them
+    # (GoodSorcery.test_performance_payloads).
+    module TestPayloads
+      PAYLOADS_LOCK = Mutex.new
+      PAYLOADS = [] # rubocop:disable Style/MutableConstant
+
+      def store_test_payload(path, body)
+        PAYLOADS_LOCK.synchronize { PAYLOADS << { "path" => path, "body" => body } }
+      end
+
+      def test_payloads
+        PAYLOADS_LOCK.synchronize { PAYLOADS.dup }
+      end
+
+      def clear_test_payloads
+        PAYLOADS_LOCK.synchronize { PAYLOADS.clear }
+      end
+    end
+  end
+end

data/lib/goodsorcery/performance/net_http.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module GoodSorcery
+  module Performance
+    # Prepended to Net::HTTP: one "http" span per request made while a unit
+    # of work is recorded (DESIGN.md 8.3). It always calls super and never
+    # rescues the app's exceptions. The recursive call Net::HTTP#request
+    # makes through #start is skipped, and nothing is recorded while
+    # Performance.suppressed? (the gem's own requests).
+    module NetHttp
+      ACTIVE_KEY = :__goodsorcery_http_request
+
+      def self.install
+        ::Net::HTTP.prepend(self) unless installed?
+      end
+
+      def self.installed?
+        defined?(::Net::HTTP) ? ::Net::HTTP.ancestors.include?(self) : false
+      end
+
+      def request(req, body = nil, &)
+        recorder = Thread.current[ACTIVE_KEY] ? nil : Performance.http_recorder
+        return super unless recorder
+
+        Thread.current[ACTIVE_KEY] = true
+        started = Performance.monotonic
+        response = nil
+        begin
+          response = super
+        ensure
+          Thread.current[ACTIVE_KEY] = nil
+          Performance.record_http(recorder, started, address, req, response)
+        end
+      end
+    end
+  end
+end

data/lib/goodsorcery/performance/payloads.rb

@@ -0,0 +1,110 @@
+# frozen_string_literal: true
+
+module GoodSorcery
+  module Performance
+    # Builds the request bodies for POST /api/v1/apm/metrics and
+    # /api/v1/apm/traces (INGEST_API.md › Performance monitoring), split so
+    # each stays under the server's size and count limits. Each body gets a
+    # UUIDv7 batch_id that it keeps on every retry.
+    module Payloads
+      METRICS_PATH = "/api/v1/apm/metrics"
+      TRACES_PATH = "/api/v1/apm/traces"
+      METRICS_MAX_BYTES = 450 * 1_024
+      TRACES_MAX_BYTES = 1_800 * 1_024
+      MAX_TRANSACTIONS_PER_MINUTE = 1_000
+      MAX_TRACES_PER_PAYLOAD = 50
+
+      # One JSON body with its path and how many minutes it carries.
+      Body = Struct.new(:path, :json, :minutes)
+
+      module_function
+
+      # +minutes+ as Aggregator#take returns them.
+      def metrics(minutes)
+        chunks = MetricsChunks.new
+        minutes.each do |minute, kinds|
+          kinds.each do |kind, names|
+            names.each { |name, stats| chunks.add(minute, JSON.generate(stats.to_h(kind, name))) }
+          end
+        end
+        chunks.bodies
+      end
+
+      # Every trace (slowest first, as TraceBuffer#take orders them), in as
+      # few bodies as the size limit and the server's 50 traces per body
+      # allow. A flush can carry more than one minute (the exit flush adds
+      # the open one); the Outbox bounds how many bodies wait.
+      def traces(traces)
+        jsons = traces.map { |trace| JSON.generate(trace) }
+        Packer.pack(jsons, TRACES_MAX_BYTES, MAX_TRACES_PER_PAYLOAD).map do |chunk|
+          Body.new(TRACES_PATH, envelope("traces" => "[#{chunk.join(",")}]"), 0)
+        end
+      end
+
+      def header
+        { "batch_id" => UuidV7.generate, "notifier" => { "name" => NOTIFIER_NAME, "version" => VERSION } }
+      end
+
+      # The envelope as JSON, with already serialised members spliced in.
+      def envelope(fields, extra = {})
+        head = JSON.generate(header.merge(extra))
+        members = fields.map { |key, json| "#{JSON.generate(key)}:#{json}" }
+        "#{head[0..-2]},#{members.join(",")}}"
+      end
+
+      # Greedy packing of JSON strings into groups under a byte budget.
+      module Packer
+        def self.pack(jsons, max_bytes, max_items)
+          jsons.each_with_object([[]]) do |json, chunks|
+            current = chunks.last
+            size = current.sum(&:bytesize) + json.bytesize
+            chunks << [] if !current.empty? && (size > max_bytes || current.size >= max_items)
+            chunks.last << json
+          end.reject(&:empty?)
+        end
+      end
+    end
+
+    # Packs serialised transactions into metrics bodies: a new body when the
+    # current one would pass the size limit or a minute's transaction limit.
+    class MetricsChunks
+      def initialize
+        @bodies = []
+        @current = {}
+        @bytes = 0
+      end
+
+      def add(minute, json)
+        start_new_body if full?(minute, json)
+        (@current[minute] ||= []) << json
+        @bytes += json.bytesize + 1
+      end
+
+      def bodies
+        start_new_body
+        @bodies
+      end
+
+      private
+
+      def full?(minute, json)
+        return false if @current.empty?
+
+        @bytes + json.bytesize > Payloads::METRICS_MAX_BYTES ||
+          @current.fetch(minute, []).size >= Payloads::MAX_TRANSACTIONS_PER_MINUTE
+      end
+
+      def start_new_body
+        return if @current.empty?
+
+        minutes = @current.map do |minute, jsons|
+          "{\"minute\":#{JSON.generate(Performance.minute_iso(minute))},\"transactions\":[#{jsons.join(",")}]}"
+        end
+        json = Payloads.envelope({ "minutes" => "[#{minutes.join(",")}]" }, "histogram" => Histogram::SCHEME)
+        @bodies << Payloads::Body.new(Payloads::METRICS_PATH, json, @current.size)
+        @current = {}
+        @bytes = 0
+      end
+    end
+  end
+end

data/lib/goodsorcery/performance/process_label.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module GoodSorcery
+  module Performance
+    # A short name for the current process, from its title: "Puma worker 3",
+    # "Puma", "Sidekiq", else the program's basename. Nothing else from the
+    # command line is used (it can carry arguments).
+    module ProcessLabel
+      MAX_LENGTH = 255
+
+      def self.current(title = $PROGRAM_NAME)
+        text = title.to_s
+        worker = text[/\Apuma: cluster worker (\d+)/, 1]
+        return "Puma worker #{worker}" if worker
+        return "Puma" if text.start_with?("puma")
+        return "Sidekiq" if text.start_with?("sidekiq")
+
+        Text.limit(File.basename(text.split.first.to_s), MAX_LENGTH)
+      end
+    end
+  end
+end