devbench 0.6.0 → 0.8.0

data/lib/devbench/logs.rb

@@ -4,7 +4,7 @@ require 'logger'
 require_relative 'current'
 
 module Devbench
-  # In-process log capture (docs/SERVER_SDK_SPEC.md, "In-process log capture
+  # In-process log capture (docs/server-sdk/configuration.md, "In-process log capture
   # (direct mode)"; DECISIONS #161): recent log lines kept in memory, keyed
   # by the trace that was current when each was written, so triage can be
   # handed the server lines of one user action without a sidecar.
@@ -28,6 +28,10 @@ module Devbench
     MAX_BYTES = 4 * 1024 * 1024
     MAX_AGE = 15 * 60
     MAX_LINE_BYTES = 4096
+    # Lines held from one multi-line message (an exception and its
+    # backtrace): the top of it, so one framework backtrace cannot push the
+    # rest of the request out of a 200-line slice.
+    MAX_MESSAGE_LINES = 50
 
     SEVERITY = %w[DEBUG INFO WARN ERROR FATAL].freeze
     # The last line captured on this thread, and by which hook: see #capture.
@@ -149,7 +153,7 @@ module Devbench
         if message.nil?
           message = block_given? ? yield : progname
         end
-        Logs.capture(severity, message, :broadcast)
+        Logs.capture(severity, message, :broadcast, Logs.broadcast_tags(devbench_broadcast, self))
         true
       rescue StandardError, SystemStackError
         true
@@ -157,7 +161,7 @@ module Devbench
       alias log add
 
       def <<(message)
-        Logs.capture(nil, message, :broadcast) unless Current.trace.nil?
+        Logs.capture(nil, message, :broadcast, Logs.broadcast_tags(devbench_broadcast, self)) unless Current.trace.nil?
         self
       rescue StandardError, SystemStackError
         self
@@ -257,6 +261,17 @@ module Devbench
         []
       end
 
+      # The tag prefixes of the app's loggers in a broadcast (not ours): the
+      # one Rails wrote after each newline of a multi-line message is the
+      # first broadcast's, but which logger is first is the app's choice.
+      # A Proc, so a single-line message never computes it.
+      def broadcast_tags(broadcast, own)
+        lambda do
+          loggers = broadcast&.broadcasts || []
+          loggers.reject { |l| l.equal?(own) }.filter_map { |l| tags_of(l) }
+        end
+      end
+
       # Called by Tee after the logger's own #add returned.
       def tee(logger, severity, message, progname, had_block, yielded, value)
         severity ||= ::Logger::UNKNOWN
@@ -267,32 +282,38 @@ module Devbench
 
           message = had_block ? value : progname
         end
-        capture(severity, message, :tee)
+        capture(severity, message, :tee, -> { tags_of(logger) })
       rescue StandardError, SystemStackError
         nil
       end
 
-      # Keeps one line under the current trace. Never raises.
+      # Keeps one message under the current trace — one entry per line of
+      # it (see #lines_for). Never raises. `tags` is the tag prefix of the
+      # logger written to (an Array of candidates, or a Proc giving one),
+      # needed only for a multi-line message.
       #
       # One line can reach both hooks: in a Sidekiq process Rails.logger may
       # broadcast to Sidekiq.logger. Both run on this thread, one right
       # after the other, so a line equal to the one just captured *by the
       # other hook* is that same write and is skipped. The same hook
       # repeating a line is a real repeat and is kept.
-      def capture(severity, message, source)
+      def capture(severity, message, source, tags = nil)
         return nil unless active?
 
         trace = Current.trace
         return nil if trace.nil?
 
-        line = line_for(trace, severity, message)
+        lines = lines_for(trace, severity, message, tags)
+        return nil if lines.empty?
+
+        written = lines.join("\n")
         last = Thread.current[LAST]
-        if last && last[0] != source && last[1] == line
+        if last && last[0] != source && last[1] == written
           Thread.current[LAST] = nil
           return nil
         end
-        Thread.current[LAST] = [source, line]
-        buffer.push(trace.key, line)
+        Thread.current[LAST] = [source, written]
+        lines.each { |line| buffer.push(trace.key, line) }
         start_polling if @polling_pid != Process.pid
         nil
       rescue StandardError, SystemStackError, ThreadError
@@ -310,11 +331,61 @@ module Devbench
         transport.start if transport.respond_to?(:start)
       end
 
-      # "[v1/s/i/h] INFO message", at most MAX_LINE_BYTES, valid UTF-8.
-      def line_for(trace, severity, message)
+      # "[a] [b] " — what ActiveSupport::TaggedLogging puts before a line,
+      # and what ActionDispatch::DebugExceptions puts after every newline of
+      # the exception it logs as one message. nil when the logger has none.
+      def tags_of(logger)
+        formatter = logger.respond_to?(:formatter) ? logger.formatter : nil
+        return nil unless formatter.respond_to?(:tags_text)
+
+        text = formatter.tags_text.to_s
+        text.empty? ? nil : text
+      end
+
+      # "[v1/s/i/h] INFO message", at most MAX_LINE_BYTES, valid UTF-8 —
+      # one per line of the message. A multi-line message (Rails logs an
+      # exception and its backtrace as one) is split so each line reaches
+      # triage on its own, with the logger's tags removed from the start of
+      # continuation lines (the trace frames each line once) and blank
+      # lines dropped. At most MAX_MESSAGE_LINES; when more, the last says
+      # how many were left out.
+      def lines_for(trace, severity, message, tags)
         label = severity.is_a?(Integer) ? SEVERITY[severity] : nil
+        prefix = label ? "[#{trace}] #{label} " : "[#{trace}] "
         text = message_text(message)
-        line = label ? "[#{trace}] #{label} #{text}" : "[#{trace}] #{text}"
+        text = text.dup.force_encoding(Encoding::UTF_8) unless text.encoding == Encoding::UTF_8
+        text = text.scrub('') unless text.valid_encoding?
+        return [bound_line(prefix + text)] unless text.include?("\n")
+
+        prefixes = tag_prefixes(tags)
+        parts = text.split("\n").each_with_index.filter_map do |part, i|
+          part = untag(part, prefixes) if i.positive?
+          part = part.chomp("\r")
+          part unless part.strip.empty?
+        end
+        return [] if parts.empty?
+
+        if parts.length > MAX_MESSAGE_LINES
+          omitted = parts.length - (MAX_MESSAGE_LINES - 1)
+          parts = parts.first(MAX_MESSAGE_LINES - 1) << "... (#{omitted} more lines)"
+        end
+        parts.map { |part| bound_line(prefix + part) }
+      end
+
+      # A logger whose formatter fails to give its tags costs the untagging,
+      # never the line.
+      def tag_prefixes(tags)
+        Array(tags.respond_to?(:call) ? tags.call : tags).select { |t| t.is_a?(::String) && !t.empty? }
+      rescue StandardError, SystemStackError
+        []
+      end
+
+      def untag(part, prefixes)
+        found = prefixes.find { |p| part.start_with?(p) }
+        found ? part[found.length..] : part
+      end
+
+      def bound_line(line)
         line = line.byteslice(0, MAX_LINE_BYTES) if line.bytesize > MAX_LINE_BYTES
         line = line.dup.force_encoding(Encoding::UTF_8) unless line.encoding == Encoding::UTF_8
         line.valid_encoding? ? line : line.scrub('')

data/lib/devbench/middleware.rb

@@ -3,10 +3,13 @@
 require_relative 'trace'
 require_relative 'current'
 require_relative 'reporter'
+require_relative 'request_signals'
 
 module Devbench
   # Rack middleware: accepts the correlation id and makes it available for the
-  # duration of the request, and reports exceptions that escape the app.
+  # duration of the request, reports exceptions that escape the app, and
+  # reports what a request does without raising (Capability 2b: a 5xx with
+  # no exception, a request over 3 s, a route's 429 spike; RequestSignals).
   #
   # Never rejects a request, never alters the response body, and never raises
   # anything of its own. An exception from the application is reported and
@@ -19,6 +22,10 @@ module Devbench
   class Middleware
     HANDLED_HEADER = 'x-adt-handled'
     EXPOSE_HEADER = 'Access-Control-Expose-Headers'
+    # Set while the outermost instance observes a request, so a second
+    # instance in the stack (an app that inserted it twice, an engine's own)
+    # does not count the request twice.
+    OBSERVING_KEY = :__adt_observing_request
 
     def initialize(app)
       @app = app
@@ -34,23 +41,31 @@ module Devbench
         nil
       end
 
+      observing = Thread.current[OBSERVING_KEY] != true
+      started = observing ? RequestSignals.clock.monotonic_ms : nil
+
       # Scoped even without a trace: identity set during this request must
       # end with it, and a thread's next request must not inherit it.
       status, headers, body = Current.with(trace) do
+        Thread.current[OBSERVING_KEY] = true if observing
         result = begin
           @app.call(env)
         rescue Exception => e # rubocop:disable Lint/RescueException
           # Reported inside the scope so the report carries this request's
           # trace and user. capture never raises; if it somehow did, the
           # application's exception would be lost, which is the one outcome
-          # this must not have.
+          # this must not have. A request that raised is not evaluated as a
+          # request signal: the exception is its report.
           report(e, env)
           raise e
+        ensure
+          Thread.current[OBSERVING_KEY] = nil if observing
         end
 
         # Rails' ShowExceptions turns an exception into a 500 page itself, so
         # nothing escapes to here. It leaves the exception in env.
         report(env['action_dispatch.exception'], env)
+        observe(env, result, started) if observing
 
         trace.nil? ? result : mark_handled(result)
       end
@@ -60,6 +75,47 @@ module Devbench
 
     private
 
+    # Classifies the finished request (failed / slow / nothing) and feeds the
+    # 429 spike detector. Inside the request's scope, so a signal carries its
+    # trace and user. Duration runs from entry to the application handing
+    # back its response; streaming a body after that is not counted.
+    def observe(env, result, started)
+      finished = RequestSignals.clock
+      duration_ms = (finished.monotonic_ms - started).floor
+      status, headers, = result
+      RequestSignals.observe(
+        status: status.is_a?(Integer) ? status : Integer(status.to_s, 10),
+        method: env['REQUEST_METHOD'],
+        route: symbol(env),
+        duration_ms: duration_ms,
+        streaming: streaming?(env, status.to_i, headers),
+        # Rails rendering a 500 itself is an exception: reported, or one the
+        # app chose to ignore (Devbench.ignore_exceptions), which must not
+        # come back as an http_500 either.
+        exception_reported: Current.exception_reported? || !env['action_dispatch.exception'].nil?,
+        finished_ms: finished.wall_ms
+      )
+    rescue StandardError, SystemStackError
+      # Never fail a response over instrumentation.
+      nil
+    end
+
+    # A 101 upgrade, a hijacked connection (a websocket takes the socket) or
+    # a text/event-stream response: its duration is a connection's lifetime,
+    # never a slow request.
+    def streaming?(env, status, headers)
+      return true if status == 101 || !env['rack.hijack_io'].nil?
+      return false unless headers.respond_to?(:each)
+
+      streaming = false
+      headers.each do |name, value|
+        key = name.to_s.downcase
+        streaming ||= key == 'rack.hijack'
+        streaming ||= key == 'content-type' && value.to_s.strip.downcase.start_with?('text/event-stream')
+      end
+      streaming
+    end
+
     def report(error, env)
       return if error.nil?
 

data/lib/devbench/rails_hooks.rb

@@ -3,7 +3,7 @@
 require_relative 'reporter'
 
 module Devbench
-  # The Rails-side exception hooks (docs/SERVER_SDK_SPEC.md, Capability 2).
+  # The Rails-side exception hooks (docs/server-sdk/unhandled-failures.md, Capability 2).
   #
   # Plain Ruby: nothing here names a Rails constant, so this file loads
   # without Rails. Devbench::Railtie installs the hooks against the real

data/lib/devbench/railtie.rb

@@ -3,6 +3,7 @@
 require_relative 'middleware'
 require_relative 'rails_hooks'
 require_relative 'sidekiq_hooks'
+require_relative 'active_job_hooks'
 require_relative 'view_helper'
 
 module Devbench
@@ -29,7 +30,7 @@ module Devbench
       Railtie.auto_insert(app.config.middleware)
     end
 
-    # Puts the request's trace on every Rails log line (SERVER_SDK_SPEC,
+    # Puts the request's trace on every Rails log line (docs/server-sdk/trace-propagation.md,
     # Capability 1, requirement 2), so the sidecar can hand triage the server
     # lines of one user action. Appended to whatever the app already tags
     # with — e.g. [:request_id] — never replacing it. Before the middleware
@@ -65,6 +66,11 @@ module Devbench
         if defined?(::ActiveSupport::Notifications)
           Devbench::RailsHooks.install_active_job(::ActiveSupport::Notifications)
         end
+        # The request's trace into ActiveJob jobs, on any adapter. on_load
+        # runs now if ActiveJob::Base is already loaded, else when it loads.
+        if defined?(::ActiveSupport) && ::ActiveSupport.respond_to?(:on_load)
+          ::ActiveSupport.on_load(:active_job) { Devbench::ActiveJobHooks.install(self) }
+        end
         # Native Sidekiq jobs never reach the ActiveJob hook. after_initialize
         # runs after every gem is required, so Gemfile order does not matter.
         Devbench::SidekiqHooks.install if defined?(::Sidekiq)

data/lib/devbench/reporter.rb

@@ -6,7 +6,7 @@ require_relative 'transport'
 
 module Devbench
   # Turns a failure into a report — the control message of
-  # docs/SERVER_SDK_SPEC.md, "Control socket — v1" — and hands it to the
+  # docs/server-sdk/control-socket.md, "Control socket — v1" — and hands it to the
   # active transport (Devbench.transport): direct to Dev Bench when a DSN is
   # set, else the local sidecar's socket.
   #
@@ -19,7 +19,7 @@ module Devbench
   module Reporter
     DEFAULT_SOCKET = '/tmp/adt-sidecar.sock'
 
-    # docs/SERVER_SDK_SPEC.md, "Control socket — v1".
+    # docs/server-sdk/control-socket.md, "Control socket — v1".
     MAX_MESSAGE = 2000
     MAX_HANDLED_MESSAGE = 500
 
@@ -111,7 +111,13 @@ module Devbench
       # would replace the application's own exception, so nothing is.
       def capture(error, context:, handled:, symbol: nil)
         return nil unless error.is_a?(Exception)
-        return nil if reported?(error) || ignored?(error)
+        return nil if ignored?(error)
+
+        # The request (or job) this happened in has reported an exception,
+        # whichever hook sent it first: a 5xx after it is not also a
+        # `failed` request (Capability 2b).
+        note_exception
+        return nil if reported?(error)
 
         # Marked before sending, so a second hook reached while this one is
         # still writing does not send it again.
@@ -151,6 +157,12 @@ module Devbench
 
       private
 
+      def note_exception
+        Current.record_exception if Current.scoped?
+      rescue StandardError, SystemStackError
+        nil
+      end
+
       def mark_reported(error)
         error.instance_variable_set(REPORTED_IVAR, true)
       rescue FrozenError
@@ -177,7 +189,7 @@ module Devbench
       # message is detail.
       def handled_payload(error, symbol, reason)
         {
-          v: 1,
+          v: PROTOCOL_VERSION,
           kind: 'handled_failure',
           symbol: symbol,
           error: safely { error.class.name },
@@ -190,7 +202,7 @@ module Devbench
 
       def exception_payload(error, context, handled, symbol)
         {
-          v: 1,
+          v: PROTOCOL_VERSION,
           kind: 'exception',
           context: CONTEXTS.include?(context) ? context : 'explicit',
           handled: handled ? true : false,