bulldogger 0.1.0

data/lib/bulldogger/config.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+module Bulldogger
+  # Runtime knobs for capture, redaction, and file output. Each attribute
+  # has a hard-coded default so the tool works with zero setup; the
+  # environment variable overrides exist so an agent running tests in a
+  # child process can change behavior (for example, pointing output_dir
+  # at a scratch directory) without editing the app's own config file.
+  class Config
+    DEFAULT_REDACT_PATTERNS = [
+      /pass(?:word|wd)?/i,
+      /secret/i,
+      /token/i,
+      /api[_-]?key/i,
+      /\bkey\b/i,
+      /credential/i,
+      /auth/i,
+      /session/i,
+      /cookie/i
+    ].freeze
+
+    attr_accessor :enabled, :output_dir, :max_frames, :max_locals,
+                   :max_value_length, :max_pending, :max_samples, :redact_patterns,
+                   :frame_source
+
+    def initialize
+      @enabled = true
+      @output_dir = "tmp/bulldogger"
+      @max_frames = 20
+      @max_locals = 50
+      @max_value_length = 200
+      @max_pending = 32
+      @max_samples = 10
+      @redact_patterns = DEFAULT_REDACT_PATTERNS.dup
+      @frame_source = :auto
+      apply_env_overrides
+    end
+
+    private
+
+    def apply_env_overrides
+      output_dir_override = ENV["BULLDOGGER_OUTPUT_DIR"]
+      @output_dir = output_dir_override if output_dir_override && !output_dir_override.empty?
+
+      case ENV["BULLDOGGER_FRAME_SOURCE"]
+      when "capture_frames" then @frame_source = :capture_frames
+      when "degraded" then @frame_source = :degraded
+      end
+
+      # BULLDOGGER_DISABLE is the documented name; BULLDOGGER_DISABLED is
+      # accepted too. An adjective is the form English speakers reach
+      # for first, and a kill switch that silently ignores the natural
+      # spelling is worse than one that accepts an extra name.
+      @enabled = false if ENV["BULLDOGGER_DISABLE"] == "1" || ENV["BULLDOGGER_DISABLED"] == "1"
+    end
+  end
+end

data/lib/bulldogger/evidence.rb

@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+
+require "json"
+require_relative "version"
+
+module Bulldogger
+  # Assembles and writes one evidence file: the exception, the test it
+  # failed in, and (if the ring still has it) the snapshot captured at
+  # :raise time.
+  #
+  # The exception's message and backtrace are read here, at report
+  # time, not inside the :raise hook. Exception#backtrace can still be
+  # nil when :raise fires -- Ruby fills it in as the exception
+  # unwinds -- so reading it from the hook would record less than what
+  # a test framework's own failure report already has by the time it
+  # calls record_failure.
+  class Evidence
+    SLUG_MAX_LENGTH = 80
+
+    def initialize(config:, run:, capture:)
+      @config = config
+      @run = run
+      @capture = capture
+    end
+
+    def record_failure(exception:, test:)
+      # A disabled switch means "wrote nothing" -- not "wrote an empty
+      # missed record". Writing capture_mode: missed here would mean
+      # a user who turned this off still gets a run directory and a
+      # frames_unavailable_reason, which reads as "tried and failed"
+      # rather than "did not run", the opposite of what they asked for.
+      return nil unless @config.enabled
+      return nil if exception.nil?
+
+      path = @run.next_path(slug_for(test))
+      payload = build_payload(exception: exception, test: test)
+      File.write(path, "#{JSON.pretty_generate(payload)}\n")
+      @run.record(path, test: payload["test"], exception_summary: payload["exception"].slice("class", "message"))
+      path
+    end
+
+    private
+
+    def build_payload(exception:, test:)
+      snapshot = @capture.snapshot_for(exception)
+      payload = {
+        "schema_version" => 1,
+        "tool" => { "name" => "bulldogger", "version" => Bulldogger::VERSION },
+        "captured_at" => Time.now.utc.strftime("%Y-%m-%dT%H:%M:%SZ"),
+        "capture_mode" => snapshot ? snapshot["capture_mode"] : "missed",
+        "test" => normalize_test(test),
+        "exception" => build_exception_section(exception),
+        "frames" => snapshot ? snapshot["frames"] : []
+      }
+      frames_omitted = snapshot ? snapshot["frames_omitted"] : 0
+      payload["frames_omitted"] = frames_omitted if frames_omitted&.positive?
+      payload["frames_unavailable_reason"] = @capture.reason_for_missing(exception) unless snapshot
+      payload["limits"] = limits_section
+      payload
+    end
+
+    def build_exception_section(exception)
+      message = exception.message.to_s
+      limit = @config.max_value_length * 5
+      section = {
+        "class" => exception_class_name(exception),
+        "message" => message.length > limit ? "#{message[0, limit]}…" : message
+      }
+      if message.length > limit
+        section["message_truncated"] = true
+        section["message_original_length"] = message.length
+      end
+      section["backtrace"] = Array(exception.backtrace).first(@config.max_frames)
+      section
+    end
+
+    def exception_class_name(exception)
+      exception.class.name || exception.class.to_s
+    rescue Exception # rubocop:disable Lint/RescueException
+      "Object"
+    end
+
+    def normalize_test(test)
+      test ||= {}
+      {
+        "framework" => test[:framework],
+        "id" => test[:id],
+        "file" => test[:file],
+        "line" => test[:line]
+      }
+    end
+
+    def limits_section
+      {
+        "max_frames" => @config.max_frames,
+        "max_locals" => @config.max_locals,
+        "max_value_length" => @config.max_value_length
+      }
+    end
+
+    def slug_for(test)
+      raw = (test && test[:id] || "unknown").to_s
+      raw.gsub(/[^A-Za-z0-9_-]/, "-")[0, SLUG_MAX_LENGTH]
+    end
+  end
+end

data/lib/bulldogger/formatter.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+module Bulldogger
+  # Turns an arbitrary Ruby value into a bounded, JSON-safe String.
+  # Bounded because a snapshot is written on every failing test, so an
+  # unbounded value (a huge String, a deep object graph) would make
+  # capture itself expensive. Safe because `inspect` is arbitrary user
+  # code: it can raise, and the object it runs on may descend from
+  # BasicObject, where even `#class` is undefined.
+  class Formatter
+    MAX_ELEMENTS = 10
+
+    def initialize(config:, redactor:)
+      @max_value_length = config.max_value_length
+      @redactor = redactor
+    end
+
+    # Returns the per-local entry shape: {"value" => str} normally, plus
+    # "truncated"/"original_length" when the final string was too long.
+    # Those extra keys are the record that something was cut -- they
+    # must never appear on a value that was not, so a reader can trust
+    # their absence.
+    def format(value)
+      truncate_entry(render(value))
+    end
+
+    # Returns just the (possibly truncated) String, for JSON slots that
+    # hold a plain string rather than a {"value"=>...} entry -- a
+    # frame's "self", for example.
+    def format_self(value)
+      format(value)["value"]
+    end
+
+    private
+
+    def render(value)
+      case value
+      when Array
+        render_array(value)
+      when Hash
+        render_hash(value)
+      else
+        safe_inspect(value)
+      end
+    end
+
+    def render_array(array)
+      kept = array.first(MAX_ELEMENTS)
+      parts = kept.map { |element| render_nested(element) }
+      parts << "…" if array.size > kept.size
+      "[#{parts.join(', ')}]"
+    end
+
+    def render_hash(hash)
+      kept = hash.first(MAX_ELEMENTS)
+      parts = kept.map { |key, value| render_pair(key, value) }
+      parts << "…" if hash.size > kept.size
+      "{#{parts.join(', ')}}"
+    end
+
+    def render_pair(key, value)
+      key_repr = render_nested(key)
+      value_repr = @redactor.redact_key?(key) ? '"[REDACTED]"' : render_nested(value)
+      "#{key_repr} => #{value_repr}"
+    end
+
+    # One level of expansion only: an Array/Hash found *inside* an
+    # Array/Hash is shown as a placeholder, not walked further. Without
+    # this, a self-referential or very deep structure could make a
+    # single value's rendering unbounded even with max_value_length
+    # trimming the end result.
+    def render_nested(value)
+      case value
+      when Array then "[…]"
+      when Hash then "{…}"
+      else safe_inspect(value)
+      end
+    end
+
+    def safe_inspect(value)
+      value.inspect
+    rescue Exception => e # rubocop:disable Lint/RescueException
+      "#<#{safe_class_name(value)} (inspect raised #{e.class})>"
+    end
+
+    def safe_class_name(value)
+      klass = value.class
+      klass.respond_to?(:name) ? (klass.name || klass.to_s) : klass.to_s
+    rescue Exception # rubocop:disable Lint/RescueException
+      "Object"
+    end
+
+    def truncate_entry(str)
+      return { "value" => str } if str.length <= @max_value_length
+
+      {
+        "value" => str[0, @max_value_length] + "…",
+        "truncated" => true,
+        "original_length" => str.length
+      }
+    end
+  end
+end

data/lib/bulldogger/frame_source.rb

@@ -0,0 +1,147 @@
+# frozen_string_literal: true
+
+module Bulldogger
+  # Turns a :raise TracePoint event into a bounded array of frame
+  # descriptions. Prefers DEBUGGER__.capture_frames, which yields every
+  # frame's Binding; falls back to TracePoint#binding (raising frame
+  # only) plus the exception's backtrace locations (position only, no
+  # locals) when `debug/frame_info` cannot be loaded. That fallback is
+  # not a rare edge case: `debug` is a bundled gem, not a default gem,
+  # so under Bundler it is only present when the app's own Gemfile asks
+  # for it.
+  class FrameSource
+    def initialize(config:, formatter:, redactor:, skip_path_prefix: self.class.default_skip_path_prefix)
+      @config = config
+      @formatter = formatter
+      @redactor = redactor
+      @skip_path_prefix = skip_path_prefix
+      @mode = nil
+    end
+
+    # skip_path_prefix must be *this library's* lib directory, not the
+    # app's. capture_frames drops every frame whose path starts with
+    # the given prefix; pointing it at the app's own lib directory was
+    # measured to make the app's frames disappear instead of ours,
+    # leaving a snapshot with zero useful frames.
+    def self.default_skip_path_prefix
+      File.expand_path("..", __dir__)
+    end
+
+    # :auto resolves once, here, and is cached for the life of this
+    # object -- not re-checked on every raise, which would repeat a
+    # `require` check thousands of times in a large suite.
+    def resolve!
+      @mode ||= resolve_mode
+      self
+    end
+
+    def mode
+      @mode ||= resolve_mode
+    end
+
+    def capture(tp)
+      mode == :capture_frames ? capture_via_debugger(tp) : capture_degraded(tp)
+    end
+
+    private
+
+    def resolve_mode
+      configured = @config.frame_source
+      return :degraded if configured == :degraded
+
+      # Explicit :capture_frames still needs this require to have run --
+      # it is what defines the DEBUGGER__ constant this class calls
+      # into. Only :degraded can skip it; :auto needs the result to
+      # decide, and explicit :capture_frames needs it as a side effect
+      # even though the caller has already made the decision.
+      available = capture_frames_available?
+      return available ? :capture_frames : :degraded if configured == :auto
+
+      configured
+    end
+
+    def capture_frames_available?
+      require "debug/frame_info"
+      DEBUGGER__.respond_to?(:capture_frames)
+    rescue LoadError
+      false
+    end
+
+    def capture_via_debugger(tp)
+      frame_infos = DEBUGGER__.capture_frames(@skip_path_prefix)
+      kept = frame_infos.first(@config.max_frames)
+      frames = kept.each_with_index.map { |frame_info, index| build_frame(frame_info, index) }
+      [frames, frame_infos.size - kept.size]
+    end
+
+    def build_frame(frame_info, index)
+      location = frame_info.location
+      frame = {
+        "index" => index,
+        "path" => location&.path,
+        "line" => location&.lineno,
+        "label" => frame_info.name,
+        "self" => @formatter.format_self(frame_info.self)
+      }
+      binding = frame_info.binding
+      locals, locals_omitted = binding ? build_locals(binding) : [{}, 0]
+      frame["locals"] = locals
+      frame["locals_omitted"] = locals_omitted if locals_omitted.positive?
+      frame
+    end
+
+    def capture_degraded(tp)
+      locations = degraded_locations(tp)
+      kept = locations.first(@config.max_frames)
+      frames = kept.each_with_index.map { |location, index| build_degraded_frame(tp, location, index) }
+      [frames, locations.size - kept.size]
+    end
+
+    def degraded_locations(tp)
+      locations = tp.raised_exception.backtrace_locations
+      return locations if locations
+
+      # caller_locations here includes this hook's own frames (unlike
+      # backtrace_locations, which is the app's own backtrace and never
+      # contains ours), so they need the same prefix filter.
+      Array(caller_locations).reject { |location| location.path&.start_with?(@skip_path_prefix) }
+    end
+
+    def build_degraded_frame(tp, location, index)
+      frame = {
+        "index" => index,
+        "path" => location.path,
+        "line" => location.lineno,
+        "label" => location.label
+      }
+      if index.zero?
+        locals, locals_omitted = build_frame0_locals(tp)
+        frame["locals"] = locals
+        frame["locals_omitted"] = locals_omitted if locals_omitted.positive?
+        frame["self"] = @formatter.format_self(tp.self)
+      else
+        frame["locals_unavailable"] = true
+      end
+      frame
+    end
+
+    def build_frame0_locals(tp)
+      binding = tp.binding
+      binding ? build_locals(binding) : [{}, 0]
+    end
+
+    def build_locals(binding)
+      names = binding.local_variables
+      kept = names.first(@config.max_locals)
+      locals = {}
+      kept.each { |name| locals[name.to_s] = build_local_entry(name, binding) }
+      [locals, names.size - kept.size]
+    end
+
+    def build_local_entry(name, binding)
+      return { "redacted" => true, "reason" => "name" } if @redactor.redact_name?(name)
+
+      @formatter.format(binding.local_variable_get(name))
+    end
+  end
+end

data/lib/bulldogger/integrations/minitest.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+require "minitest"
+require_relative "../../bulldogger"
+
+module Bulldogger
+  module Minitest
+    # Minitest's own extension point: Minitest.register_plugin with a
+    # Module makes Minitest call #minitest_plugin_init once, during
+    # Minitest.run, at the one moment Minitest.reporter is set (the
+    # accessor is nil outside that window). That is why Bulldogger.start
+    # and the reporter wiring happen here and not at require time --
+    # nothing has run yet, so nothing needs the TracePoint before this.
+    def self.minitest_plugin_init(_options)
+      return if @wired
+
+      @wired = true
+      Bulldogger.start
+      ::Minitest.reporter << Reporter.new
+      ::Minitest.after_run do
+        Bulldogger.finish
+        Bulldogger.stop
+      end
+    end
+
+    # Turns one finished test into a Bulldogger evidence file.
+    # Registered into Minitest's CompositeReporter, so #record runs
+    # once per test method, pass or fail. Minitest.run_one_method
+    # always returns a Minitest::Result, never the Test instance
+    # itself, so the test's id/file/line come from Result's own
+    # klass/name/source_location rather than from re-deriving them.
+    class Reporter < ::Minitest::AbstractReporter
+      def record(result)
+        failure = result.failure
+        return if failure.nil? || result.skipped?
+
+        exception = exception_for(failure)
+        path = Bulldogger.record_failure(exception: exception, test: test_for(result))
+        annotate!(failure, path) if path
+      end
+
+      private
+
+      # UnexpectedError wraps whatever the app actually raised;
+      # #error unwraps it, and is defined to return self on a plain
+      # Assertion, so this one line also covers assertion failures
+      # without a separate branch. The wrapper itself is only ever
+      # `.new`'d, never `raise`'d, so :raise only fires for the inner
+      # exception -- snapshot_for is still tried on both, in that
+      # order, so nothing breaks if a future Minitest version starts
+      # re-raising the wrapper instead.
+      def exception_for(failure)
+        inner = failure.respond_to?(:error) ? failure.error : failure
+        return inner if Bulldogger.snapshot_for(inner)
+        return failure if Bulldogger.snapshot_for(failure)
+
+        inner
+      end
+
+      def test_for(result)
+        file, line = result.source_location
+        {
+          framework: "minitest",
+          id: "#{result.class_name}##{result.name}",
+          file: file,
+          line: line
+        }
+      end
+
+      # First choice: tag the failure exception's own #message, so the
+      # one line sits inside the same text Minitest's SummaryReporter
+      # already prints for this failure. A raised exception can be
+      # frozen (a re-raised literal, an app that freezes its errors),
+      # and #define_singleton_method on a frozen object raises --
+      # fall back to printing the line ourselves so a frozen exception
+      # never means a silently missing evidence line.
+      def annotate!(failure, path)
+        original = failure.message
+        failure.define_singleton_method(:message) { "#{original}\nbulldogger evidence: #{path}" }
+      rescue FrozenError
+        $stdout.puts "bulldogger evidence: #{path}"
+      end
+    end
+  end
+end
+
+::Minitest.register_plugin(Bulldogger::Minitest)

data/lib/bulldogger/integrations/rspec.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require "rspec/core"
+require_relative "../../bulldogger"
+
+module Bulldogger
+  module RSpec
+    # RSpec has already scored the example's description and its
+    # file/line into metadata by the time after(:each) runs; test_for
+    # reads those off the Example instead of re-deriving them.
+    def self.test_for(example)
+      {
+        framework: "rspec",
+        id: example.full_description,
+        file: example.metadata[:file_path],
+        line: example.metadata[:line_number]
+      }
+    end
+
+    # Mirrors the minitest integration's annotate!: tag the exception's
+    # own #message first, so the evidence line sits inside the same
+    # text RSpec's own formatter already prints for this failure. A
+    # frozen exception can't take a singleton method; fall back to
+    # printing the line directly so a frozen exception never means a
+    # silently missing evidence line.
+    def self.annotate!(exception, path)
+      original = exception.message
+      exception.define_singleton_method(:message) { "#{original}\nbulldogger evidence: #{path}" }
+    rescue FrozenError
+      $stdout.puts "bulldogger evidence: #{path}"
+    end
+  end
+end
+
+::RSpec.configure do |config|
+  config.before(:suite) { Bulldogger.start }
+
+  # after(:each), not a formatter hook: example.exception is already
+  # set by the time this runs (Example#run assigns it before
+  # run_after_example fires the after(:each) hooks), and evidence
+  # must be written -- and the exception's #message annotated -- before
+  # RSpec's own formatters render the failure, or the added line would
+  # never reach the user's terminal.
+  config.after(:each) do |example|
+    exception = example.exception
+    next unless exception
+
+    path = Bulldogger.record_failure(exception: exception, test: Bulldogger::RSpec.test_for(example))
+    Bulldogger::RSpec.annotate!(exception, path) if path
+  end
+
+  config.after(:suite) do
+    Bulldogger.finish
+    Bulldogger.stop
+  end
+end

data/lib/bulldogger/minitest.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+# The one line a test_helper.rb adds to turn Bulldogger on for
+# Minitest. Kept separate from lib/bulldogger/integrations/minitest.rb
+# so that file can stay organized by framework alongside rspec.rb,
+# while this stays the short, memorable require path.
+require_relative "integrations/minitest"

data/lib/bulldogger/pending.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module Bulldogger
+  # Insertion-ordered, bounded map from an exception object to its
+  # captured snapshot. Bounded because most raises are caught and
+  # handled by the app and never become a test failure -- without a
+  # cap, a suite that raises-and-rescues heavily would grow this map
+  # without limit. Matches by object identity (`equal?`), not `hash`/
+  # `eql?`, so an exception class that overrides those can't collide
+  # with an unrelated exception instance.
+  #
+  # ObjectSpace::WeakMap was considered and rejected: its eviction is
+  # driven by GC timing, which we cannot observe or bound, and its
+  # membership semantics would need the same identity-matching logic
+  # this class already provides directly.
+  class Pending
+    Entry = Struct.new(:exception, :snapshot)
+
+    def initialize(max_size)
+      @max_size = max_size
+      @entries = []
+      @evicted = []
+      @mutex = Mutex.new
+    end
+
+    # First-write-wins: a re-raised exception (rescue; raise) fires
+    # :raise again from the rescue frame. That second capture is worth
+    # less than the first -- it points at the handler, not the bug --
+    # so an exception already in the ring keeps its original snapshot.
+    def put(exception, snapshot)
+      @mutex.synchronize do
+        next if find(exception)
+
+        @entries << Entry.new(exception, snapshot)
+        next unless @entries.size > @max_size
+
+        evicted_entry = @entries.shift
+        @evicted << evicted_entry.exception
+        @evicted.shift if @evicted.size > @max_size
+      end
+      nil
+    end
+
+    def get(exception)
+      @mutex.synchronize { find(exception)&.snapshot }
+    end
+
+    def evicted?(exception)
+      @mutex.synchronize { @evicted.any? { |e| e.equal?(exception) } }
+    end
+
+    private
+
+    def find(exception)
+      @entries.find { |entry| entry.exception.equal?(exception) }
+    end
+  end
+end