saboteur 0.2.0 → 0.4.0

data/lib/saboteur/frameworks/rails.rb

@@ -1,11 +1,15 @@
 # SPDX-License-Identifier: MIT
 # frozen_string_literal: true
 
+require "coverage"
 require "fileutils"
+require "pathname"
+require "English"
+require "tmpdir"
 
 module Saboteur
   module Frameworks
-    # Opt-in Rails support (`rails: true` in .saboteur.yml).
+    # Opt-in Rails support (`c.rails = true` in config/saboteur.rb).
     #
     # Refuses to run unless RAILS_ENV is test and every configured database is
     # local, eager loads the app so every subject's owner is defined before
@@ -14,9 +18,23 @@ module Saboteur
     class Rails
       LOCAL_HOSTS = ["localhost", "127.0.0.1", "::1", "[::1]"].freeze
 
+      # How many blocking session names the template-copy error lists.
+      MAX_BLOCKER_NAMES = 5
+
+      # sqlite3-ruby's Backup#step returns raw SQLite status codes instead
+      # of raising, and finish discards its own: without an explicit check
+      # a partial destination would pass as a copy.
+      SQLITE_DONE = 101
+      SQLITE_BUSY = 5
+      SQLITE_LOCKED = 6
+      BACKUP_ATTEMPTS = 5
+      BACKUP_RETRY_DELAY = 0.05
+
       def initialize(config, hooks)
         @config = config
         @hooks = hooks
+        @worker_copies = []
+        @memory_snapshots = {}
       end
 
       def before_boot
@@ -27,18 +45,31 @@ module Saboteur
         # on macOS that goes through XPC, which crashes a forked process. The
         # databases are local only (check_databases), so it buys nothing.
         ENV["PGGSSENCMODE"] ||= "disable"
+
+        # Bytecode from Bootsnap's compile cache carries no coverage counters,
+        # so a file served from the cache never appears in Coverage.result.
+        # Turn the cache off for this process before config/boot loads
+        # Bootsnap; the load-path cache still works. Bootsnap checks
+        # DISABLE_<feature> by presence, so a value the user set disables the
+        # cache too. Files loaded during the eager load are also covered by
+        # eager_load_with_coverage_running.
+        ENV["DISABLE_BOOTSNAP_COMPILE_CACHE"] ||= "1"
       end
 
       def after_boot
         unless defined?(::Rails.env)
           raise Error,
-                "rails: true but Rails did not load (add config/environment to require)"
+                "rails: true but Rails did not load (add config/environment to require - for an engine, " \
+                "its dummy app's environment, such as test/dummy/config/environment.rb; only a dynamically " \
+                "created dummy app has no environment file, and there you require the helper that calls " \
+                "setup instead, such as DummyApp.setup in spec/rails_helper)"
         end
         raise Error, "rails integration refuses to run: Rails.env is #{::Rails.env}" unless ::Rails.env.test?
 
         check_databases
-        ::Rails.application.eager_load!
+        eager_load_with_coverage_running
         disconnect
+        snapshot_memory_databases
         @hooks.register(:before_fork) { |_mutation| disconnect }
         @hooks.register(:after_fork) { reconnect }
       end
@@ -54,16 +85,58 @@ module Saboteur
       end
 
       # Parent, before workers start: one copy of each test database per
-      # extra worker, made from the prepared test database.
+      # extra worker, made from the prepared test database. Every attempted
+      # copy is tracked before it runs, so cleanup_workers drops them all,
+      # including the partial set when a copy failed.
       def prepare_workers(jobs)
         return unless defined?(::ActiveRecord::Base) && jobs > 1
 
+        @worker_copies = []
         test_configs.each do |db|
-          (1...jobs).each { |number| copy_database(db, self.class.worker_database(db.database, number)) }
+          # An in-memory SQLite database lives in the parent's connection;
+          # workers inherit it through fork, so there is nothing to copy.
+          next if self.class.memory_database?(db)
+
+          (1...jobs).each do |number|
+            name = self.class.worker_database(db.database, number)
+            # Track the target before the copy, not after it: an attempt
+            # that created the database and then failed (a post-CREATE
+            # error, a partial SQLite copy) must still be dropped at the
+            # end of the run. Dropping is idempotent (DROP IF EXISTS,
+            # rm -f), so tracking a copy that never happened costs nothing.
+            @worker_copies << [db, name]
+            copy_database(db, name)
+          end
         end
         disconnect
       end
 
+      # Parent, after the run: drop every copy prepare_workers made, so runs
+      # do not collect one database per worker. Every copy is attempted even
+      # when a drop fails; the failures are reported together at the end.
+      def cleanup_workers
+        return if @worker_copies.empty? && @memory_snapshot_dir.nil?
+
+        failures = @worker_copies.filter_map do |db, name|
+          drop_database(db, name)
+          nil
+        rescue StandardError => e
+          "#{name}: #{e.message}"
+        end
+        @worker_copies = []
+        if @memory_snapshot_dir
+          FileUtils.rm_rf(@memory_snapshot_dir)
+          @memory_snapshot_dir = nil
+          @memory_snapshots = {}
+        end
+        begin
+          disconnect
+        rescue StandardError => e
+          failures << "disconnect: #{e.message}"
+        end
+        raise Error, "could not drop worker database copies: #{failures.join('; ')}" if failures.any?
+      end
+
       # Worker: point every connection pool (primary, and any cache, queue or
       # connects_to databases, for every role and shard) at this worker's
       # copy of its database.
@@ -74,11 +147,16 @@ module Saboteur
         handler.connection_pool_list(:all).to_a.each do |pool|
           config = pool.pool_config
           db = pool.db_config
+          # An in-memory SQLite database would be discarded by
+          # re-establishing: the worker keeps its own connection.
+          next if self.class.memory_database?(db)
+
           handler.establish_connection(db.configuration_hash.merge(database: self.class.worker_database(db.database,
                                                                                                         number)),
                                        owner_name: self.class.owner_of(config), role: config.role,
                                        shard: config.shard)
         end
+        restore_memory_databases
       end
 
       def self.owner_of(pool_config)
@@ -87,12 +165,35 @@ module Saboteur
         Object.const_get(pool_config.connection_descriptor.name)
       end
 
+      # Only SQLite can be in-memory; every other adapter names a server.
+      def self.memory_database?(db)
+        db.adapter.to_s.match?(/sqlite/) && db.database.to_s == ":memory:"
+      end
+
       def self.local_host?(host)
         host.nil? || host.to_s.empty? || host.to_s.start_with?("/") || LOCAL_HOSTS.include?(host.to_s.downcase)
       end
 
       private
 
+      # Bootsnap before 1.25 skips its bytecode cache only while Coverage is
+      # running (1.25 and later skip it whenever Coverage is set up, suspended
+      # or not), and Coverage is suspended while the app boots (exe/saboteur
+      # sets it up suspended). Cached bytecode carries no coverage counters.
+      # Resume Coverage just for the eager load, so every app file compiles
+      # with counters. The coverage recorder clears the counts before the
+      # suite runs, so load-time hits are never credited to a test.
+      def eager_load_with_coverage_running
+        return ::Rails.application.eager_load! unless Coverage.state == :suspended
+
+        Coverage.resume
+        begin
+          ::Rails.application.eager_load!
+        ensure
+          Coverage.suspend
+        end
+      end
+
       def check_databases
         return unless defined?(::ActiveRecord::Base)
 
@@ -118,29 +219,157 @@ module Saboteur
           disconnect
           copy_postgres(db, name)
         when /sqlite/
-          # A database in WAL mode keeps recent writes in -wal/-shm files.
-          ["", "-wal", "-shm"].each do |suffix|
-            FileUtils.cp("#{db.database}#{suffix}", "#{name}#{suffix}") if File.exist?("#{db.database}#{suffix}")
-          end
+          copy_sqlite(db, name)
         else
           raise Error, "--jobs > 1 with Rails needs Postgres or SQLite (got #{db.adapter})"
         end
       end
 
+      # The mirror of copy_database. Adapters copy_database rejects never get
+      # copies, so there is nothing to drop for them.
+      def drop_database(db, name)
+        case db.adapter
+        when /postg/
+          disconnect
+          drop_postgres(db, name)
+        when /sqlite/
+          target = sqlite_absolute(name)
+          ["", "-wal", "-shm"].each { |suffix| FileUtils.rm_f("#{target}#{suffix}") }
+        end
+      end
+
+      # A relative sqlite path means something only to Rails: 7.1+ resolves
+      # it against Rails.root, not the process working directory. Resolve
+      # the source the same way, checkpoint so a live WAL database holds
+      # every committed write in its main file, then snapshot it through
+      # the backup API. A missing source is an error, not a skip: otherwise
+      # the worker starts from a fresh empty database and silently tests
+      # nothing.
+      def copy_sqlite(db, name)
+        source = sqlite_absolute(db.database)
+        unless File.exist?(source)
+          raise Error, "test database #{source} does not exist; run the test suite once to create it"
+        end
+
+        checkpoint_sqlite(source)
+        backup_sqlite(source, sqlite_absolute(name))
+      end
+
+      def sqlite_absolute(database)
+        return database if Pathname.new(database).absolute?
+        return database unless defined?(::Rails)
+
+        File.expand_path(database, ::Rails.root)
+      end
+
+      # wal_checkpoint(TRUNCATE) reports instead of raising: [busy, frames
+      # in the WAL, frames checkpointed]. A busy reader or an incomplete
+      # pass leaves committed writes outside the main file, and copying on
+      # would silently miss them - fail closed instead. Through a
+      # connection of saboteur's own, so the app's connections never point
+      # at the database being checkpointed.
+      def checkpoint_sqlite(source)
+        with_sqlite(source) do |database|
+          busy, frames, checkpointed = database.execute("PRAGMA wal_checkpoint(TRUNCATE)").first
+          return if busy.zero? && frames == checkpointed
+
+          raise Error, "could not checkpoint #{source} before copying it " \
+                       "(busy: #{busy}, frames: #{frames}, checkpointed: #{checkpointed}); " \
+                       "close the connections holding it and run again"
+        end
+      rescue SQLite3::BusyException => e
+        raise Error, "could not checkpoint #{source} before copying it (#{e.message}); " \
+                     "close the connections holding it and run again"
+      end
+
+      # A byte copy of main + WAL + SHM is no snapshot: a write landing
+      # between the file copies leaves the set malformed. The backup API
+      # copies a consistent image and restarts when the source changes
+      # under it. The target starts clean so stale sidecars from an
+      # earlier byte copy cannot shadow the fresh image.
+      def backup_sqlite(source, target)
+        ["", "-wal", "-shm"].each { |suffix| FileUtils.rm_f("#{target}#{suffix}") }
+        with_sqlite(source, readonly: true) do |src|
+          with_sqlite(target) do |dst|
+            backup = SQLite3::Backup.new(dst, "main", src, "main")
+            begin
+              step_backup(backup, target)
+            ensure
+              backup.finish
+            end
+          end
+        end
+      end
+
+      # BUSY and LOCKED earn a bounded retry; DONE means the target holds
+      # every page; anything else fails the copy.
+      def step_backup(backup, target)
+        BACKUP_ATTEMPTS.times do
+          case (status = backup.step(-1))
+          when SQLITE_DONE then return
+          when SQLITE_BUSY, SQLITE_LOCKED
+            sleep(BACKUP_RETRY_DELAY)
+          else
+            raise Error, "could not copy the database to #{target}: sqlite backup failed with status #{status}"
+          end
+        end
+        raise Error, "could not copy the database to #{target}: still busy after #{BACKUP_ATTEMPTS} attempts"
+      end
+
+      def with_sqlite(path, **)
+        require "sqlite3"
+
+        database = SQLite3::Database.new(path, **)
+        begin
+          yield database
+        ensure
+          database.close
+        end
+      end
+
       # Through a connection class of saboteur's own, so the app's
       # connections never point at the maintenance database.
       def copy_postgres(db, name)
-        admin = maintenance_record
-        admin.establish_connection(db.configuration_hash.merge(database: "postgres"))
-        admin.connection_pool.with_connection do |connection|
+        with_maintenance_connection(db) do |connection|
           connection.execute("DROP DATABASE IF EXISTS #{connection.quote_table_name(name)}")
+          refuse_busy_template(connection, db.database)
           connection.execute("CREATE DATABASE #{connection.quote_table_name(name)} " \
                              "TEMPLATE #{connection.quote_table_name(db.database)}")
         end
+      end
+
+      def drop_postgres(db, name)
+        with_maintenance_connection(db) do |connection|
+          connection.execute("DROP DATABASE IF EXISTS #{connection.quote_table_name(name)}")
+        end
+      end
+
+      # Through a connection class of saboteur's own, so the app's
+      # connections never point at the maintenance database.
+      def with_maintenance_connection(db, &)
+        admin = maintenance_record
+        admin.establish_connection(db.configuration_hash.merge(database: "postgres"))
+        admin.connection_pool.with_connection(&)
       ensure
         admin&.remove_connection
       end
 
+      # Postgres refuses CREATE DATABASE ... TEMPLATE while any other session
+      # is connected to the template, so name the blockers instead of dying
+      # mid-run with the raw error. Never terminates other sessions.
+      def refuse_busy_template(connection, database)
+        blockers = connection.select_values(<<~SQL)
+          SELECT application_name FROM pg_stat_activity
+          WHERE datname = #{connection.quote(database)} AND pid <> pg_backend_pid()
+        SQL
+        return if blockers.empty?
+
+        names = blockers.map { |app| app.to_s.empty? ? "unnamed" : app }.first(MAX_BLOCKER_NAMES)
+        raise Error, "cannot copy the test database #{database}: #{blockers.size} other session(s) connected " \
+                     "(#{names.join(', ')})#{', ...' if blockers.size > names.size}. " \
+                     "Stop other test runs, consoles or Spring, then run saboteur again."
+      end
+
       # ActiveRecord names a connection's owner by class name, so the class
       # needs a constant.
       def maintenance_record
@@ -149,10 +378,15 @@ module Saboteur
         self.class.const_set(:MaintenanceRecord, Class.new(::ActiveRecord::Base) { self.abstract_class = true })
       end
 
+      # Disconnecting an in-memory SQLite pool destroys the database, so
+      # those pools stay connected everywhere saboteur disconnects: the
+      # schema loaded at boot survives, and workers inherit it via fork.
       def disconnect
         return unless defined?(::ActiveRecord::Base)
 
-        ::ActiveRecord::Base.connection_handler.connection_pool_list(:all).each(&:disconnect!)
+        ::ActiveRecord::Base.connection_handler.connection_pool_list(:all).each do |pool|
+          pool.disconnect! unless self.class.memory_database?(pool.db_config)
+        end
       end
 
       # The parent disconnected before fork, so inherited pools hold no sockets.
@@ -160,6 +394,88 @@ module Saboteur
       # checks out a fresh connection of its own on first use.
       def reconnect
         disconnect
+        restore_memory_databases
+      end
+
+      # Parent, once the app booted: an in-memory SQLite database survives
+      # only inside its own connection, and a forked child cannot keep it -
+      # ActiveRecord discards inherited pools and the sqlite3 gem closes
+      # writable handles in the child, so the next access there opens an
+      # empty database. Snapshot each in-memory database to a file the
+      # children can restore from.
+      def snapshot_memory_databases
+        return unless defined?(::ActiveRecord::Base)
+
+        pools = ::ActiveRecord::Base.connection_handler.connection_pool_list(:all)
+        memory_pools = pools.select { |pool| self.class.memory_database?(pool.db_config) }
+        return if memory_pools.empty?
+
+        @memory_snapshot_dir = Dir.mktmpdir("saboteur-memory-")
+        own_memory_snapshots
+        memory_pools.each_with_index do |pool, index|
+          path = File.join(@memory_snapshot_dir, "#{index}.sqlite3")
+          pool.with_connection do |connection|
+            file = SQLite3::Database.new(path)
+            with_backup(file, connection.raw_connection, close: file) { |backup| step_backup(backup, path) }
+          end
+          @memory_snapshots[pool.db_config] = path
+        end
+      end
+
+      # The safety net for an abrupt process exit: Engine.call removes the
+      # snapshots at the run boundary on every normal path, so this handler
+      # fires only when the owner process dies first. Forked children
+      # inherit the handler but not the pid.
+      def own_memory_snapshots
+        owner = Process.pid
+        at_exit { cleanup_memory_snapshots(owner) }
+      end
+
+      def cleanup_memory_snapshots(owner)
+        FileUtils.rm_rf(@memory_snapshot_dir) if Process.pid == owner && @memory_snapshot_dir
+      end
+
+      # Yields a backup and always returns both handles: exactly one finish
+      # per init (skipped only when backup_init itself raised, leaving no
+      # handle), and the database closes even when finish fails.
+      def with_backup(destination, source, close:)
+        backup = SQLite3::Backup.new(destination, "main", source, "main")
+        yield backup
+      ensure
+        begin
+          backup&.finish
+        ensure
+          close_without_masking(close)
+        end
+      end
+
+      # A close can fail too, and a close that fails on the way out must
+      # not hide the error already in flight.
+      def close_without_masking(database)
+        in_flight = $ERROR_INFO
+        database.close
+      rescue SQLite3::Exception
+        raise unless in_flight
+      end
+
+      # Child: load the parent's snapshot into this process's own fresh
+      # in-memory database. The backup overwrites the destination, so
+      # restoring more than once is harmless.
+      def restore_memory_databases
+        return unless defined?(::ActiveRecord::Base)
+        return if @memory_snapshots.empty?
+
+        ::ActiveRecord::Base.connection_handler.connection_pool_list(:all).each do |pool|
+          snapshot = @memory_snapshots[pool.db_config]
+          next unless snapshot
+
+          pool.with_connection do |connection|
+            source = SQLite3::Database.new(snapshot, readonly: true)
+            with_backup(connection.raw_connection, source, close: source) do |backup|
+              step_backup(backup, pool.db_config.database)
+            end
+          end
+        end
       end
     end
   end

data/lib/saboteur/help.rb

@@ -0,0 +1,210 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+require_relative "config"
+
+module Saboteur
+  # The text printed by `saboteur --help` and `saboteur help <command>`.
+  #
+  # It is a map of everything saboteur can do, for people and for AI agents that
+  # have no docs. The layout is fixed: section headers are upper case on their
+  # own line, command and key rows start with two spaces. Option lines come from
+  # the CLI's own option parser, so a new option shows up here without an edit.
+  class Help
+    Command = Data.define(:name, :summary, :description, :takes_options, :exit_codes, :examples)
+
+    PURPOSE = "mutation testing for Ruby and RSpec"
+    DOCUMENTATION = "https://github.com/vamsipavanmahesh/saboteur"
+
+    EXIT_CODE_MEANINGS = {
+      0 => "Success. Without --threshold: no unignored survivors, and at most half of the executed mutants " \
+           "errored. With --threshold: the score is at or above it, even if mutants survived.",
+      1 => "Without --threshold: at least one mutant survived and is not on the ignore list. With --threshold " \
+           "(survivors alone no longer fail the run): the score is below it, or no mutant was tested so " \
+           "there is no score.",
+      2 => "Usage or configuration error: fix the command or config/saboteur.rb. A replay --id that matches no " \
+           "mutant is only found after the target has booted. A named source or test file that does not exist.",
+      3 => "Tool error: saboteur could not finish, or more than half of the executed mutants errored (such " \
+           "a run tested almost nothing). With --threshold, also any errored or resource-limited mutant."
+    }.freeze
+
+    DOCTOR_EXIT_CODE_MEANINGS = {
+      0 => "Healthy: the environment checks passed and the baseline passed twice in the same order.",
+      2 => "Usage or configuration error: fix the command or config/saboteur.rb.",
+      3 => "A check failed: the target did not boot, or the baseline failed or changed between runs."
+    }.freeze
+
+    COMMANDS = [
+      Command.new(
+        name: "run",
+        summary: "Mutate your code and report the mutants your tests do not catch",
+        description: "Builds a per-test coverage map, makes small changes to your methods, runs only the " \
+                     "tests that cover each change, and reports survivors (changes no test noticed). Source files or " \
+                     "directories after run replace include and exclude; --test limits the specs that run; " \
+                     "--threshold PERCENT replaces the survivor check: exit 0 when the score is at or above " \
+                     "PERCENT, exit 1 when it is below.",
+        takes_options: true,
+        exit_codes: EXIT_CODE_MEANINGS,
+        examples: [
+          ["Pull request: only methods changed since the target branch", "saboteur run --since origin/main"],
+          ["Whole project, four workers, repeatable seed", "saboteur run --jobs 4 --seed 42"],
+          ["One class or method", "saboteur run --subject 'Person#adult?'"],
+          ["Replay one mutant by the digest printed next to a survivor", "saboteur run --id 1a2b3c4d5e6f7a8b"],
+          ["Mutate named files against named specs, skipping the whole-suite coverage run",
+           "saboteur run lib/a.rb lib/b.rb --test spec/a_spec.rb --test spec/b_spec.rb"],
+          ["CI gate: exit 1 when the score is below 90 percent", "saboteur run --threshold 90"],
+          ["AI agent: bounded JSON summary and strict exit codes", "saboteur run --agent --since origin/main"]
+        ]
+      ),
+      Command.new(
+        name: "doctor",
+        summary: "Check the environment and run the baseline twice, without mutating anything",
+        description: "Boots the target like a run, checks runtime, coverage and boot files, then runs your " \
+                     "suite twice in the same order with one seed and reports whether both runs passed. " \
+                     "It never mutates anything.",
+        takes_options: true,
+        exit_codes: DOCTOR_EXIT_CODE_MEANINGS,
+        examples: [["Check the setup before the first run", "saboteur doctor"],
+                   ["Use another config file and seed", "saboteur doctor --config config/ci.rb --seed 7"]]
+      ),
+      Command.new(
+        name: "version",
+        summary: "Print the installed version",
+        description: "Prints the version number and nothing else.",
+        takes_options: false,
+        exit_codes: nil,
+        examples: [["Print the version", "saboteur version"]]
+      ),
+      Command.new(
+        name: "help",
+        summary: "Show this map, or details for one command: saboteur help <command>",
+        description: "Without an argument, prints every command, option, config key and exit code. " \
+                     "With a command name, prints that command's usage, options and examples.",
+        takes_options: false,
+        exit_codes: nil,
+        examples: [["Show everything", "saboteur help"], ["Show one command", "saboteur help run"]]
+      ),
+      Command.new(
+        name: "agent-skill",
+        summary: "Print instructions for AI coding agents to paste into AGENTS.md or CLAUDE.md",
+        description: "Prints a short Markdown block that tells an agent when to run saboteur, how to read " \
+                     "`saboteur run --agent`, and how to handle survivors.",
+        takes_options: false,
+        exit_codes: nil,
+        examples: [["Add the agent instructions to your repo", "saboteur agent-skill >> AGENTS.md"]]
+      )
+    ].freeze
+
+    AGENT_EXIT_CODES = "  With --agent the codes are stricter: 0 only when the run was complete and clean; " \
+                       "1 survivors to review;\n  2 usage or config error; 3 incomplete (no coverage, caps, errors, " \
+                       "nothing selected) or tool failure."
+
+    FILES = [
+      ["config/saboteur.rb", "Settings as Ruby (Saboteur.configure). Command line options override it."],
+      ["config/saboteur_ignore.rb", "Accepted survivors: mutation digests with subject and a required reason."],
+      [".saboteur/report.json", "Stryker mutation-testing-report-schema JSON (--json changes the path)."],
+      [".saboteur/report.html", "Self-contained HTML report (--html changes the path)."],
+      [".saboteur/", "Cache: coverage map and incremental verdicts. Safe to delete."]
+    ].freeze
+
+    # option_lines: the option lines of each command's parser, keyed by command name.
+    def initialize(option_lines:, commands: COMMANDS)
+      @option_lines = option_lines
+      @commands = commands
+    end
+
+    def overview
+      [title, usage_section, commands_section, *option_sections, examples_section(all_examples),
+       *exit_code_sections, config_section, files_section, "Documentation: #{DOCUMENTATION}\n"].join("\n")
+    end
+
+    # Details for one command, or nil when there is no such command.
+    def command(name)
+      found = @commands.find { |candidate| candidate.name == name }
+      return unless found
+
+      sections = ["saboteur #{found.name} - #{found.summary}\n", "#{found.description}\n", command_usage(found)]
+      sections << options_section(found) if found.takes_options
+      sections << examples_section(found.examples)
+      sections << exit_codes_section(found.exit_codes) if found.exit_codes
+      sections.join("\n")
+    end
+
+    def command_names
+      @commands.map(&:name)
+    end
+
+    private
+
+    # One section for every command that takes options, so the overview needs no other page.
+    def option_sections
+      @commands.select(&:takes_options).map { |command| options_section(command) }
+    end
+
+    def exit_code_sections
+      @commands.select(&:exit_codes).map { |command| exit_codes_section(command.exit_codes, command.name) }
+    end
+
+    def title
+      "saboteur #{VERSION} - #{PURPOSE}\n"
+    end
+
+    def usage_section
+      "USAGE\n  saboteur <command> [options]\n  saboteur help [command]\n"
+    end
+
+    def command_usage(command)
+      suffix = command.takes_options ? " [options]" : ""
+      "USAGE\n  saboteur #{command.name}#{suffix}\n"
+    end
+
+    def commands_section
+      rows = @commands.map { |command| "  #{command.name.ljust(name_width)}  #{command.summary}" }
+      "COMMANDS\n#{rows.join("\n")}\n"
+    end
+
+    def name_width
+      @commands.map { |command| command.name.length }.max
+    end
+
+    def options_section(command)
+      "OPTIONS (saboteur #{command.name})\n#{@option_lines.fetch(command.name).map(&:rstrip).join("\n")}\n"
+    end
+
+    def all_examples
+      @commands.flat_map(&:examples)
+    end
+
+    def examples_section(examples)
+      "EXAMPLES\n#{examples.map { |note, line| "  # #{note}\n  #{line}" }.join("\n")}\n"
+    end
+
+    def exit_codes_section(codes, command_name = nil)
+      rows = codes.map { |code, meaning| "  #{code}  #{meaning}" }
+      title = command_name ? "EXIT CODES (saboteur #{command_name})" : "EXIT CODES"
+      note = codes.equal?(EXIT_CODE_MEANINGS) ? "#{AGENT_EXIT_CODES}\n" : ""
+      "#{title}\n#{rows.join("\n")}\n#{note}"
+    end
+
+    def config_section
+      rows = Config::DEFAULTS.map { |key, value| "  c.#{key} = #{describe_default(value)}" }
+      intro = "Every setting with its default value, as a Ruby assignment. Unknown settings are an error."
+      "CONFIG FILE (config/saboteur.rb)\n#{intro}\n#{rows.join("\n")}\n"
+    end
+
+    # Ruby literals, so a copied line loads as the same value: inspect prints
+    # nil as "nil" (JSON would print null, which is not Ruby) and quotes
+    # strings and arrays the way Ruby reads them back.
+    def describe_default(value)
+      value.inspect
+    end
+
+    def files_section
+      "FILES\n#{FILES.map { |path, meaning| "  #{path.ljust(files_width)}  #{meaning}" }.join("\n")}\n"
+    end
+
+    def files_width
+      FILES.map { |path, _| path.length }.max
+    end
+  end
+end