reeve 0.4.0 → 0.6.0

data/lib/reeve/tasks.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Reeve
+  # `bin/rails reeve:verify_ledger` (specs/002 FR-017, contracts/ledger-chain.md "Rake").
+  #
+  # A Ruby file rather than a .rake one on purpose: reeve.gemspec packages lib/**/*.rb, and a
+  # .rake file would pass every spec in the working tree and be missing from the gem users
+  # install. Registered by Reeve::Railtie; callable without Rake for anything else that
+  # wants the same answer — a deploy gate, a scheduled job, a console.
+  #
+  #   status = Reeve::Tasks.verify_ledger(env: { "LIMIT" => "0" })
+  module Tasks
+    EXIT = { intact: 0, broken: 1, unverifiable: 2, no_chain: 2 }.freeze
+    FORMATS = %w[text json].freeze
+
+    module_function
+
+    # Verifies the ledger, writes the report to `out`, and returns the exit status:
+    # 0 intact, 1 broken, 2 could not verify (unverifiable, or no chain).
+    #
+    # LIMIT caps the breaks listed in the text report (default 100; the totals are never
+    # capped). FORMAT=json writes the whole result, every break, and ignores LIMIT.
+    def verify_ledger(env: ENV, out: $stdout)
+      format = env.fetch("FORMAT", "text")
+      unless FORMATS.include?(format)
+        raise ArgumentError, "FORMAT must be one of #{FORMATS.join(', ')}, got #{format.inspect}"
+      end
+
+      limit = limit_from(env)
+      require "reeve/audit"
+      result = Reeve::Audit::Chain.verify
+      out.puts(format == "json" ? JSON.generate(result.as_json) : result.to_s(limit: limit))
+      EXIT.fetch(result.status)
+    end
+
+    def limit_from(env)
+      raw = env.fetch("LIMIT", Reeve::Audit::Chain::Verification::DEFAULT_LIMIT.to_s)
+      Integer(raw, 10).tap { |limit| raise ArgumentError if limit.negative? }
+    rescue ArgumentError, TypeError
+      raise ArgumentError, "LIMIT must be a whole number of breaks to list, got #{raw.inspect}"
+    end
+
+    # The Rake task, defined on the current Rake application. Depends on :environment, so
+    # the host's initializer — and with it the ledger key — is loaded first.
+    #
+    # The DSL is borrowed by a throwaway object, so Reeve::Tasks itself never gains `task`,
+    # `sh` and the rest of Rake's methods.
+    def define
+      Object.new.extend(Rake::DSL).instance_exec do
+        namespace :reeve do
+          desc "Verify the audit ledger's chain " \
+               "(exit 0 intact, 1 broken, 2 could not verify; LIMIT=, FORMAT=json)"
+          task verify_ledger: :environment do
+            exit Reeve::Tasks.verify_ledger
+          end
+        end
+      end
+    end
+  end
+end

data/lib/reeve/testing/assertions.rb

@@ -56,14 +56,14 @@ module Reeve
         )
       end
 
-      # SC-009's escape hatch: any of the seven, straight from Minitest.
+      # SC-009's escape hatch: any check, straight from Minitest.
       def assert_reeve_check(check)
         result = check.call
         assert result.passed?, result.message
         result
       end
 
-      # A whole Report at once, for a host that would rather have one test than seven.
+      # A whole Report at once, for a host that would rather have one test than one per check.
       def assert_reeve_compliance(principals: nil, **options)
         report = Checks.run_all(
           principals: principals || Testing.compliance_principals, **options

data/lib/reeve/testing/checks/contract_version.rb

@@ -46,9 +46,10 @@ module Reeve
           invocation_id occurred_at agent_id agent_name principal_type principal_id
           tool_name arguments outcome rule detail record_type record_ids record_count
           truncated derived guard duration_ms metadata contract_version
+          chain_sequence chain_key_id chain_digest
         ].freeze
 
-        EXPECTED_VERSION = 2
+        EXPECTED_VERSION = 3
 
         def initialize(tool: nil, expected: EXPECTED_VERSION, ledger: nil)
           super(tool: tool, ledger: ledger)

data/lib/reeve/testing/checks/endpoint_coverage.rb

@@ -0,0 +1,133 @@
+# frozen_string_literal: true
+
+module Reeve
+  module Testing
+    module Checks
+      # Whether the run covered the endpoint, or only the tools reeve was told about.
+      #
+      # Every other check walks a population reeve assembled itself: the classes that
+      # included `Reeve::Guard`. A server that dispatches some tools without ever handing
+      # them to reeve leaves them out of that population, and the run goes green on the
+      # subset it could see. This check compares the run against the host's own list of
+      # registered names (`config.inventory`), so a green run can only mean the endpoint.
+      #
+      # Four answers, and only one of them is a pass:
+      #
+      # * **no inventory declared** — skipped. Reeve cannot know what the server registers
+      #   unless told, and a pass here would be the green that means "did not look".
+      # * **a subset** — skipped, when the host narrowed the run with `compliance_tools` or
+      #   `tools:`. That is the incremental-adoption path; the run says what it certified
+      #   and that it is not the endpoint, without turning the build red mid-retrofit.
+      # * **incomplete** — failed: a registered tool bypasses reeve or reaches it unguarded,
+      #   and the run claimed to cover everything.
+      # * **complete** — passed.
+      #
+      #   Reeve::Checks::EndpointCoverage.new.call
+      #   Reeve::Checks::EndpointCoverage.new(inventory: McpInventory).call
+      class EndpointCoverage < Base
+        def initialize(inventory: :configured, tools: nil, ledger: nil)
+          super(ledger: ledger)
+          @inventory = inventory == :configured ? Reeve.config.inventory : inventory
+          @tools = tools
+        end
+
+        def call
+          return unverified if inventory.nil?
+
+          report = inventory.report
+          if narrowed?
+            subset(report)
+          elsif report.complete?
+            complete(report)
+          else
+            incomplete(report)
+          end
+        rescue StandardError => e
+          failed("the endpoint inventory could not be read, so its coverage is unknown: " \
+                 "#{e.class}: #{e.message}", coverage: :unknown)
+        end
+
+        private
+
+        attr_reader :inventory
+
+        def narrowed?
+          !@tools.nil? || Testing.compliance_tools_narrowed?
+        end
+
+        def certified
+          @certified ||= @tools || Testing.compliance_tools
+        end
+
+        def unverified
+          skipped(
+            "no endpoint inventory is declared, so this run certifies the " \
+            "#{pluralize(certified.size, 'tool')} reeve knows about and cannot say whether " \
+            "the server dispatches others outside it — set config.inventory to a " \
+            "Reeve::Inventory to verify the endpoint",
+            coverage: :unverified
+          )
+        end
+
+        def complete(report)
+          passed(
+            "every one of the #{pluralize(report.registered.size, 'tool')} the server " \
+            "registers is protected and guarded (#{report.guarded.size}) or exempted " \
+            "(#{report.exempted.size})",
+            coverage: :complete
+          )
+        end
+
+        def incomplete(report)
+          failed(
+            "expected every tool the server registers to be protected by reeve or " \
+            "exempted, but #{gaps(report)}\n\n#{report}",
+            coverage: :incomplete
+          )
+        end
+
+        # Complete coverage with a narrowed run is still a subset: the inventory is sound,
+        # but this run did not certify all of it.
+        def subset(report)
+          routed = report.routed_tools
+          missing = routed - certified
+          lines = [
+            "this run certifies a subset, not the endpoint: " \
+            "#{(routed - missing).size} of the #{pluralize(routed.size, 'tool')} routed " \
+            "through reeve (compliance_tools or tools: narrowed it)"
+          ]
+          lines << "not certified: #{missing.map { |tool| tool.name || tool.inspect }.join(', ')}" \
+            unless missing.empty?
+          lines << "" << report.to_s unless report.complete?
+
+          skipped(lines.join("\n"), coverage: :subset)
+        end
+
+        def gaps(report)
+          [
+            gap(report.bypassing, "bypasses it", "bypass it"),
+            gap(report.unguarded, "is routed through it unguarded",
+                "are routed through it unguarded"),
+            gap(report.unregistered_entries, "declared in the inventory is not registered",
+                "declared in the inventory are not registered")
+          ].compact.join(", ")
+        end
+
+        def gap(entries, one, many)
+          return nil if entries.empty?
+
+          "#{entries.size} #{entries.size == 1 ? one : many}"
+        end
+
+        # Only this check can come back with nothing established either way.
+        def skipped(message, details = {})
+          Result.skipped(check: check_name, message: message, details: details)
+        end
+
+        def base_details
+          {}
+        end
+      end
+    end
+  end
+end

data/lib/reeve/testing/checks/ledger_chain.rb

@@ -0,0 +1,116 @@
+# frozen_string_literal: true
+
+module Reeve
+  module Testing
+    module Checks
+      # Whether the ledger's chain holds (specs/002 FR-018).
+      #
+      # It writes entries of its own first — the first compliance tool, once per fixture
+      # principal — because a suite with transactional fixtures starts every example with
+      # an empty ledger, and a check that only looked would find nothing and say nothing.
+      # Then it verifies the whole ledger the run can see, not only its own rows: a break an
+      # earlier run left behind is reported, not hidden.
+      #
+      # * **intact** — passed; the message is the verification's own summary line, so it
+      #   states the range covered and what was not.
+      # * **broken, unverifiable, or no chain after writing** — failed. The run signed its own
+      #   entries, so a key it lacks or a recorder that did not chain is a configuration
+      #   error, not an absence.
+      # * **no key configured** — failed, naming the setting.
+      # * **nothing to write or read** (no compliance tools, ledger not loaded) — skipped,
+      #   saying the chain was not verified. Never passed.
+      #
+      #   Reeve::Checks::LedgerChain.new.call
+      class LedgerChain < Base
+        NOT_VERIFIED = "the ledger chain was not verified"
+
+        def initialize(tools: nil, principals: nil, arguments: {}, invoke: nil, ledger: nil)
+          @tools = tools
+          @principals = principals
+          super(tool: nil, arguments: arguments, invoke: invoke, ledger: ledger)
+        end
+
+        def call
+          return skipped("there is no compliance tool to write an entry with") if subject.nil?
+          return skipped(unreadable_reason) unless readable?
+          return no_key if Reeve.config.ledger_key.nil?
+
+          unwritten = write_entries
+          return unwritten_entry(unwritten) if unwritten
+
+          judge(ledger.verify_chain)
+        end
+
+        private
+
+        def subject
+          @subject ||= Array(@tools || Testing.compliance_tools).first
+        end
+
+        def principals
+          configured = Testing.compliance_principals? ? Testing.compliance_principals : nil
+          list = Array(@principals || configured)
+          list.empty? ? [nil] : list
+        end
+
+        # Each principal on its own, so one refusal does not stop the next. A denial, or the
+        # tool's own error, still leaves a ledger entry — the envelope records before it
+        # re-raises — which is all this needs. A failed ledger write leaves none, and is
+        # returned as the reason the chain could not be checked.
+        #
+        # Returns nil when entries were written, or the reason none were — including the case
+        # where every invocation failed before reaching reeve, which a chain left by an
+        # earlier run must not be allowed to pass for.
+        def write_entries
+          before = ledger.count
+          principals.each do |principal|
+            invoke.call(tool: subject, arguments: arguments, principal: principal)
+          rescue Reeve::AuditWriteError => e
+            return "#{e.class}: #{e.message}"
+          rescue StandardError
+            next
+          end
+          ledger.count == before ? "its invocations wrote no entry" : nil
+        end
+
+        def readable?
+          ledger.available? && ledger.respond_to?(:verify_chain)
+        end
+
+        def unwritten_entry(reason)
+          failed("#{NOT_VERIFIED}: it wrote no entry of its own — #{reason}",
+                 coverage: :unwritten)
+        end
+
+        def unreadable_reason
+          ledger.unavailable_reason || "this ledger cannot verify a chain"
+        end
+
+        def judge(verification)
+          return skipped("the audit ledger is not loaded") if verification.nil?
+
+          summary = verification.to_s
+          return passed(summary.lines.first.chomp, coverage: :intact) if verification.intact?
+
+          failed(summary, coverage: verification.status)
+        end
+
+        def no_key
+          failed("expected the ledger to be chained, but no ledger key is configured: set " \
+                 "config.ledger_key (the generated initializer derives one from secret_key_base)",
+                 coverage: :no_key)
+        end
+
+        # Every skip says, first, that the chain was not verified — then why.
+        def skipped(reason)
+          Result.skipped(check: check_name, message: "#{NOT_VERIFIED}: #{reason}",
+                         details: { coverage: :unverified })
+        end
+
+        def base_details
+          {}
+        end
+      end
+    end
+  end
+end

data/lib/reeve/testing/checks.rb

@@ -8,17 +8,19 @@ require_relative "checks/rule_present"
 require_relative "checks/redaction_holds"
 require_relative "checks/principal_required"
 require_relative "checks/contract_version"
+require_relative "checks/endpoint_coverage"
+require_relative "checks/ledger_chain"
 
 module Reeve
   module Testing
-    # The seven guarantees, as objects.
+    # The guarantees, as objects.
     #
     # This is the whole of the testing kit's logic. The RSpec matchers and the Minitest
     # assertions are adapters over it and contain no assertions of their own, which is why
     # the same violation reads identically from either — and from neither:
     #
     #   report = Reeve::Checks.run_all(principals: [alice, bob])
-    #   abort report.to_s unless report.passed?
+    #   abort report.to_s if report.failed? || report.nothing_checked?
     #
     # Nothing here loads a test framework (FR-026).
     module Checks
@@ -29,11 +31,14 @@ module Reeve
         RulePresent,
         RedactionHolds,
         PrincipalRequired,
-        ContractVersion
+        ContractVersion,
+        EndpointCoverage,
+        LedgerChain
       ].freeze
 
-      # The checks that are asked once about the ledger rather than once per tool.
-      GLOBAL = [ContractVersion].freeze
+      # The checks that are asked once — about the ledger, or about the endpoint as a
+      # whole — rather than once per tool.
+      GLOBAL = [ContractVersion, EndpointCoverage, LedgerChain].freeze
 
       # The compliance suite's engine (FR-018): every check, against every registered
       # guarded tool, in one Report.
@@ -53,9 +58,14 @@ module Reeve
       # test method out of, so that a failing suite names the guarantee that broke rather
       # than reporting "compliance" as one undifferentiated red.
       def self.run(check, principals:, tools: nil, arguments: {}, invoke: nil, ledger: nil)
-        return Report.new([check.new(ledger: ledger).call]) if GLOBAL.include?(check)
+        if GLOBAL.include?(check)
+          return Report.new([build(check, tool: nil, principals: principals, tools: tools,
+                                          invoke: invoke, ledger: ledger).call])
+        end
 
         subjects = tools || Testing.compliance_tools
+        return Report.new([nothing_to_check(check, tools)]) if subjects.empty?
+
         Report.new(
           subjects.map do |tool|
             build(check, tool: tool, principals: principals, arguments: arguments,
@@ -64,6 +74,25 @@ module Reeve
         )
       end
 
+      # A per-tool check over no tools establishes nothing, and an empty Report reads as
+      # passed — the green that means "did not look". So an empty population is one skip,
+      # saying which of the three ways the run came to have nothing in it.
+      def self.nothing_to_check(check, tools)
+        reason =
+          if !tools.nil? then "the run was given tools: []"
+          elsif Testing.compliance_tools_narrowed? then "compliance_tools is []"
+          else "the registry is empty and no endpoint inventory routes a tool"
+          end
+
+        Result.skipped(
+          check: check.check_name,
+          message: "no compliance tools to check, so #{check.check_name} established " \
+                   "nothing — #{reason}",
+          details: { coverage: :none }
+        )
+      end
+      private_class_method :nothing_to_check
+
       # Every per-tool check for one tool, instantiated but not run.
       def self.for_tool(tool, principals:, arguments: {}, invoke: nil, ledger: nil)
         (ALL - GLOBAL).map do |check|
@@ -74,12 +103,19 @@ module Reeve
 
       # The one place that knows what each check's constructor wants. Front-ends and hosts
       # ask for a check by class and get a configured one back.
-      def self.build(check, tool:, principals:, arguments: {}, invoke: nil, ledger: nil)
+      #
+      # +tools+ is the population the run was given, which only EndpointCoverage reads: an
+      # explicit list is a narrowed run, and it must say so.
+      def self.build(check, tool:, principals:, arguments: {}, invoke: nil, ledger: nil,
+                     tools: nil)
         common = { tool: tool, arguments: arguments, invoke: invoke, ledger: ledger }
 
         case check.check_name
         when "GuardDeclared"      then GuardDeclared.new(tool: tool, ledger: ledger)
         when "ContractVersion"    then ContractVersion.new(ledger: ledger)
+        when "EndpointCoverage"   then EndpointCoverage.new(tools: tools, ledger: ledger)
+        when "LedgerChain"        then LedgerChain.new(tools: tools, principals: principals,
+                                                       invoke: invoke, ledger: ledger)
         when "PrincipalRequired"  then PrincipalRequired.new(**common)
         when "CrossPrincipalLeak" then CrossPrincipalLeak.new(principals: Array(principals),
                                                               **common)

data/lib/reeve/testing/compliance_assertions.rb

@@ -15,7 +15,9 @@ module Reeve
     #
     # One method per check, matching the RSpec shared example group one for one, so a red
     # build names the guarantee that broke rather than reporting "compliance" as one
-    # undifferentiated failure.
+    # undifferentiated failure. A check that could establish nothing either way — no
+    # inventory declared, or no tools to run a per-tool check over — is skipped with its
+    # reason, never passed.
     module ComplianceAssertions
       include Assertions
 
@@ -25,6 +27,7 @@ module Reeve
 
         define_method(method_name) do
           report = Checks.run(check, principals: Testing.compliance_principals)
+          skip(report.to_s) if report.skipped?
           assert report.passed?, report.to_s
           report
         end

data/lib/reeve/testing/compliance_suite.rb

@@ -11,11 +11,14 @@
 #   Reeve::Testing.compliance_principals = -> { [users(:alice), users(:bob)] }
 #
 # One example per check rather than one for the lot, so a red build names the guarantee
-# that broke — "CrossPrincipalLeak" is an incident, "compliance" is a shrug.
+# that broke — "CrossPrincipalLeak" is an incident, "compliance" is a shrug. A check that
+# could establish nothing either way (EndpointCoverage with no inventory, or a per-tool
+# check with no tools to run over) is skipped, with its reason, rather than passed.
 RSpec.shared_examples "a reeve-compliant server" do
   Reeve::Checks::ALL.each do |check|
     it "satisfies #{check.check_name} for every registered guarded tool" do
       report = Reeve::Checks.run(check, principals: Reeve::Testing.compliance_principals)
+      skip(report.to_s) if report.skipped?
 
       expect(report).to be_passed, report.to_s
     end

data/lib/reeve/testing/ledger.rb

@@ -10,7 +10,8 @@ module Reeve
     # the ledger is absent.
     #
     # A host with its own non-ActiveRecord recorder can pass any object with these six
-    # methods as `ledger:` to any ledger-reading check.
+    # methods as `ledger:` to any ledger-reading check; LedgerChain also asks for
+    # #verify_chain, and is skipped by a ledger that has none.
     class Ledger
       def self.default
         new
@@ -67,6 +68,15 @@ module Reeve
         entry_class.column_names.map(&:to_s)
       end
 
+      # The chain's verification of the whole ledger (specs/002), or nil when the ledger is
+      # not loaded. The kit's only way to the chain: it names Reeve::Audit::Chain here, at
+      # call time, so `require "reeve/testing"` still loads with no ActiveRecord at all.
+      def verify_chain
+        return nil unless available? && Reeve::Audit.const_defined?(:Chain, false)
+
+        Reeve::Audit::Chain.verify(entry_class: entry_class)
+      end
+
       private
 
       def entry_class

data/lib/reeve/testing/matchers/pass_reeve_check.rb

@@ -3,7 +3,7 @@
 module Reeve
   module Testing
     module Matchers
-      # The escape hatch that keeps SC-009 honest: every one of the seven checks is
+      # The escape hatch that keeps SC-009 honest: every check is
       # assertable from RSpec, not only the two with names of their own.
       #
       #   expect(Reeve::Checks::RedactionHolds.new(tool: T, principal: alice))

data/lib/reeve/testing/report.rb

@@ -15,6 +15,8 @@ module Reeve
         freeze
       end
 
+      # Nothing failed. A skip is not a failure — and it is printed, so it is not
+      # mistaken for a pass either.
       def passed?
         failures.empty?
       end
@@ -23,20 +25,39 @@ module Reeve
         !passed?
       end
 
+      # Every result was a skip: the run established nothing either way. The front-ends
+      # report this as a skipped test rather than a green one.
+      def skipped?
+        !results.empty? && results.all?(&:skipped?)
+      end
+
+      # A check had nothing to examine — a per-tool check run over no tools at all. passed?
+      # is still true then, since nothing failed; a test framework shows the skip, but a
+      # gate with no framework (a CI script, a deploy step) has only this to stop on. A skip
+      # for a missing inventory is not this: there the tools existed and were checked.
+      def nothing_checked?
+        results.any? { |result| result.skipped? && result.details[:coverage] == :none }
+      end
+
       def failures
-        results.reject(&:passed?)
+        results.select(&:failed?)
       end
 
       def passes
         results.select(&:passed?)
       end
 
+      def skips
+        results.select(&:skipped?)
+      end
+
       def size
         results.size
       end
 
       def to_s
-        [summary, *failures.map { |result| detail(result) }].join("\n")
+        [summary, *failures.map { |result| detail("FAIL", result) },
+         *skips.map { |result| detail("SKIP", result) }].join("\n")
       end
 
       def inspect
@@ -46,12 +67,15 @@ module Reeve
       private
 
       def summary
-        "reeve compliance: #{size} #{size == 1 ? 'check' : 'checks'}, " \
-          "#{passes.size} passed, #{failures.size} failed"
+        line = "reeve compliance: #{size} #{size == 1 ? 'check' : 'checks'}, " \
+               "#{passes.size} passed, #{failures.size} failed"
+        skips.empty? ? line : "#{line}, #{skips.size} skipped"
       end
 
-      def detail(result)
-        "\nFAIL #{result.check}\n  #{result.message}"
+      # Every line of a multi-line message indented, so it reads as one block under its
+      # heading rather than as stray lines of the report.
+      def detail(heading, result)
+        "\n#{heading} #{result.check}\n  #{result.message.gsub("\n", "\n  ")}"
       end
     end
   end

data/lib/reeve/testing/result.rb

@@ -9,31 +9,51 @@ module Reeve
     # Minitest assertion, a rake task — reads +message+ and prints it; none of them
     # composes their own (FR-019). That is what makes the same violation read identically
     # from all three.
+    #
+    # A result is passed, failed, or skipped. Skipped is for a guarantee the run could not
+    # establish either way — endpoint coverage with no inventory declared. Reporting that
+    # as a pass would be the green that means "we did not look"; reporting it as a failure
+    # would turn every build red over a question the host never asked. A skip is printed,
+    # and shows up yellow in both test frameworks, so it is neither.
     class Result
-      attr_reader :check, :message, :details
+      STATUSES = %i[passed failed skipped].freeze
+
+      attr_reader :check, :message, :details, :status
 
       def self.passed(check:, message:, details: {})
-        new(check: check, passed: true, message: message, details: details)
+        new(check: check, status: :passed, message: message, details: details)
       end
 
       def self.failed(check:, message:, details: {})
-        new(check: check, passed: false, message: message, details: details)
+        new(check: check, status: :failed, message: message, details: details)
+      end
+
+      def self.skipped(check:, message:, details: {})
+        new(check: check, status: :skipped, message: message, details: details)
       end
 
-      def initialize(check:, passed:, message:, details: {})
+      def initialize(check:, status:, message:, details: {})
+        unless STATUSES.include?(status)
+          raise ArgumentError, "status must be one of #{STATUSES.inspect}, got #{status.inspect}"
+        end
+
         @check   = check.to_s
-        @passed  = passed ? true : false
+        @status  = status
         @message = message.to_s
         @details = details.freeze
         freeze
       end
 
       def passed?
-        @passed
+        status == :passed
       end
 
       def failed?
-        !@passed
+        status == :failed
+      end
+
+      def skipped?
+        status == :skipped
       end
 
       def to_s
@@ -41,7 +61,7 @@ module Reeve
       end
 
       def inspect
-        "#<Reeve::Testing::Result #{check} #{passed? ? 'passed' : 'failed'} " \
+        "#<Reeve::Testing::Result #{check} #{status} " \
           "#{message.inspect}>"
       end
     end