reeve 0.6.0 → 0.8.0

data/lib/reeve/audit/chain/head.rb

@@ -18,6 +18,17 @@ module Reeve
       # engines: SQLite cannot upgrade a read lock while another writer waits, and ignores
       # SELECT ... FOR UPDATE.
       class Head
+        LOCKING_READ_ADAPTERS = /mysql|trilogy/i
+
+        # A read made under the head's lock must see the latest committed rows. PostgreSQL
+        # and SQLite read them anyway; InnoDB at REPEATABLE READ serves a plain SELECT from
+        # the snapshot of the caller's transaction, which a writer that committed while this
+        # one queued for the lock is not in (code review, specs/003). A locking read is
+        # current; LOCK IN SHARE MODE is the spelling every MySQL version accepts.
+        def self.current(connection, sql)
+          connection.adapter_name.match?(LOCKING_READ_ADAPTERS) ? "#{sql} LOCK IN SHARE MODE" : sql
+        end
+
         def initialize(connection:, entry_class:)
           @connection = connection
           @entry_class = entry_class
@@ -35,7 +46,7 @@ module Reeve
         # So a started chain can exist with nothing in it yet (head 0) — if the first chained
         # write then fails, say. The verifier reports that as no chain, not as a pass.
         def start!(signer)
-          return if connection.select_value("SELECT 1 FROM #{TABLE} WHERE #{named}")
+          return if started?
 
           genesis = genesis_for(signer)
           digest = Canonical.genesis_digest(signer.signing_secret, genesis)
@@ -62,14 +73,56 @@ module Reeve
           [Integer(head["head_sequence"]), head["head_digest"]]
         end
 
+        # Takes the head's lock without consuming a sequence (specs/003 FR-008) and returns the
+        # head: a rotation queues at the same point as a write, so the epoch it opens begins
+        # exactly one past the last row any writer committed. A write that changes nothing
+        # still takes the row lock on PostgreSQL and MySQL and the write lock on SQLite.
+        def lock!
+          updated = connection.update("UPDATE #{TABLE} SET head_sequence = head_sequence " \
+                                      "WHERE #{named}")
+          raise ActiveRecord::StatementInvalid, "the ledger chain record vanished" if updated.zero?
+
+          sql = "SELECT head_sequence FROM #{TABLE} WHERE #{named}"
+          Integer(connection.select_value(Head.current(connection, sql)))
+        end
+
+        # A Checkpoint of the head and the latest epoch, read under #lock!, which a write holds
+        # while it moves the head and opens its epoch, so the two are one committed state
+        # (specs/004 FR-005, research R2). Nil when the chain has not started or holds nothing.
+        def checkpoint(signer)
+          return nil unless started?
+
+          entry_class.transaction(requires_new: true) do
+            next nil unless lock!.positive?
+
+            Checkpoint.build(state: state, epoch: Epochs.new(connection: connection).latest,
+                             keyring: signer)
+          end
+        end
+
+        # The chain record's genesis and head as one current read.
+        def state
+          sql = "SELECT genesis_digest, pre_chain_count, pre_chain_unchained_count, " \
+                "head_sequence, head_digest FROM #{TABLE} WHERE #{named}"
+          connection.select_one(Head.current(connection, sql))
+        end
+
+        def started?
+          !connection.select_value("SELECT 1 FROM #{TABLE} WHERE #{named}").nil?
+        end
+
         def point_at!(digest)
           connection.update("UPDATE #{TABLE} SET head_digest = #{connection.quote(digest)} " \
                             "WHERE #{named}")
         end
 
+        # The connection this append runs on; Epochs uses it so the epoch an append opens
+        # commits or rolls back with the append.
+        attr_reader :connection
+
         private
 
-        attr_reader :connection, :entry_class
+        attr_reader :entry_class
 
         def increment
           connection.update("UPDATE #{TABLE} SET head_sequence = head_sequence + 1 WHERE #{named}")

data/lib/reeve/audit/chain/keyring.rb

@@ -16,6 +16,12 @@ module Reeve
           new(signing: config.ledger_key, retired: config.retired_ledger_keys)
         end
 
+        # The configured keys, or only `keys` when given: how an auditor verifies with their
+        # own copy.
+        def self.verifying(keys)
+          keys ? new(signing: nil, retired: keys) : from(Reeve.config)
+        end
+
         def initialize(signing:, retired: [])
           @signing_secret = signing
           @by_id = {}

data/lib/reeve/audit/chain/report.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "time"
+
 module Reeve
   module Audit
     class Chain
@@ -15,6 +17,7 @@ module Reeve
         LABEL_WIDTH = 14
         KIND_WIDTH = 17
         PREFIX = "reeve ledger:"
+        NOTHING_AFTER_PURGE = "no entries after the purge to verify"
 
         def self.number(value)
           digits = value.abs.to_s.reverse.scan(/\d{1,3}/).join(",").reverse
@@ -27,10 +30,11 @@ module Reeve
 
         def to_s(limit:)
           listed = @v.breaks.reject { |found| found.kind == :unknown_key }
-          return summary if listed.empty?
+          epochs = EpochLines.new(@v).to_a
+          return [summary, *epochs].join("\n") if listed.empty?
 
           lines = listed.first(limit).map { |found| line_for(found) }
-          [summary, totals, *lines, more(listed, limit)].compact.join("\n")
+          [summary, totals, *epochs, *lines, more(listed, limit)].compact.join("\n")
         end
 
         private
@@ -45,48 +49,50 @@ module Reeve
         end
 
         def intact
-          "#{PREFIX} INTACT — sequences #{range(@v.covered)} verified " \
-            "(head at start: #{number(@v.head_at_start)}); #{uncovered}"
+          if @v.walked_from
+            return "#{PREFIX} INTACT — #{anchor_text.walked_from}; #{uncovered}; #{anchor_text}"
+          end
+
+          walked = @v.covered ? "sequences #{range(@v.covered)} verified" : NOTHING_AFTER_PURGE
+          "#{PREFIX} INTACT — #{walked} (head at start: #{number(@v.head_at_start)}); " \
+            "#{uncovered}; #{anchor_text}"
         end
 
         def broken
           count = @v.breaks.size
+          breaks = "#{number(count)} break#{'s' unless count == 1}"
+          if anchor_text.emptied?
+            return "#{PREFIX} BROKEN — #{breaks} in the ledger; #{anchor_text.emptied}"
+          end
+
           where = @v.covered ? "in sequences #{range(@v.covered)}" : "in the ledger"
-          "#{PREFIX} BROKEN — #{number(count)} break#{'s' unless count == 1} #{where}; #{uncovered}"
+          "#{PREFIX} BROKEN — #{breaks} #{where}; #{uncovered}; #{anchor_text}"
         end
 
+        def anchor_text = (@anchor_text ||= AnchorText.new(@v))
+
         def no_chain
           "#{PREFIX} NO CHAIN — #{number(@v.uncovered_now + @v.unchained_now)} entries, " \
             "none chained; nothing was verified"
         end
 
-        def no_key
-          "#{PREFIX} UNVERIFIABLE — no ledger key configured; nothing was verified"
-        end
+        def no_key = "#{PREFIX} UNVERIFIABLE — no ledger key configured; nothing was verified"
 
+        # The purge clause leads here too (FR-011), but the uncovered count stays as 0.6.0 wrote it.
         def unverifiable
-          unknown = @v.breaks.select { |found| found.kind == :unknown_key }
-          rows = unknown.count { |found| entry_position?(found.sequence) }
-          checked = @v.covered ? @v.covered.size - rows : 0
-          "#{PREFIX} UNVERIFIABLE — #{number(rows)} entries signed by #{missing_keys(unknown)} " \
-            "not configured; the other #{number(checked)} in #{range(@v.covered)} show no " \
-            "breaks, which is not a pass; #{number(@v.uncovered_now)} earlier entries not covered"
-        end
-
-        # The genesis is position 0 and is not an entry.
-        def entry_position?(sequence)
-          sequence.is_a?(Integer) && sequence.positive?
-        end
-
-        def missing_keys(unknown)
-          keys = unknown.map(&:key_id).compact.uniq
-          keys.one? ? "key #{keys.first}, which is" : "keys #{keys.join(', ')}, which are"
+          "#{PREFIX} UNVERIFIABLE — #{UnverifiableText.new(@v)}; #{RetentionText.new(@v).clause}" \
+            "#{number(@v.uncovered_now)} earlier entries not covered; #{anchor_text}"
         end
 
+        # After a purge, what it removed comes first (specs/005 FR-011): no line says what was
+        # verified without saying what was purged before it.
         def uncovered
-          parts = ["#{number(@v.uncovered_now)} earlier entries not covered"]
+          retention = RetentionText.new(@v)
+          parts = [retention.clause, "#{number(@v.uncovered_now)} earlier entries not covered"]
           at_genesis = @v.uncovered_at_genesis
-          parts << " (#{number(at_genesis)} when the chain started)" if at_genesis
+          if at_genesis
+            parts << " (#{number(at_genesis)} when the chain started#{retention.pre_chain})"
+          end
           unless @v.unchained_at_genesis.to_i.zero?
             parts << "; #{number(@v.unchained_at_genesis)} unchained entries from before the " \
                      "chain started, not covered"
@@ -94,9 +100,7 @@ module Reeve
           parts.join
         end
 
-        def totals
-          "  totals: #{@v.totals.map { |kind, count| "#{kind} #{count}" }.join(', ')}"
-        end
+        def totals = "  totals: #{@v.totals.map { |kind, count| "#{kind} #{count}" }.join(', ')}"
 
         def more(listed, limit)
           remaining = listed.size - limit
@@ -105,14 +109,24 @@ module Reeve
           "  … and #{number(remaining)} more (the totals above are complete)"
         end
 
+        # One space at least between columns, so a label or kind longer than its column
+        # (`checkpoint #48,000`, `checkpoint_not_reached`) never runs into the next.
         def line_for(found)
-          kind = found.kind.to_s.ljust(KIND_WIDTH)
-          "  #{label(found).ljust(LABEL_WIDTH)}#{kind}#{found.detail}".rstrip
+          label = label(found).ljust(LABEL_WIDTH - 1)
+          kind = found.kind.to_s.ljust(KIND_WIDTH - 1)
+          "  #{label} #{kind} #{found.detail}".rstrip
         end
 
         def label(found)
-          sequence = found.sequence
           return "invocation #{found.invocation_id}" if found.invocation_id
+          return "checkpoint ##{number(found.checkpoint)}" if found.checkpoint
+          return "retention #{found.retention}" if found.retention
+          return "epoch #{found.epoch}" if found.sequence.nil? && found.epoch
+
+          position(found.sequence)
+        end
+
+        def position(sequence)
           return "—" if sequence.nil?
           return "##{range(sequence)}" if sequence.is_a?(Range)
 
@@ -129,6 +143,192 @@ module Reeve
           self.class.number(value.to_i)
         end
       end
+
+      # What an unverifiable run could not check, and what it did (the UNVERIFIABLE line). Rows
+      # signed by a key that is not configured are counted; a checkpoint whose key is missing is
+      # named apart from them and is not one (specs/004).
+      class UnverifiableText
+        def initialize(verification)
+          @v = verification
+          unknown = verification.breaks.select { |found| found.kind == :unknown_key }
+          @checkpoints, unknown = unknown.partition(&:checkpoint)
+          @records, @signed = unknown.partition(&:retention)
+        end
+
+        def to_s
+          rows = @signed.count { |found| entry?(found.sequence) }
+          checked = @v.covered ? @v.covered.size - rows : 0
+          "#{unchecked(rows)}; the other #{Report.number(checked)} in #{range} show no breaks, " \
+            "which is not a pass"
+        end
+
+        private
+
+        def unchecked(rows)
+          parts = []
+          parts << "#{Report.number(rows)} entries signed by #{missing(@signed)}" if @signed.any?
+          parts << other("checkpoint", @checkpoints) if @checkpoints.any?
+          parts << other("retention record", @records) if @records.any?
+          parts.join("; ")
+        end
+
+        # Not entries, so named apart and not counted as rows.
+        def other(noun, found)
+          count = found.size
+          "#{Report.number(count)} #{noun}#{'s' unless count == 1} signed by #{missing(found)}"
+        end
+
+        # The genesis is position 0 and is not an entry.
+        def entry?(sequence)
+          sequence.is_a?(Integer) && sequence.positive?
+        end
+
+        def missing(unknown)
+          keys = unknown.map(&:key_id).compact.uniq
+          named = keys.one? ? "key #{keys.first}, which is" : "keys #{keys.join(', ')}, which are"
+          "#{named} not configured"
+        end
+
+        def range
+          return "—" if @v.covered.nil?
+
+          "#{Report.number(@v.covered.first)}–#{Report.number(@v.covered.last)}"
+        end
+      end
+
+      # What the ledger's purges removed, in words (specs/005 FR-011): the clause that leads the
+      # coverage on every walked summary line, and the pre-chain rows a purge removed.
+      class RetentionText
+        def initialize(verification)
+          @retention = verification.retention
+          @awaiting = verification.awaiting_purge.to_i
+        end
+
+        def clause
+          boundary = @retention&.boundary
+          return "" if boundary.nil?
+
+          text = "1–#{Report.number(boundary.through_sequence)} purged (retention record " \
+                 "#{boundary.purge_number}, #{boundary.purged_at.utc.strftime('%Y-%m-%d')}); "
+          text += "#{Report.number(@awaiting)} entries awaiting purge; " if @awaiting.positive?
+          text
+        end
+
+        def pre_chain
+          purged = if @retention
+                     @retention.summaries.reject { |found| found.status == :altered }
+                               .sum(&:removed_pre_chain)
+                   else
+                     0
+                   end
+          purged.positive? ? ", #{Report.number(purged)} purged" : ""
+        end
+      end
+
+      # What a run was anchored by, in words (specs/004 FR-014, FR-015): the clause every walked
+      # summary line ends with, and the wording for a walk from a checkpoint and for an emptied
+      # ledger that checkpoints saw.
+      class AnchorText
+        # A checkpoint's time for people: "2026-10-25 14:00 UTC". A time that does not parse is
+        # shown as written; verification never depends on it.
+        def self.made_on(made_at)
+          Time.iso8601(made_at).utc.strftime("%Y-%m-%d %H:%M UTC")
+        rescue ArgumentError
+          made_at
+        end
+
+        def initialize(verification)
+          @v = verification
+        end
+
+        # The highest checkpoint that checked, and how many entries it does not cover.
+        def to_s
+          anchor = @v.anchored_at
+          return "not anchored" if anchor.nil?
+
+          after = @v.beyond_anchor.to_i
+          "anchored at ##{number(anchor.head_sequence)} (checkpoint #{made_on(anchor)}), " \
+            "#{number(after)} #{after == 1 ? 'entry' : 'entries'} after it"
+        end
+
+        def walked_from
+          start = @v.walked_from
+          from = "checkpoint ##{number(start.head_sequence)} (#{made_on(start)})"
+          walked = if @v.covered
+                     "sequences #{range} verified from #{from}"
+                   else
+                     "no entries after #{from}"
+                   end
+          "#{walked}; #{number(start.head_sequence)} earlier chained entries not walked by " \
+            "this run"
+        end
+
+        # No walk, and checkpoints that saw a chain.
+        def emptied?
+          @v.head_at_start.nil? && @v.checkpoints.any?
+        end
+
+        def emptied
+          count = @v.checkpoints.size
+          "the ledger has no chain, but #{number(count)} checkpoint#{'s' unless count == 1} " \
+            "name#{'s' if count == 1} one"
+        end
+
+        private
+
+        def number(value)
+          Report.number(value)
+        end
+
+        def made_on(checkpoint)
+          AnchorText.made_on(checkpoint.made_at)
+        end
+
+        def range
+          "#{number(@v.covered.first)}–#{number(@v.covered.last)}"
+        end
+      end
+
+      # The key epochs, one line each (specs/003 FR-013), shown once they say something the
+      # summary does not: more than one, one that is not verified, or rows chained before the
+      # first. A ledger with one verified epoch and nothing before it reads as 0.6.0's did.
+      class EpochLines
+        STATUS_WORDS = {
+          verified: "verified", key_not_configured: "key not configured", altered: "altered"
+        }.freeze
+
+        def initialize(verification)
+          @epochs = verification.epochs
+          @before = verification.before_first_epoch.to_i
+        end
+
+        def to_a
+          return [] unless shown?
+
+          lines = @epochs.map { |epoch| line(epoch) }
+          return lines unless @before.positive?
+
+          ["  before epoch 0  #{entries(@before)} chained before key epochs began", *lines]
+        end
+
+        private
+
+        def shown?
+          return false if @epochs.empty?
+
+          @epochs.size > 1 || @epochs.first.status != :verified || @before.positive?
+        end
+
+        def line(epoch)
+          "  epoch #{epoch.epoch}  key #{epoch.key_id}  " \
+            "from sequence #{Report.number(epoch.first_sequence)}  " \
+            "#{entries(epoch.entry_count)}  #{STATUS_WORDS.fetch(epoch.status)}"
+        end
+
+        def entries(count)
+          "#{Report.number(count)} #{count == 1 ? 'entry' : 'entries'}"
+        end
+      end
     end
   end
 end

data/lib/reeve/audit/chain/retention/archive.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+require "digest"
+require "fileutils"
+require "json"
+
+module Reeve
+  module Audit
+    class Chain
+      module Retention
+        # The archive a purge writes before it removes anything (specs/005 FR-007,
+        # contracts/archive-v1.md): one JSON line of header, then every purged row with every
+        # column, chain columns included, so the range can be verified on its own.
+        #
+        # Written to `path.tmp`, flushed and renamed, so a partial archive is never at `path`.
+        module Archive
+          FORMAT = "reeve-ledger-archive-v1"
+          BATCH = 1_000
+
+          module_function
+
+          # Writes the archive for `plan` and returns its SHA-256.
+          def write(path, plan, entry_class:)
+            partial = "#{path}.tmp"
+            File.open(partial, File::WRONLY | File::CREAT | File::EXCL, 0o644) do |file|
+              file.puts(JSON.generate(header(plan)))
+              each_row(plan, entry_class) { |row| file.puts(JSON.generate(line(row))) }
+              file.flush
+              file.fsync
+            end
+            File.rename(partial, path)
+            ::Digest::SHA256.file(path).hexdigest
+          rescue StandardError
+            FileUtils.rm_f(partial)
+            raise
+          end
+
+          def header(plan)
+            {
+              "format" => FORMAT, "ledger" => ChainRecord::LEDGER, "purge" => plan.purge,
+              "from_sequence" => plan.from, "through_sequence" => plan.through,
+              "through_checkpoint" => plan.checkpoint.to_line, "start" => plan.start || genesis,
+              "chained" => plan.removed_chained, "pre_chain" => plan.removed_pre_chain,
+              "reeve_version" => Reeve::VERSION
+            }.sort.to_h
+          end
+
+          # Purge 0 starts at the genesis, so the archive carries it.
+          def genesis
+            record = ChainRecord.ledger
+            fields = Canonical::GENESIS_FIELDS + ["genesis_digest"]
+            { "genesis" => fields.to_h { |field| [field, value(record[field])] } }
+          end
+
+          # Pre-chain rows first, then the chained range in sequence order, a batch at a time.
+          def each_row(plan, entry_class, &block)
+            entry_class.where("contract_version < 3").order(:id).find_each(batch_size: BATCH,
+&block)
+            after = plan.from - 1
+            loop do
+              batch = entry_class.where(chain_sequence: (after + 1)..plan.through)
+                                 .order(:chain_sequence).limit(BATCH).to_a
+              batch.each(&block)
+              break if batch.size < BATCH
+
+              after = batch.last.chain_sequence
+            end
+          end
+
+          # The header of the archive at `path`, refusing one in another format: a verifier
+          # never guesses at a format it does not know.
+          def read_header(path)
+            header = JSON.parse(File.foreach(path).first.to_s)
+            return header if header.is_a?(Hash) && header["format"] == FORMAT
+
+            found = header.is_a?(Hash) ? header["format"].inspect : "unreadable"
+            raise ArgumentError, "#{path} is not a #{FORMAT} archive (format #{found})"
+          rescue JSON::ParserError
+            raise ArgumentError, "#{path} is not a #{FORMAT} archive (its first line is not JSON)"
+          end
+
+          # The header's genesis, with its time as a Time again, for Canonical.genesis_digest.
+          def genesis_values(genesis)
+            genesis.merge("genesis_at" => Time.iso8601(genesis["genesis_at"]))
+          end
+
+          def line(row)
+            row.attributes.except("id").transform_values { |item| value(item) }.sort.to_h
+          end
+
+          def value(item)
+            item.respond_to?(:utc) ? item.utc.iso8601(6) : item
+          end
+        end
+      end
+    end
+  end
+end