reeve 0.6.0 → 0.8.0

data/lib/generators/reeve/install/templates/create_audit_entries.rb.tt

@@ -16,8 +16,21 @@
 #   -- the chain record: its head moves on every write
 #   GRANT INSERT, SELECT, UPDATE ON reeve_audit_chains TO app_role;
 #   GRANT USAGE  ON SEQUENCE reeve_audit_chains_id_seq TO app_role;
+#   -- the key epochs: written once each, never updated
+#   GRANT INSERT, SELECT ON reeve_audit_chain_epochs TO app_role;
+#   GRANT USAGE  ON SEQUENCE reeve_audit_chain_epochs_id_seq TO app_role;
+#   -- retention records: the application only reads them
+#   GRANT SELECT ON reeve_audit_retentions TO app_role;
 #
-# Every row is chained to the one before it with a keyed digest (audit-entry contract 3),
+#   -- the retention role, used only by `bin/rails reeve:purge_ledger`
+#   GRANT SELECT ON reeve_audit_entries, reeve_audit_chains, reeve_audit_chain_epochs,
+#                   reeve_audit_retentions TO retention_role;
+#   GRANT DELETE ON reeve_audit_entries TO retention_role;
+#   GRANT INSERT ON reeve_audit_retentions TO retention_role;
+#   GRANT USAGE  ON SEQUENCE reeve_audit_retentions_id_seq TO retention_role;
+#
+# Every row is chained to the one before it with a keyed digest, and names the key epoch
+# that signed it (audit-entry contract 4),
 # so a row changed, removed or reordered outside reeve is reported by
 # `bin/rails reeve:verify_ledger`. That is tamper-evidence, not tamper-proofing: it
 # detects a change after the fact and prevents nothing — the grant above is what
@@ -26,10 +39,10 @@
 # reeve_audit_chains; that role needs UPDATE there, which is why the chain table is
 # granted separately from the ledger.
 #
-# Also explicitly out of scope for now, so you can plan around it:
-#   * no retention, rotation or archival — this table grows until you prune it, and
-#     pruning is your policy, run by a role that is allowed to delete. Deleting rows
-#     breaks the chain by design.
+# The table grows until you purge it. Deleting rows by hand breaks the chain by design and
+# is reported as tampering; `bin/rails reeve:purge_ledger` removes the oldest entries up to
+# a checkpoint, archives them, and leaves a signed record in reeve_audit_retentions so the
+# rest still verifies. It runs on the retention role above, never the application's.
 #
 # `invocation_id` is a UUID stored as a string rather than a native `uuid` column,
 # because reeve supports every database ActiveRecord does and only PostgreSQL has the
@@ -79,6 +92,10 @@ class CreateReeveAuditEntries < ActiveRecord::Migration[7.0]
       t.bigint   :chain_sequence
       t.string   :chain_key_id,   limit: 16
       t.string   :chain_digest,   limit: 64
+
+      # The key epoch whose key signed the row (contract 4), inside the digest. Nullable for
+      # the same reason as the chain columns, and NULL on an unchained row.
+      t.integer  :chain_epoch
     end
 
     add_index :reeve_audit_entries, :invocation_id, unique: true
@@ -111,5 +128,35 @@ class CreateReeveAuditEntries < ActiveRecord::Migration[7.0]
     end
 
     add_index :reeve_audit_chains, :name, unique: true
+
+    # One row per key that has signed the ledger (contract 4): written when the key first
+    # signs, or by `bin/rails reeve:rotate_ledger_key`, and never updated. A key opens one
+    # epoch only, hence the unique key_id.
+    create_table :reeve_audit_chain_epochs do |t|
+      t.integer  :epoch,          null: false
+      t.string   :key_id,         null: false, limit: 16
+      t.bigint   :first_sequence, null: false
+      t.datetime :opened_at,      null: false, precision: 6
+      t.string   :epoch_digest,   null: false, limit: 64
+    end
+
+    add_index :reeve_audit_chain_epochs, :epoch, unique: true
+    add_index :reeve_audit_chain_epochs, :key_id, unique: true
+
+    create_table :reeve_audit_retentions do |t|
+      t.integer  :purge_number,      null: false
+      t.bigint   :through_sequence,  null: false
+      t.string   :through_digest,    null: false, limit: 64
+      t.bigint   :previous_through,  null: false
+      t.bigint   :removed_chained,   null: false
+      t.bigint   :removed_pre_chain, null: false
+      t.boolean  :archived,          null: false
+      t.string   :archive_sha256,    limit: 64
+      t.datetime :purged_at,         null: false, precision: 6
+      t.string   :key_id,            null: false, limit: 16
+      t.string   :retention_digest,  null: false, limit: 64
+    end
+
+    add_index :reeve_audit_retentions, :purge_number, unique: true
   end
 end

data/lib/generators/reeve/install/templates/initializer.rb.tt

@@ -102,3 +102,13 @@ end
 
 # The ledger itself is opt-in, so the core stays loadable without ActiveRecord.
 require "reeve/audit"
+
+# Checkpoints of the chain, kept where the database's users cannot change them,
+# catch what the chain alone cannot: the newest entries removed with the head
+# rewound, or the whole ledger deleted. Run `bin/rails reeve:checkpoint_ledger`
+# on a schedule; `reeve:verify_ledger` then checks against them. One anchor per
+# environment, on storage outside the database host — a local file protects
+# nothing:
+#
+#   Reeve.config.ledger_anchor =
+#     Reeve::Audit::FileAnchor.new("/mnt/ledger-anchor/#{Rails.env}.checkpoints")

data/lib/generators/reeve/upgrade/templates/add_key_epochs_to_reeve_audit_entries.rb.tt

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+# Audit-entry contract 3 → 4. Generated by `bin/rails generate reeve:upgrade`.
+#
+# Records which key signed which stretch of the chain. Each key that signs the ledger opens
+# a numbered key epoch, recorded once in reeve_audit_chain_epochs, and every row reeve chains
+# from here on names its epoch in `chain_epoch`, inside its digest. Verification then reports
+# a row whose key or epoch was changed as tampering, where before it could only say the row
+# was signed by a key it did not have.
+#
+# Rows already in the table are not touched. Rows chained before the first epoch opens keep
+# exactly the verification they had; the column is nullable for that reason. The first
+# chained write after this migration opens epoch 0 with the configured ledger key.
+#
+# Rotate the key with `bin/rails reeve:rotate_ledger_key` (see the CHANGELOG).
+#
+# The application role needs INSERT and SELECT on the new table, never UPDATE:
+#
+#   GRANT INSERT, SELECT ON reeve_audit_chain_epochs TO app_role;
+class AddKeyEpochsToReeveAuditEntries < ActiveRecord::Migration[7.0]
+  def change
+    add_column :reeve_audit_entries, :chain_epoch, :integer
+
+    create_table :reeve_audit_chain_epochs do |t|
+      t.integer  :epoch,          null: false
+      t.string   :key_id,         null: false, limit: 16
+      t.bigint   :first_sequence, null: false
+      t.datetime :opened_at,      null: false, precision: 6
+      t.string   :epoch_digest,   null: false, limit: 64
+    end
+
+    add_index :reeve_audit_chain_epochs, :epoch, unique: true
+    add_index :reeve_audit_chain_epochs, :key_id, unique: true
+  end
+end

data/lib/generators/reeve/upgrade/templates/add_ledger_retention_to_reeve_audit_entries.rb.tt

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+# Ledger retention. Generated by `bin/rails generate reeve:upgrade`. The audit-entry contract
+# stays 4: no column of reeve_audit_entries changes.
+#
+# Adds reeve_audit_retentions, where `bin/rails reeve:purge_ledger` records each purge: the
+# checkpoint it purged up to, how many rows it removed, and the archive it wrote, signed with
+# the ledger key. Verification reads it to tell a purge from a deletion. Nothing is purged
+# until you run the task.
+#
+# The application role only reads the new table. The retention role, which only the purge
+# uses, needs DELETE on the ledger and INSERT here:
+#
+#   GRANT SELECT ON reeve_audit_retentions TO app_role;
+#   GRANT SELECT ON reeve_audit_entries, reeve_audit_chains, reeve_audit_chain_epochs,
+#                   reeve_audit_retentions TO retention_role;
+#   GRANT DELETE ON reeve_audit_entries TO retention_role;
+#   GRANT INSERT ON reeve_audit_retentions TO retention_role;
+class AddLedgerRetentionToReeveAuditEntries < ActiveRecord::Migration[7.0]
+  def change
+    create_table :reeve_audit_retentions do |t|
+      t.integer  :purge_number,      null: false
+      t.bigint   :through_sequence,  null: false
+      t.string   :through_digest,    null: false, limit: 64
+      t.bigint   :previous_through,  null: false
+      t.bigint   :removed_chained,   null: false
+      t.bigint   :removed_pre_chain, null: false
+      t.boolean  :archived,          null: false
+      t.string   :archive_sha256,    limit: 64
+      t.datetime :purged_at,         null: false, precision: 6
+      t.string   :key_id,            null: false, limit: 16
+      t.string   :retention_digest,  null: false, limit: 64
+    end
+
+    add_index :reeve_audit_retentions, :purge_number, unique: true
+  end
+end

data/lib/generators/reeve/upgrade/upgrade_generator.rb

@@ -2,6 +2,7 @@
 
 require "rails/generators"
 require "rails/generators/active_record"
+require_relative "upgrade_notes"
 
 module Reeve
   module Generators
@@ -50,11 +51,26 @@ module Reeve
           creates: %w[reeve_audit_chains],
           template: "add_ledger_chain_to_reeve_audit_entries.rb.tt",
           destination: "add_ledger_chain_to_reeve_audit_entries.rb"
+        },
+        {
+          contract: 4,
+          adds: %w[chain_epoch],
+          creates: %w[reeve_audit_chain_epochs],
+          template: "add_key_epochs_to_reeve_audit_entries.rb.tt",
+          destination: "add_key_epochs_to_reeve_audit_entries.rb"
+        },
+        # Retention (specs/005) adds a table and no column, so the contract stays 4.
+        {
+          contract: 4,
+          note: :retention,
+          adds: [],
+          creates: %w[reeve_audit_retentions],
+          template: "add_ledger_retention_to_reeve_audit_entries.rb.tt",
+          destination: "add_ledger_retention_to_reeve_audit_entries.rb"
         }
       ].freeze
 
-      KEY_LINE = "config.ledger_key = " \
-                 'Rails.application.key_generator.generate_key("reeve audit ledger", 32)'
+      KEY_LINE = UpgradeNotes::KEY_LINE
 
       source_root File.expand_path("templates", __dir__)
 
@@ -117,29 +133,7 @@ module Reeve
           Existing rows are stamped with the contract they were actually written under,
           not with the current one.
         NEXT
-        message += chain_message if @emitted.any? { |step| step[:contract] == 3 }
-        message
-      end
-
-      # Contract 3 changes what the application must configure, not only the table: without
-      # a key, every guarded call fails closed once the migration runs. The host is reading
-      # this output at exactly the moment they need to know that.
-      def chain_message
-        <<~CHAIN
-
-          The ledger is now chained (contract 3). Add this line to config/initializers/reeve.rb
-          before you deploy, or every guarded call fails closed:
-
-            #{KEY_LINE}
-
-          It derives the key from secret_key_base, so rotating secret_key_base rotates the
-          ledger key: keep the old derived key in config.retired_ledger_keys, or entries it
-          signed become unverifiable. A dedicated key avoids that coupling.
-
-          Very large ledger on PostgreSQL? The migration builds a unique index that blocks
-          writes while it scans; the migration's header shows how to build it with
-          algorithm: :concurrently instead.
-        CHAIN
+        message + UpgradeNotes.for(@emitted)
       end
 
       # Asked of the table rather than of a version the host recorded: the table is the

data/lib/generators/reeve/upgrade/upgrade_notes.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module Reeve
+  module Generators
+    # What `reeve:upgrade` tells the host after writing a step whose contract changes more
+    # than the table. The host is reading this output at exactly the moment they need it.
+    module UpgradeNotes
+      KEY_LINE = "config.ledger_key = " \
+                 'Rails.application.key_generator.generate_key("reeve audit ledger", 32)'
+
+      # Contract 3 changes what the application must configure, not only the table: without
+      # a key, every guarded call fails closed once the migration runs.
+      CHAIN = <<~CHAIN.freeze
+
+        The ledger is now chained (contract 3). Add this line to config/initializers/reeve.rb
+        before you deploy, or every guarded call fails closed:
+
+          #{KEY_LINE}
+
+        It derives the key from secret_key_base, so rotating secret_key_base rotates the
+        ledger key: keep the old derived key in config.retired_ledger_keys, or entries it
+        signed become unverifiable. A dedicated key avoids that coupling.
+
+        Very large ledger on PostgreSQL? The migration builds a unique index that blocks
+        writes while it scans; the migration's header shows how to build it with
+        algorithm: :concurrently instead.
+      CHAIN
+
+      # Contract 4 changes how a rotation is done, so the host learns the command here.
+      EPOCHS = <<~EPOCHS
+
+        The ledger now records key epochs (contract 4): the first chained write opens
+        epoch 0 with the configured ledger key. To rotate the key later, set the new key
+        as config.ledger_key, keep the old one in config.retired_ledger_keys, deploy, and run:
+
+          bin/rails reeve:rotate_ledger_key
+
+        The application role needs INSERT and SELECT on reeve_audit_chain_epochs.
+      EPOCHS
+
+      # Retention changes nothing until the host runs a purge, but the purge needs a role the
+      # application does not have, and the host should hear that before it needs one.
+      RETENTION = <<~RETENTION
+
+        The ledger can now be purged (reeve_audit_retentions): bin/rails reeve:purge_ledger
+        removes the oldest entries up to a checkpoint and records the purge, so the rest still
+        verifies. Nothing is purged until you run it. The application role needs SELECT on
+        the new table; the purge runs on a retention role of its own (see the migration).
+      RETENTION
+
+      # Keyed by the step's note, or by its contract when it has none.
+      BY_STEP = { 3 => CHAIN, 4 => EPOCHS, retention: RETENTION }.freeze
+
+      module_function
+
+      # The notes for the steps just written, oldest first.
+      def for(steps)
+        steps.filter_map { |step| BY_STEP[step.fetch(:note, step[:contract])] }.join
+      end
+    end
+  end
+end

data/lib/reeve/audit/chain/anchoring.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+module Reeve
+  module Audit
+    class Chain
+      # The verifier's checkpoints (specs/004): what it checks the ledger against, and the one,
+      # if any, it walks from instead of the genesis.
+      #
+      # Checkpoints come from outside the database (FR-010). With none, the verifier is exactly
+      # 0.7.0's and its report says the run was not anchored.
+      class Anchoring
+        attr_reader :checkpoints, :from
+
+        # `from` is :latest, a Checkpoint, or nil to walk from the genesis (Checkpoint.start).
+        def initialize(entry_class:, keyring:, checkpoints:, from: nil)
+          @entry_class = entry_class
+          @keyring = keyring
+          @checkpoints = checkpoints
+          @from = Checkpoint.start(from, checkpoints)
+        end
+
+        def any?
+          checkpoints.any?
+        end
+
+        # The latest retention record that holds (specs/005), or nil: the walk never starts
+        # below it, and a `from` at or below it is moot.
+        attr_reader :floor
+
+        def floor=(record)
+          @floor = record
+          @from = nil if record && from && from.head_sequence <= record.through_sequence
+        end
+
+        # Where the walk starts, and the digest its first row links to.
+        def start
+          return from.head_sequence if from
+          return floor.through_sequence if floor
+
+          0
+        end
+
+        def link(record)
+          return from.head_digest if from
+          return floor.through_digest if floor
+
+          record&.genesis_digest
+        end
+
+        # Every checkpoint, after the walk, against the head read after it. The checked
+        # checkpoint with the highest sequence anchors the run (data-model).
+        def check(record, head, epoch_records, report:)
+          results = check_with(report).call(checkpoints, record: record, head: head,
+                                                         epochs: epoch_records, floor: floor)
+          anchor = results.select { |found| found.status == :checked }.max_by(&:head_sequence)
+          beyond = anchor && record && [record.head_sequence - anchor.head_sequence, 0].max
+          { checkpoints: results, anchored_at: anchor, beyond_anchor: beyond }
+        end
+
+        # A ledger with no chain, which every checkpoint saw reach an entry: emptied, not never
+        # chained (FR-013).
+        def emptied(record, report:)
+          check_with(report).call(checkpoints, record: record, head: record, epochs: [],
+                                               chained: false)
+        end
+
+        # The starting checkpoint, checked alone before the walk relies on it (FR-015). Returns
+        # its results when it does not hold, reporting its breaks; nil when it holds, so its
+        # breaks are left to the full check after the walk.
+        def failed_start(record, epoch_records, report:)
+          return nil if from.nil?
+
+          found = []
+          collect = ->(*args, **options) { found << [args, options] }
+          results = check_with(collect).call([from], record: record, head: record,
+                                                     epochs: epoch_records, floor: floor)
+          return nil if results.first.status == :checked
+
+          found.each { |args, options| report.call(*args, **options) }
+          results
+        end
+
+        private
+
+        def check_with(report)
+          CheckpointCheck.new(entry_class: @entry_class, keyring: @keyring, report: report)
+        end
+      end
+    end
+  end
+end

data/lib/reeve/audit/chain/canonical.rb

@@ -31,19 +31,36 @@ module Reeve
         ENTRY_TAG = "reeve-ledger-entry-v1"
         GENESIS_TAG = "reeve-ledger-genesis-v1"
 
+        EPOCH_TAG = "reeve-ledger-epoch-v1"
+
+        CHECKPOINT_TAG = "reeve-ledger-checkpoint-v1"
+
+        CONTRACT_3_FIELDS = %w[
+          chain_sequence chain_key_id contract_version
+          invocation_id occurred_at agent_id agent_name principal_type principal_id
+          tool_name arguments outcome rule detail record_type record_ids record_count
+          truncated derived guard duration_ms metadata
+        ].freeze
+
+        # Contract 4 (specs/003) adds the key epoch, next to the key it names.
         ENTRY_FIELDS = {
-          3 => %w[
-            chain_sequence chain_key_id contract_version
-            invocation_id occurred_at agent_id agent_name principal_type principal_id
-            tool_name arguments outcome rule detail record_type record_ids record_count
-            truncated derived guard duration_ms metadata
-          ].freeze
+          3 => CONTRACT_3_FIELDS,
+          4 => CONTRACT_3_FIELDS.dup.insert(2, "chain_epoch").freeze
         }.freeze
 
         GENESIS_FIELDS = %w[genesis_at pre_chain_count pre_chain_unchained_count
                             genesis_key_id].freeze
 
-        TIME_FIELDS = %w[occurred_at genesis_at].freeze
+        EPOCH_FIELDS = %w[epoch key_id first_sequence opened_at].freeze
+
+        # A checkpoint's signed fields (specs/004 contracts/checkpoint-v1.md), in this order.
+        CHECKPOINT_FIELDS = %w[
+          format ledger genesis_digest pre_chain_count pre_chain_unchained_count
+          head_sequence head_digest epoch epoch_digest key_id made_at reeve_version
+        ].freeze
+
+        # `purged_at` is a retention record's (Retention::Digest).
+        TIME_FIELDS = %w[occurred_at genesis_at opened_at purged_at].freeze
 
         module_function
 
@@ -56,6 +73,20 @@ module Reeve
           hmac(secret, GENESIS_TAG, dump(pairs))
         end
 
+        # An epoch record (specs/003 FR-002), linked to the epoch before it — or, for epoch 0,
+        # to the genesis — so the epochs are a chain of their own.
+        def epoch_digest(secret, epoch, previous_digest:)
+          pairs = EPOCH_FIELDS.map { |field| [field, value(epoch, field)] }
+          hmac(secret, EPOCH_TAG, dump([["previous_digest", previous_digest], *pairs]))
+        end
+
+        # A checkpoint (specs/004). `made_at` is signed as the string it is stored as, so a line
+        # verifies exactly as written, whatever its reader does with times.
+        def checkpoint_signature(secret, values)
+          pairs = CHECKPOINT_FIELDS.map { |field| [field, values.fetch(field)] }
+          hmac(secret, CHECKPOINT_TAG, dump(pairs))
+        end
+
         def entry_message_body(row, previous_digest)
           version = value(row, "contract_version")
           fields = ENTRY_FIELDS.fetch(version) do