reeve 0.6.0 → 0.8.0

data/lib/reeve/audit/chain/retention/archive_check.rb

@@ -0,0 +1,179 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Reeve
+  module Audit
+    class Chain
+      module Retention
+        # Verifies an archive on its own (specs/005 FR-012, contracts/archive-v1.md): its boundary
+        # checkpoint is genuine, its start link is the genesis (or the boundary before it), and
+        # every chained row, rebuilt as an unsaved entry so it casts as the database did,
+        # recomputes its digest against the row before it, ending at the checkpoint's digest.
+        #
+        # Needs the entries table's columns, for the casts, but none of its rows.
+        class ArchiveCheck
+          BOUNDARY = "its boundary checkpoint"
+
+          def initialize(path, keyring:, entry_class:)
+            @path = path
+            @keyring = keyring
+            @entry_class = entry_class
+            @breaks = []
+          end
+
+          def call
+            header = Archive.read_header(@path)
+            checkpoint = Checkpoint.parse(header.fetch("through_checkpoint"), position: 1)
+            return result(header, checkpoint) unless genuine?(checkpoint)
+
+            check_end(header, checkpoint, walk(header, start_link(header, checkpoint)))
+            result(header, checkpoint)
+          end
+
+          private
+
+          attr_reader :keyring, :breaks
+
+          def genuine?(checkpoint)
+            signed = checkpoint.signed_by(keyring)
+            key = checkpoint.key_id
+            add(:unknown_key, nil, "#{BOUNDARY} is signed by key #{key}", key_id: key) if
+              signed == :key_not_configured
+            if signed == :altered
+              add(:checkpoint_altered, nil,
+                  "#{BOUNDARY} does not match its signature")
+            end
+            signed == :valid
+          end
+
+          # Purge 0 starts at the genesis, which is recomputed under its own key and must be the
+          # one the checkpoint names; a later purge starts at the previous boundary's digest,
+          # which the first row's own digest then binds.
+          def start_link(header, checkpoint)
+            genesis = header.dig("start", "genesis")
+            return header.dig("start", "through_digest") unless genesis
+
+            digest = genesis["genesis_digest"]
+            secret = keyring.for(genesis["genesis_key_id"])
+            if secret.nil?
+              add(:unknown_key, 0, "the genesis is signed by key #{genesis['genesis_key_id']}",
+                  key_id: genesis["genesis_key_id"])
+            elsif Canonical.genesis_digest(secret, Archive.genesis_values(genesis)) != digest ||
+                  checkpoint.genesis_digest != digest
+              add(:genesis_altered, 0,
+                  "the archive's genesis does not match its digest or its checkpoint")
+            end
+            digest
+          end
+
+          # Every row after the header: pre-chain rows counted, chained rows checked in order.
+          def walk(header, link)
+            expected = header["from_sequence"]
+            last = nil
+            @chained = @pre_chain = 0
+            File.foreach(@path).drop(1).each do |line|
+              entry = @entry_class.new(JSON.parse(line))
+              next @pre_chain += 1 if entry.chain_sequence.nil?
+
+              expected, link = visit(entry, expected, link)
+              last = entry
+            end
+            last
+          end
+
+          def visit(entry, expected, link)
+            @chained += 1
+            sequence = entry.chain_sequence
+            if sequence != expected
+              kind = sequence < expected ? :duplicate : :missing
+              add(kind, sequence < expected ? sequence : expected..(sequence - 1),
+                  "out of order in the archive")
+            end
+            check_digest(entry, link)
+            [sequence + 1, entry.chain_digest]
+          end
+
+          def check_digest(entry, link)
+            secret = keyring.for(entry.chain_key_id)
+            if secret.nil?
+              return add(:unknown_key, entry.chain_sequence, "signed by key #{entry.chain_key_id}",
+                         key_id: entry.chain_key_id)
+            end
+            return if Canonical.entry_digest(secret, entry,
+                                             previous_digest: link) == entry.chain_digest
+
+            add(:digest_mismatch, entry.chain_sequence, Walk::MISMATCH)
+          rescue ArgumentError => e
+            add(:digest_mismatch, entry.chain_sequence, e.message)
+          end
+
+          def check_end(header, checkpoint, last)
+            reached = last&.chain_sequence == header["through_sequence"] &&
+                      last&.chain_digest == checkpoint.head_digest
+            unless reached
+              add(:checkpoint_not_reached, nil, "the archive does not end at its boundary " \
+                                                "checkpoint " \
+                                                "##{Report.number(checkpoint.head_sequence)}")
+            end
+            return if @chained == header["chained"] && @pre_chain == header["pre_chain"]
+
+            add(:missing, nil, "the header counts #{header['chained']} chained and " \
+                               "#{header['pre_chain']} " \
+                               "pre-chain rows; the archive holds #{@chained} and #{@pre_chain}")
+          end
+
+          def add(kind, sequence, detail, key_id: nil)
+            breaks << Verification::Break.new(kind: kind, sequence: sequence, detail: detail,
+                                              key_id: key_id)
+          end
+
+          def result(header, checkpoint)
+            ArchiveResult.new(status: Verification.status_for(breaks), breaks: breaks,
+                              covered: header["from_sequence"]..header["through_sequence"],
+                              checkpoint: checkpoint, pre_chain: @pre_chain.to_i)
+          end
+        end
+
+        # What verifying an archive found, in words (`bin/rails reeve:verify_archive`).
+        ArchiveResult = Struct.new(:status, :breaks, :covered, :checkpoint, :pre_chain,
+                                   keyword_init: true) do
+          def intact? = status == :intact
+
+          def to_s
+            listed = breaks.reject { |found| found.kind == :unknown_key }
+            [summary, *listed.map { |found| line(found) }].join("\n")
+          end
+
+          private
+
+          def summary
+            word, found =
+              case status
+              when :intact then ["INTACT", "sequences #{range} verified, ending at #{ending}"]
+              when :broken then ["BROKEN", "#{n(breaks.size)} breaks in sequences #{range}"]
+              else ["UNVERIFIABLE", "signed by a key not configured; nothing in #{range} verified"]
+              end
+            "reeve archive: #{word} — #{found}; #{counted}"
+          end
+
+          def range = "#{n(covered.first)}–#{n(covered.last)}"
+
+          def ending
+            "checkpoint ##{n(checkpoint.head_sequence)} (#{AnchorText.made_on(checkpoint.made_at)})"
+          end
+
+          def counted = "#{n(pre_chain)} pre-chain rows counted, not verified"
+
+          def line(found)
+            sequence = found.sequence
+            where = sequence.is_a?(Range) ? "##{sequence.first}–#{sequence.last}" : "##{sequence}"
+            "  #{sequence.nil? ? '—' : where} #{found.kind} #{found.detail}"
+          end
+
+          def n(value) = Report.number(value.to_i)
+        end
+      end
+    end
+  end
+end

data/lib/reeve/audit/chain/retention/check.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+module Reeve
+  module Audit
+    class Chain
+      module Retention
+        # Checks the retention records (specs/005 FR-009): numbered 0 to n, each signed under a
+        # key this verifier holds, each linked to the one before and record 0 to the genesis,
+        # each boundary following on from the last. Leaves the boundary the walk starts after:
+        # the last record of the unbroken run from record 0.
+        #
+        # A record whose key is not held cannot be checked; it still sets the boundary (the
+        # rows it accounts for are gone either way) and makes the run unverifiable, never
+        # intact and never broken. A record that fails ends the run: the walk starts at the one
+        # before it, so the rows the failing record claims to account for are reported missing.
+        class Check
+          Summary = Struct.new(:purge, :through_sequence, :purged_at, :removed_chained,
+                               :removed_pre_chain, :archived, :status, keyword_init: true) do
+            def as_json(*)
+              to_h.transform_keys(&:to_s).merge("status" => status.to_s)
+            end
+          end
+
+          Result = Struct.new(:boundary, :summaries, keyword_init: true) do
+            def as_json(*)
+              { "through" => boundary&.through_sequence, "records" => summaries.map(&:as_json) }
+            end
+          end
+
+          # `report` receives (kind, sequence, detail, invocation_id, key_id, retention:).
+          def initialize(keyring:, report:)
+            @keyring = keyring
+            @report = report
+          end
+
+          def call(records, genesis_digest:)
+            boundary = nil
+            previous = nil
+            summaries = records.each_with_index.map do |record, index|
+              status = judge(record, index, previous,
+                             previous ? previous.retention_digest : genesis_digest)
+              boundary = record if boundary.equal?(previous) && status != :altered
+              previous = record
+              summary(record, status)
+            end
+            Result.new(boundary: boundary, summaries: summaries)
+          end
+
+          private
+
+          attr_reader :keyring
+
+          def judge(record, index, previous, link)
+            if record.purge_number != index
+              return altered(record,
+                             "is numbered #{record.purge_number}, not #{index}")
+            end
+            return altered(record, "does not follow on from the purge before it") unless follows?(
+              record, previous
+            )
+
+            secret = keyring.for(record.key_id)
+            if secret.nil?
+              @report.call(:unknown_key, nil, "signed by key #{record.key_id}", nil, record.key_id,
+                           retention: record.purge_number)
+              return :key_not_configured
+            end
+            return :verified if Digest.call(secret, record, previous_digest: link) ==
+                                record.retention_digest
+
+            altered(record, "does not match its digest; it was edited, or a record before it was")
+          end
+
+          def follows?(record, previous)
+            through = previous ? previous.through_sequence : 0
+            record.previous_through == through && record.through_sequence > through
+          end
+
+          def altered(record, detail)
+            @report.call(:retention_altered, nil, detail, nil, nil, retention: record.purge_number)
+            :altered
+          end
+
+          def summary(record, status)
+            Summary.new(purge: record.purge_number, through_sequence: record.through_sequence,
+                        purged_at: record.purged_at, removed_chained: record.removed_chained,
+                        removed_pre_chain: record.removed_pre_chain, archived: record.archived,
+                        status: status)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/reeve/audit/chain/retention/connection.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module Reeve
+  module Audit
+    class Chain
+      module Retention
+        # The guard on config.ledger_retention_connection (specs/005 FR-006; constitution II,
+        # 1.1.0): the purge deletes only through a connection that reaches the ledger's own
+        # database and is not the application's.
+        #
+        # "Not the application's" is told by the username, which is all a connection config
+        # says about the role behind it. SQLite has no roles at all, so there the guard can
+        # only say so.
+        class Connection
+          SQLITE_WARNING = "SQLite has no database roles, so the retention connection cannot be " \
+                           "told apart from the application's; on a server database it must be"
+
+          SAME_DATABASE = %i[adapter database host port].freeze
+
+          def initialize(retention, entry_class:)
+            @retention = retention
+            @entry_class = entry_class
+          end
+
+          # nil, or why the purge must refuse: :no_retention_connection, :other_database,
+          # :application_role.
+          def problem
+            return :no_retention_connection if @retention.nil?
+            return :other_database unless same_database?
+
+            if roles? && retention_config[:username].to_s == ledger_config[:username].to_s
+              :application_role
+            end
+          end
+
+          def warnings
+            !@retention.nil? && !roles? ? [SQLITE_WARNING] : []
+          end
+
+          def pool
+            @retention.connection_pool
+          end
+
+          private
+
+          def same_database?
+            SAME_DATABASE.all? { |key| retention_config[key].to_s == ledger_config[key].to_s }
+          end
+
+          def roles?
+            !ledger_config[:adapter].to_s.include?("sqlite")
+          end
+
+          def retention_config
+            @retention.connection_db_config.configuration_hash
+          end
+
+          def ledger_config
+            @entry_class.connection_db_config.configuration_hash
+          end
+        end
+      end
+    end
+  end
+end

data/lib/reeve/audit/chain/retention/digest.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Reeve
+  module Audit
+    class Chain
+      module Retention
+        # A retention record's digest (specs/005 data-model): an HMAC under the signing key over
+        # the record's fields and the previous record's digest, or the genesis digest for purge
+        # 0, so the records are a chain of their own. Canonical's form, with a tag of its own.
+        module Digest
+          TAG = "reeve-ledger-retention-v1"
+
+          FIELDS = %w[
+            purge_number through_sequence through_digest previous_through removed_chained
+            removed_pre_chain archived archive_sha256 purged_at key_id
+          ].freeze
+
+          module_function
+
+          def call(secret, record, previous_digest:)
+            pairs = FIELDS.map { |field| [field, Canonical.value(record, field)] }
+            Canonical.hmac(secret, TAG,
+                           Canonical.dump([["previous_digest", previous_digest], *pairs]))
+          end
+        end
+      end
+    end
+  end
+end

data/lib/reeve/audit/chain/retention/plan.rb

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+module Reeve
+  module Audit
+    class Chain
+      module Retention
+        # Which checkpoint a purge stops at (specs/005 FR-002): one the anchor holds, that
+        # verification checked, above the current boundary.
+        class Boundary
+          def initialize(entry_class, checkpoints, verification)
+            @entry_class = entry_class
+            @current = current_through(verification)
+            checked = checkpoints.zip(verification.checkpoints)
+                                 .select { |_, found| found.status == :checked }.map(&:first)
+            @candidates = checked.select { |checkpoint| checkpoint.head_sequence > @current }
+                                 .uniq(&:head_sequence).sort_by(&:head_sequence)
+          end
+
+          # The checkpoint at `sequence`, or the PurgeResult that says why there is none.
+          def at(sequence)
+            return PurgeResult.nothing(:at_or_below, current: @current) if sequence <= @current
+
+            found = @candidates.find { |checkpoint| checkpoint.head_sequence == sequence }
+            found || PurgeResult.refused(:no_checkpoint, detail: nearest(sequence))
+          end
+
+          # The highest checkpoint whose entry occurred before `cutoff`.
+          def older_than(cutoff)
+            times = occurred_at(@candidates.map(&:head_sequence))
+            older = @candidates.select { |checkpoint| times[checkpoint.head_sequence]&.<(cutoff) }
+            older.last || PurgeResult.nothing(:none_older, current: @current)
+          end
+
+          private
+
+          def current_through(verification)
+            boundary = verification.retention&.boundary
+            boundary ? boundary.through_sequence : 0
+          end
+
+          def nearest(sequence)
+            below = @candidates.reverse.find { |checkpoint| checkpoint.head_sequence < sequence }
+            above = @candidates.find { |checkpoint| checkpoint.head_sequence > sequence }
+            [below, above].map do |checkpoint|
+              checkpoint ? "##{Report.number(checkpoint.head_sequence)}" : "none"
+            end
+          end
+
+          def occurred_at(sequences)
+            @entry_class.where(chain_sequence: sequences).pluck(:chain_sequence, :occurred_at).to_h
+          end
+        end
+
+        # What one purge removes and records (data-model "reeve_audit_retentions").
+        class Plan
+          attr_reader :checkpoint, :from, :through, :previous_through, :previous_digest, :purge,
+                      :removed_chained, :removed_pre_chain, :start
+
+          # `current` is the latest retention record, or nil before the first purge.
+          def initialize(entry_class, current, checkpoint)
+            @checkpoint = checkpoint
+            @through = checkpoint.head_sequence
+            current ? follow(current) : first
+            @from = @previous_through + 1
+            @removed_chained = entry_class.where(chain_sequence: from..through).count
+            @removed_pre_chain = entry_class.where("contract_version < 3").count
+          end
+
+          def record_values(archive_sha256, key_id)
+            {
+              "purge_number" => purge, "through_sequence" => through,
+              "through_digest" => checkpoint.head_digest,
+              "previous_through" => previous_through, "removed_chained" => removed_chained,
+              "removed_pre_chain" => removed_pre_chain, "archived" => !archive_sha256.nil?,
+              "archive_sha256" => archive_sha256, "purged_at" => Time.now.utc.floor(6),
+              "key_id" => key_id
+            }
+          end
+
+          private
+
+          def first
+            @previous_through = 0
+            @purge = 0
+            @previous_digest = ChainRecord.ledger.genesis_digest
+            @start = nil
+          end
+
+          def follow(current)
+            @previous_through = current.through_sequence
+            @purge = current.purge_number + 1
+            @previous_digest = current.retention_digest
+            @start = { "through_sequence" => current.through_sequence,
+                       "through_digest" => current.through_digest }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/reeve/audit/chain/retention/preflight.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Reeve
+  module Audit
+    class Chain
+      module Retention
+        # What a purge needs before it reads anything (specs/005 FR-004, FR-006, FR-007): a key,
+        # an anchor, a retention connection that is not the application's, and somewhere to put
+        # the archive, or leave to go without one. Each missing piece is a refusal that names it.
+        class Preflight
+          attr_reader :keyring, :guard
+
+          def initialize(entry_class:, archive:, no_archive:)
+            @keyring = Keyring.from(Reeve.config)
+            @guard = Connection.new(Reeve.config.ledger_retention_connection,
+                                    entry_class: entry_class)
+            @archive = archive
+            @no_archive = no_archive
+          end
+
+          def refusal
+            return PurgeResult.refused(:no_key) unless keyring.signing?
+            return PurgeResult.refused(:no_anchor) if Reeve.config.ledger_anchor.nil?
+
+            PurgeResult.refused(guard.problem) if guard.problem
+          end
+
+          # Only a new purge writes an archive; a resumed one already did.
+          def archive_refusal
+            return nil if @no_archive
+            return PurgeResult.refused(:no_archive_choice) if @archive.nil?
+            return PurgeResult.refused(:archive_exists, detail: @archive) if File.exist?(@archive)
+            return nil if writable_directory?
+
+            PurgeResult.refused(:archive_unwritable, detail: @archive)
+          end
+
+          private
+
+          def writable_directory?
+            directory = File.dirname(@archive)
+            File.directory?(directory) && File.writable?(directory)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/reeve/audit/chain/retention/purge.rb

@@ -0,0 +1,155 @@
+# frozen_string_literal: true
+
+module Reeve
+  module Audit
+    class Chain
+      module Retention
+        # `bin/rails reeve:purge_ledger` (specs/005; constitution II, 1.1.0): removes the chain's
+        # entries up to a checkpoint the anchor holds, and the rows written before the chain,
+        # leaving a signed retention record so that what remains still verifies and the
+        # removal reads as a purge, not as tampering.
+        #
+        # In this order, and nothing removed until the last step: refuse what cannot be backed;
+        # pick the boundary; require the whole ledger to verify INTACT; stop here for a dry run;
+        # write the archive; commit the record; delete in batches over the retention
+        # connection. A purge stopped during the deletes leaves rows "awaiting purge", which a
+        # rerun removes without a second record.
+        class Purge
+          BATCH_SIZE = 1_000
+
+          DAY = 86_400
+
+          EXACTLY_ONE = "give exactly one of through: (THROUGH=) and older_than: (OLDER_THAN=)"
+
+          # `through` is a checkpoint's sequence; `older_than` a number of days. `archive` is a
+          # path, or nil with `no_archive`.
+          def initialize(entry_class:, through: nil, older_than: nil, archive: nil,
+                         no_archive: false, confirm: false, batch_size: BATCH_SIZE)
+            raise ArgumentError, EXACTLY_ONE if through.nil? == older_than.nil?
+
+            @entry_class = entry_class
+            @through = through
+            @older_than = older_than
+            @archive = archive
+            @no_archive = no_archive
+            @confirm = confirm
+            @batch_size = batch_size
+          end
+
+          def call
+            @preflight = Preflight.new(entry_class: entry_class, archive: @archive,
+                                       no_archive: @no_archive)
+            refusal = @preflight.refusal
+            return refusal if refusal
+
+            verification = Chain.verify(entry_class: entry_class, checkpoints: checkpoints)
+            # INTACT first, a resume included: a forged record under a key nobody holds reads as
+            # unverifiable, and resuming from it would delete rows on the forger's word.
+            unless verification.intact?
+              return PurgeResult.refused(:not_intact, detail: verification.to_s.lines.first.chomp)
+            end
+            return resume(verification) if verification.awaiting_purge.positive?
+
+            purge(verification)
+          end
+
+          private
+
+          attr_reader :entry_class
+
+          def keyring = @preflight.keyring
+
+          def guard = @preflight.guard
+
+          def checkpoints
+            (@checkpoints ||= Checkpoint.read_all(Reeve.config.ledger_anchor.checkpoints))
+          end
+
+          # A resumed purge already wrote its archive; only a new one needs a path for it.
+          def purge(verification)
+            refusal = @preflight.archive_refusal
+            return refusal if refusal
+
+            current = verification.retention&.boundary
+            boundary = Boundary.new(entry_class, checkpoints, verification)
+            chosen = @through ? boundary.at(@through) : boundary.older_than(cutoff)
+            return chosen unless chosen.is_a?(Checkpoint)
+
+            plan = Plan.new(entry_class, current, chosen)
+            unless @confirm
+              return PurgeResult.from(plan, :dry_run, archive: @archive,
+                                                      warnings: guard.warnings)
+            end
+
+            commit(plan)
+          end
+
+          def cutoff = Time.now - (@older_than * DAY)
+
+          def commit(plan)
+            sha256 = @no_archive ? nil : Archive.write(@archive, plan, entry_class: entry_class)
+            number = insert_record(plan, sha256)
+            delete_through(plan.through)
+            PurgeResult.from(plan, :purged, archive: @archive, archive_sha256: sha256,
+                                            purge: number,
+                                            warnings: guard.warnings)
+          rescue ActiveRecord::RecordNotUnique
+            PurgeResult.refused(:purge_running)
+          end
+
+          # The record names the boundary checkpoint, what the purge removes, and the archive,
+          # signed and linked to the record before it (data-model).
+          def insert_record(plan, sha256)
+            values = plan.record_values(sha256, keyring.signing_id)
+            values["retention_digest"] =
+              Digest.call(keyring.signing_secret, values, previous_digest: plan.previous_digest)
+            guard.pool.with_connection { |connection| insert(connection, values) }
+            values["purge_number"]
+          end
+
+          def insert(connection, values)
+            quoted = values.values.map { |value| connection.quote(value) }
+            columns = values.keys.join(", ")
+            connection.insert("INSERT INTO #{RetentionRecord.table_name} (#{columns}) " \
+                              "VALUES (#{quoted.join(', ')})")
+          end
+
+          # A purge whose record is written but whose rows are not all gone: finish it.
+          def resume(verification)
+            boundary = verification.retention.boundary
+            removed = delete_through(boundary.through_sequence)
+            PurgeResult.resumed(boundary, removed, warnings: guard.warnings)
+          end
+
+          def delete_through(through)
+            removed = 0
+            loop do
+              count = delete_batch(through)
+              removed += count
+              break if count < @batch_size
+            end
+            removed
+          end
+
+          # One batch, in its own transaction: rows chained at or below the boundary, and rows
+          # written before the chain. Unchained contract-3+ rows are left (spec Clarifications).
+          def delete_batch(through)
+            guard.pool.with_connection do |connection|
+              ids = connection.select_values(doomed_ids(through)).map { |id| Integer(id) }
+              next 0 if ids.empty?
+
+              connection.delete("DELETE FROM #{entry_class.table_name} " \
+                                "WHERE id IN (#{ids.join(', ')})")
+            end
+          end
+
+          def doomed_ids(through)
+            "SELECT id FROM #{entry_class.table_name} WHERE (chain_sequence IS NOT NULL AND " \
+              "chain_sequence <= #{Integer(through)}) OR contract_version < 3 ORDER BY id " \
+              "LIMIT #{Integer(@batch_size)}"
+          end
+        end
+      end
+    end
+  end
+end