reeve 0.7.0 → 0.8.0

data/lib/reeve/audit/chain/retention/connection.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module Reeve
+  module Audit
+    class Chain
+      module Retention
+        # The guard on config.ledger_retention_connection (specs/005 FR-006; constitution II,
+        # 1.1.0): the purge deletes only through a connection that reaches the ledger's own
+        # database and is not the application's.
+        #
+        # "Not the application's" is told by the username, which is all a connection config
+        # says about the role behind it. SQLite has no roles at all, so there the guard can
+        # only say so.
+        class Connection
+          SQLITE_WARNING = "SQLite has no database roles, so the retention connection cannot be " \
+                           "told apart from the application's; on a server database it must be"
+
+          SAME_DATABASE = %i[adapter database host port].freeze
+
+          def initialize(retention, entry_class:)
+            @retention = retention
+            @entry_class = entry_class
+          end
+
+          # nil, or why the purge must refuse: :no_retention_connection, :other_database,
+          # :application_role.
+          def problem
+            return :no_retention_connection if @retention.nil?
+            return :other_database unless same_database?
+
+            if roles? && retention_config[:username].to_s == ledger_config[:username].to_s
+              :application_role
+            end
+          end
+
+          def warnings
+            !@retention.nil? && !roles? ? [SQLITE_WARNING] : []
+          end
+
+          def pool
+            @retention.connection_pool
+          end
+
+          private
+
+          def same_database?
+            SAME_DATABASE.all? { |key| retention_config[key].to_s == ledger_config[key].to_s }
+          end
+
+          def roles?
+            !ledger_config[:adapter].to_s.include?("sqlite")
+          end
+
+          def retention_config
+            @retention.connection_db_config.configuration_hash
+          end
+
+          def ledger_config
+            @entry_class.connection_db_config.configuration_hash
+          end
+        end
+      end
+    end
+  end
+end

data/lib/reeve/audit/chain/retention/digest.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Reeve
+  module Audit
+    class Chain
+      module Retention
+        # A retention record's digest (specs/005 data-model): an HMAC under the signing key over
+        # the record's fields and the previous record's digest, or the genesis digest for purge
+        # 0, so the records are a chain of their own. Canonical's form, with a tag of its own.
+        module Digest
+          TAG = "reeve-ledger-retention-v1"
+
+          FIELDS = %w[
+            purge_number through_sequence through_digest previous_through removed_chained
+            removed_pre_chain archived archive_sha256 purged_at key_id
+          ].freeze
+
+          module_function
+
+          def call(secret, record, previous_digest:)
+            pairs = FIELDS.map { |field| [field, Canonical.value(record, field)] }
+            Canonical.hmac(secret, TAG,
+                           Canonical.dump([["previous_digest", previous_digest], *pairs]))
+          end
+        end
+      end
+    end
+  end
+end

data/lib/reeve/audit/chain/retention/plan.rb

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+module Reeve
+  module Audit
+    class Chain
+      module Retention
+        # Which checkpoint a purge stops at (specs/005 FR-002): one the anchor holds, that
+        # verification checked, above the current boundary.
+        class Boundary
+          def initialize(entry_class, checkpoints, verification)
+            @entry_class = entry_class
+            @current = current_through(verification)
+            checked = checkpoints.zip(verification.checkpoints)
+                                 .select { |_, found| found.status == :checked }.map(&:first)
+            @candidates = checked.select { |checkpoint| checkpoint.head_sequence > @current }
+                                 .uniq(&:head_sequence).sort_by(&:head_sequence)
+          end
+
+          # The checkpoint at `sequence`, or the PurgeResult that says why there is none.
+          def at(sequence)
+            return PurgeResult.nothing(:at_or_below, current: @current) if sequence <= @current
+
+            found = @candidates.find { |checkpoint| checkpoint.head_sequence == sequence }
+            found || PurgeResult.refused(:no_checkpoint, detail: nearest(sequence))
+          end
+
+          # The highest checkpoint whose entry occurred before `cutoff`.
+          def older_than(cutoff)
+            times = occurred_at(@candidates.map(&:head_sequence))
+            older = @candidates.select { |checkpoint| times[checkpoint.head_sequence]&.<(cutoff) }
+            older.last || PurgeResult.nothing(:none_older, current: @current)
+          end
+
+          private
+
+          def current_through(verification)
+            boundary = verification.retention&.boundary
+            boundary ? boundary.through_sequence : 0
+          end
+
+          def nearest(sequence)
+            below = @candidates.reverse.find { |checkpoint| checkpoint.head_sequence < sequence }
+            above = @candidates.find { |checkpoint| checkpoint.head_sequence > sequence }
+            [below, above].map do |checkpoint|
+              checkpoint ? "##{Report.number(checkpoint.head_sequence)}" : "none"
+            end
+          end
+
+          def occurred_at(sequences)
+            @entry_class.where(chain_sequence: sequences).pluck(:chain_sequence, :occurred_at).to_h
+          end
+        end
+
+        # What one purge removes and records (data-model "reeve_audit_retentions").
+        class Plan
+          attr_reader :checkpoint, :from, :through, :previous_through, :previous_digest, :purge,
+                      :removed_chained, :removed_pre_chain, :start
+
+          # `current` is the latest retention record, or nil before the first purge.
+          def initialize(entry_class, current, checkpoint)
+            @checkpoint = checkpoint
+            @through = checkpoint.head_sequence
+            current ? follow(current) : first
+            @from = @previous_through + 1
+            @removed_chained = entry_class.where(chain_sequence: from..through).count
+            @removed_pre_chain = entry_class.where("contract_version < 3").count
+          end
+
+          def record_values(archive_sha256, key_id)
+            {
+              "purge_number" => purge, "through_sequence" => through,
+              "through_digest" => checkpoint.head_digest,
+              "previous_through" => previous_through, "removed_chained" => removed_chained,
+              "removed_pre_chain" => removed_pre_chain, "archived" => !archive_sha256.nil?,
+              "archive_sha256" => archive_sha256, "purged_at" => Time.now.utc.floor(6),
+              "key_id" => key_id
+            }
+          end
+
+          private
+
+          def first
+            @previous_through = 0
+            @purge = 0
+            @previous_digest = ChainRecord.ledger.genesis_digest
+            @start = nil
+          end
+
+          def follow(current)
+            @previous_through = current.through_sequence
+            @purge = current.purge_number + 1
+            @previous_digest = current.retention_digest
+            @start = { "through_sequence" => current.through_sequence,
+                       "through_digest" => current.through_digest }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/reeve/audit/chain/retention/preflight.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Reeve
+  module Audit
+    class Chain
+      module Retention
+        # What a purge needs before it reads anything (specs/005 FR-004, FR-006, FR-007): a key,
+        # an anchor, a retention connection that is not the application's, and somewhere to put
+        # the archive, or leave to go without one. Each missing piece is a refusal that names it.
+        class Preflight
+          attr_reader :keyring, :guard
+
+          def initialize(entry_class:, archive:, no_archive:)
+            @keyring = Keyring.from(Reeve.config)
+            @guard = Connection.new(Reeve.config.ledger_retention_connection,
+                                    entry_class: entry_class)
+            @archive = archive
+            @no_archive = no_archive
+          end
+
+          def refusal
+            return PurgeResult.refused(:no_key) unless keyring.signing?
+            return PurgeResult.refused(:no_anchor) if Reeve.config.ledger_anchor.nil?
+
+            PurgeResult.refused(guard.problem) if guard.problem
+          end
+
+          # Only a new purge writes an archive; a resumed one already did.
+          def archive_refusal
+            return nil if @no_archive
+            return PurgeResult.refused(:no_archive_choice) if @archive.nil?
+            return PurgeResult.refused(:archive_exists, detail: @archive) if File.exist?(@archive)
+            return nil if writable_directory?
+
+            PurgeResult.refused(:archive_unwritable, detail: @archive)
+          end
+
+          private
+
+          def writable_directory?
+            directory = File.dirname(@archive)
+            File.directory?(directory) && File.writable?(directory)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/reeve/audit/chain/retention/purge.rb

@@ -0,0 +1,155 @@
+# frozen_string_literal: true
+
+module Reeve
+  module Audit
+    class Chain
+      module Retention
+        # `bin/rails reeve:purge_ledger` (specs/005; constitution II, 1.1.0): removes the chain's
+        # entries up to a checkpoint the anchor holds, and the rows written before the chain,
+        # leaving a signed retention record so that what remains still verifies and the
+        # removal reads as a purge, not as tampering.
+        #
+        # In this order, and nothing removed until the last step: refuse what cannot be backed;
+        # pick the boundary; require the whole ledger to verify INTACT; stop here for a dry run;
+        # write the archive; commit the record; delete in batches over the retention
+        # connection. A purge stopped during the deletes leaves rows "awaiting purge", which a
+        # rerun removes without a second record.
+        class Purge
+          BATCH_SIZE = 1_000
+
+          DAY = 86_400
+
+          EXACTLY_ONE = "give exactly one of through: (THROUGH=) and older_than: (OLDER_THAN=)"
+
+          # `through` is a checkpoint's sequence; `older_than` a number of days. `archive` is a
+          # path, or nil with `no_archive`.
+          def initialize(entry_class:, through: nil, older_than: nil, archive: nil,
+                         no_archive: false, confirm: false, batch_size: BATCH_SIZE)
+            raise ArgumentError, EXACTLY_ONE if through.nil? == older_than.nil?
+
+            @entry_class = entry_class
+            @through = through
+            @older_than = older_than
+            @archive = archive
+            @no_archive = no_archive
+            @confirm = confirm
+            @batch_size = batch_size
+          end
+
+          def call
+            @preflight = Preflight.new(entry_class: entry_class, archive: @archive,
+                                       no_archive: @no_archive)
+            refusal = @preflight.refusal
+            return refusal if refusal
+
+            verification = Chain.verify(entry_class: entry_class, checkpoints: checkpoints)
+            # INTACT first, a resume included: a forged record under a key nobody holds reads as
+            # unverifiable, and resuming from it would delete rows on the forger's word.
+            unless verification.intact?
+              return PurgeResult.refused(:not_intact, detail: verification.to_s.lines.first.chomp)
+            end
+            return resume(verification) if verification.awaiting_purge.positive?
+
+            purge(verification)
+          end
+
+          private
+
+          attr_reader :entry_class
+
+          def keyring = @preflight.keyring
+
+          def guard = @preflight.guard
+
+          def checkpoints
+            (@checkpoints ||= Checkpoint.read_all(Reeve.config.ledger_anchor.checkpoints))
+          end
+
+          # A resumed purge already wrote its archive; only a new one needs a path for it.
+          def purge(verification)
+            refusal = @preflight.archive_refusal
+            return refusal if refusal
+
+            current = verification.retention&.boundary
+            boundary = Boundary.new(entry_class, checkpoints, verification)
+            chosen = @through ? boundary.at(@through) : boundary.older_than(cutoff)
+            return chosen unless chosen.is_a?(Checkpoint)
+
+            plan = Plan.new(entry_class, current, chosen)
+            unless @confirm
+              return PurgeResult.from(plan, :dry_run, archive: @archive,
+                                                      warnings: guard.warnings)
+            end
+
+            commit(plan)
+          end
+
+          def cutoff = Time.now - (@older_than * DAY)
+
+          def commit(plan)
+            sha256 = @no_archive ? nil : Archive.write(@archive, plan, entry_class: entry_class)
+            number = insert_record(plan, sha256)
+            delete_through(plan.through)
+            PurgeResult.from(plan, :purged, archive: @archive, archive_sha256: sha256,
+                                            purge: number,
+                                            warnings: guard.warnings)
+          rescue ActiveRecord::RecordNotUnique
+            PurgeResult.refused(:purge_running)
+          end
+
+          # The record names the boundary checkpoint, what the purge removes, and the archive,
+          # signed and linked to the record before it (data-model).
+          def insert_record(plan, sha256)
+            values = plan.record_values(sha256, keyring.signing_id)
+            values["retention_digest"] =
+              Digest.call(keyring.signing_secret, values, previous_digest: plan.previous_digest)
+            guard.pool.with_connection { |connection| insert(connection, values) }
+            values["purge_number"]
+          end
+
+          def insert(connection, values)
+            quoted = values.values.map { |value| connection.quote(value) }
+            columns = values.keys.join(", ")
+            connection.insert("INSERT INTO #{RetentionRecord.table_name} (#{columns}) " \
+                              "VALUES (#{quoted.join(', ')})")
+          end
+
+          # A purge whose record is written but whose rows are not all gone: finish it.
+          def resume(verification)
+            boundary = verification.retention.boundary
+            removed = delete_through(boundary.through_sequence)
+            PurgeResult.resumed(boundary, removed, warnings: guard.warnings)
+          end
+
+          def delete_through(through)
+            removed = 0
+            loop do
+              count = delete_batch(through)
+              removed += count
+              break if count < @batch_size
+            end
+            removed
+          end
+
+          # One batch, in its own transaction: rows chained at or below the boundary, and rows
+          # written before the chain. Unchained contract-3+ rows are left (spec Clarifications).
+          def delete_batch(through)
+            guard.pool.with_connection do |connection|
+              ids = connection.select_values(doomed_ids(through)).map { |id| Integer(id) }
+              next 0 if ids.empty?
+
+              connection.delete("DELETE FROM #{entry_class.table_name} " \
+                                "WHERE id IN (#{ids.join(', ')})")
+            end
+          end
+
+          def doomed_ids(through)
+            "SELECT id FROM #{entry_class.table_name} WHERE (chain_sequence IS NOT NULL AND " \
+              "chain_sequence <= #{Integer(through)}) OR contract_version < 3 ORDER BY id " \
+              "LIMIT #{Integer(@batch_size)}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/reeve/audit/chain/retention/purge_result.rb

@@ -0,0 +1,134 @@
+# frozen_string_literal: true
+
+module Reeve
+  module Audit
+    class Chain
+      # What `bin/rails reeve:purge_ledger` did, or why it did nothing (specs/005 data-model,
+      # contracts/retention.md). A value, so a script can act on it without reading the text.
+      #
+      #   :dry_run           — what a run with CONFIRM=1 would remove; nothing removed
+      #   :purged            — removed through `through`, recorded as retention record `purge`
+      #   :nothing_to_purge  — no checkpoint above the current boundary qualifies
+      #   :refused           — nothing removed; `reason` says why
+      class PurgeResult
+        PREFIX = "reeve ledger purge:"
+        AGAIN = "run again with CONFIRM=1"
+
+        FIELDS = %i[status reason detail from through through_made_at removed_chained
+                    removed_pre_chain archive archive_sha256 purge resumed warnings].freeze
+
+        REFUSALS = {
+          no_key: "no ledger key configured (config.ledger_key)",
+          no_anchor: "no anchor configured (config.ledger_anchor): a purge must stop at a " \
+                     "checkpoint kept outside the database",
+          no_retention_connection: "no retention connection configured " \
+                                   "(config.ledger_retention_connection)",
+          other_database: "config.ledger_retention_connection reaches another database than " \
+                          "the ledger's",
+          application_role: "config.ledger_retention_connection connects as the application's " \
+                            "own database user; the purge must run on a role the application " \
+                            "does not use",
+          no_checkpoint: "no checkpoint the anchor holds is at that sequence (nearest below: %s, " \
+                         "above: %s)",
+          not_intact: "the ledger does not verify intact, and a purge never removes what it " \
+                      "cannot prove: %s",
+          no_archive_choice: "give ARCHIVE=path to keep the purged entries, or NO_ARCHIVE=1 to " \
+                             "delete them without one",
+          archive_exists: "the archive %s already exists; an archive is never overwritten",
+          archive_unwritable: "the archive %s cannot be written there",
+          purge_running: "another purge took the next retention record number; run again once " \
+                         "it is done"
+        }.freeze
+
+        attr_reader(*FIELDS)
+
+        def self.refused(reason, detail: nil)
+          new(status: :refused, reason: reason, detail: detail)
+        end
+
+        def self.nothing(reason, current:)
+          new(status: :nothing_to_purge, reason: reason, through: current)
+        end
+
+        def self.from(plan, status, **fields)
+          new(status: status, from: plan.from, through: plan.through,
+              through_made_at: plan.checkpoint.made_at, removed_chained: plan.removed_chained,
+              removed_pre_chain: plan.removed_pre_chain, **fields)
+        end
+
+        def self.resumed(record, removed, warnings:)
+          new(status: :purged, from: record.previous_through + 1, through: record.through_sequence,
+              purge: record.purge_number, removed_chained: record.removed_chained,
+              removed_pre_chain: record.removed_pre_chain, resumed: removed,
+              archive_sha256: record.archive_sha256, warnings: warnings)
+        end
+
+        def initialize(status:, warnings: [], **fields)
+          @status = status
+          @warnings = warnings.freeze
+          (FIELDS - %i[status warnings]).each do |name|
+            instance_variable_set(:"@#{name}", fields[name])
+          end
+          freeze
+        end
+
+        def dry_run? = status == :dry_run
+        def purged? = status == :purged
+        def refused? = status == :refused
+
+        def to_s
+          [first_line, *warnings.map { |warning| "warning: #{warning}" }].join("\n")
+        end
+
+        private
+
+        def first_line
+          case status
+          when :dry_run then "#{PREFIX} WOULD PURGE — #{removal}; #{archive_text}; #{AGAIN}"
+          when :purged then "#{PREFIX} PURGED — #{purged_text}"
+          when :nothing_to_purge then "#{PREFIX} NOTHING TO PURGE — #{nothing_text}"
+          else "#{PREFIX} REFUSED — #{refusal}; nothing was removed"
+          end
+        end
+
+        def purged_text
+          record = "#{removal}; retention record #{purge}"
+          return "#{record} (resumed; #{n(resumed)} rows removed this run)" if resumed
+
+          "#{record}; #{archive_text}"
+        end
+
+        def removal
+          made = through_made_at ? " (#{AnchorText.made_on(through_made_at)})" : ""
+          "sequences #{n(from || 1)}–#{n(through)} through checkpoint ##{n(through)}#{made}: " \
+            "#{n(removed_chained)} chained entries and #{n(removed_pre_chain)} pre-chain rows"
+        end
+
+        def archive_text
+          return "no archive written (NO_ARCHIVE=1)" if archive.nil?
+          return "archive to #{archive}" if dry_run?
+
+          "archive #{archive} (sha256 #{archive_sha256})"
+        end
+
+        def nothing_text
+          where = "above the current boundary (##{n(through)})"
+          return "no checkpoint old enough #{where}" unless reason == :at_or_below
+
+          "the ledger is already purged through that sequence #{where}"
+        end
+
+        def refusal
+          template = REFUSALS.fetch(reason)
+          return template unless template.include?("%s")
+
+          format(template, *Array(detail))
+        end
+
+        def n(value)
+          Report.number(value.to_i)
+        end
+      end
+    end
+  end
+end

data/lib/reeve/audit/chain/retention/records.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Reeve
+  module Audit
+    class Chain
+      # The ledger's purges (specs/005), as verification and the purge read them.
+      module Retention
+        # Every retention record, oldest first, in one query. None for a ledger never purged,
+        # or one whose table has not been migrated: it verifies exactly as before.
+        module Records
+          module_function
+
+          def read(entry_class:)
+            present = entry_class.connection_pool.with_connection do |connection|
+              connection.table_exists?(RetentionRecord.table_name)
+            end
+            present ? RetentionRecord.uncached { RetentionRecord.order(:purge_number).to_a } : []
+          end
+        end
+      end
+    end
+  end
+end

data/lib/reeve/audit/chain/verification.rb

@@ -16,8 +16,11 @@ module Reeve
         # position; an unchained row has none and is named by `invocation_id` instead.
         # `key_id` is set for unknown_key: the fingerprint of the key the verifier lacked.
         # `epoch` is set for a break about a key epoch's record (specs/003), whose sequence is nil.
-        Break = Struct.new(:kind, :sequence, :detail, :invocation_id, :key_id, :epoch,
-                           keyword_init: true)
+        # `checkpoint` is the head sequence of the checkpoint a break is about (specs/004); such a
+        # break has no sequence of its own.
+        # `retention` is the number of the retention record a break is about (specs/005).
+        Break = Struct.new(:kind, :sequence, :detail, :invocation_id, :key_id, :epoch, :checkpoint,
+                           :retention, keyword_init: true)
 
         # One key epoch as verification found it (specs/003 FR-013). `entry_count` counts the
         # walked rows that named it; `status` is :verified, :key_not_configured or :altered.
@@ -37,6 +40,8 @@ module Reeve
           digest_mismatch missing duplicate unchained head_mismatch beyond_head
           genesis_altered chain_record_missing unknown_key
           epoch_missing epoch_mismatch epoch_altered
+          checkpoint_not_reached checkpoint_mismatch checkpoint_altered
+          retention_altered
         ].freeze
 
         # The status rules (data-model), for a verification that had a key and found a
@@ -53,11 +58,15 @@ module Reeve
 
         attr_reader :status, :covered, :breaks, :head_at_start, :head_at_end,
                     :uncovered_now, :uncovered_at_genesis, :unchained_now, :unchained_at_genesis,
-                    :reason, :epochs, :before_first_epoch
+                    :reason, :epochs, :before_first_epoch,
+                    :checkpoints, :anchored_at, :beyond_anchor, :walked_from,
+                    :retention, :awaiting_purge
 
         def initialize(status:, covered: nil, breaks: [], head_at_start: nil, head_at_end: nil,
                        uncovered_now: 0, uncovered_at_genesis: nil, unchained_now: 0,
-                       unchained_at_genesis: nil, reason: nil, epochs: [], before_first_epoch: nil)
+                       unchained_at_genesis: nil, reason: nil, epochs: [], before_first_epoch: nil,
+                       checkpoints: [], anchored_at: nil, beyond_anchor: nil, walked_from: nil,
+                       retention: nil, awaiting_purge: 0)
           raise ArgumentError, "unknown status #{status.inspect}" unless STATUSES.include?(status)
 
           @status = status
@@ -72,12 +81,16 @@ module Reeve
           @reason = reason
           @epochs = epochs.freeze
           @before_first_epoch = before_first_epoch
+          @checkpoints = checkpoints.freeze
+          @anchored_at = anchored_at
+          @beyond_anchor = beyond_anchor
+          @walked_from = walked_from
+          @retention = retention
+          @awaiting_purge = awaiting_purge
           freeze
         end
 
-        def intact?
-          status == :intact
-        end
+        def intact? = status == :intact
 
         # Never capped, whatever the display does with the list.
         def totals
@@ -111,14 +124,23 @@ module Reeve
 
         def listing_json
           { "breaks" => breaks.map { |b| break_json(b) }, "epochs" => epochs.map(&:as_json),
-            "before_first_epoch" => before_first_epoch }
+            "before_first_epoch" => before_first_epoch, "checkpoints" => checkpoints.map(&:as_json),
+            "anchored_at" => point_json(anchored_at), "beyond_anchor" => beyond_anchor,
+            "walked_from" => point_json(walked_from), "retention" => retention&.as_json,
+            "awaiting_purge" => awaiting_purge }
+        end
+
+        def point_json(checkpoint)
+          checkpoint && { "head_sequence" => checkpoint.head_sequence,
+                          "made_at" => checkpoint.made_at }
         end
 
         def break_json(found)
           {
             "kind" => found.kind.to_s, "sequence" => range_json(found.sequence),
             "detail" => found.detail, "invocation_id" => found.invocation_id,
-            "key_id" => found.key_id, "epoch" => found.epoch
+            "key_id" => found.key_id, "epoch" => found.epoch, "checkpoint" => found.checkpoint,
+            "retention" => found.retention
           }
         end