reeve 0.5.0 → 0.6.0

data/lib/reeve/audit/chain/head.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+module Reeve
+  module Audit
+    class Chain
+      # The chain record's head, advanced under its lock (research R3).
+      #
+      # One instance per append, bound to the connection that append runs on; every
+      # statement here goes through that connection, so the lock, the row and the head move
+      # share one transaction and commit or roll back together.
+      #
+      #   UPDATE head_sequence by one   -- the lock, taken first
+      #   SELECT the new head_sequence and the old head_digest
+      #   ... the caller inserts the row ...
+      #   UPDATE head_digest to the row's digest
+      #
+      # Taking the lock with a write is what makes this behave the same on all three
+      # engines: SQLite cannot upgrade a read lock while another writer waits, and ignores
+      # SELECT ... FOR UPDATE.
+      class Head
+        def initialize(connection:, entry_class:)
+          @connection = connection
+          @entry_class = entry_class
+        end
+
+        # Creates the chain record with its genesis — link 0, head 0 — unless it exists.
+        #
+        # Deliberately not part of the write that follows. Run first inside it, an UPDATE that
+        # matched no row took a gap lock on MySQL's unique index, and two writers starting the
+        # chain at once each tried to INSERT into the gap the other held: a deadlock, found
+        # by CI, invisible on PostgreSQL, which has no gap locks. On its own, two concurrent
+        # INSERTs of one unique name serialize: one wins and the other gets a duplicate key,
+        # which means the chain has started and is all it needs to know.
+        #
+        # So a started chain can exist with nothing in it yet (head 0) — if the first chained
+        # write then fails, say. The verifier reports that as no chain, not as a pass.
+        def start!(signer)
+          return if connection.select_value("SELECT 1 FROM #{TABLE} WHERE #{named}")
+
+          genesis = genesis_for(signer)
+          digest = Canonical.genesis_digest(signer.signing_secret, genesis)
+          values = genesis.merge(name: ChainRecord::LEDGER, genesis_digest: digest,
+                                 head_sequence: 0, head_digest: digest)
+          quoted = values.values.map { |value| connection.quote(value) }
+
+          entry_class.transaction(requires_new: true) do
+            connection.insert("INSERT INTO #{TABLE} (#{values.keys.join(', ')}) " \
+                              "VALUES (#{quoted.join(', ')})")
+          end
+        rescue ActiveRecord::RecordNotUnique
+          nil
+        end
+
+        # Locks the head and returns the next sequence and the digest it links to. The chain
+        # must have been started.
+        def advance!
+          vanished = increment.zero?
+          raise ActiveRecord::StatementInvalid, "the ledger chain record vanished" if vanished
+
+          head = connection.select_one("SELECT head_sequence, head_digest FROM #{TABLE} " \
+                                       "WHERE #{named}")
+          [Integer(head["head_sequence"]), head["head_digest"]]
+        end
+
+        def point_at!(digest)
+          connection.update("UPDATE #{TABLE} SET head_digest = #{connection.quote(digest)} " \
+                            "WHERE #{named}")
+        end
+
+        private
+
+        attr_reader :connection, :entry_class
+
+        def increment
+          connection.update("UPDATE #{TABLE} SET head_sequence = head_sequence + 1 WHERE #{named}")
+        end
+
+        def genesis_for(signer)
+          {
+            genesis_at: Time.now.utc.floor(6),
+            pre_chain_count: entry_class.where("contract_version < 3").count,
+            pre_chain_unchained_count:
+              entry_class.where(UNCHAINED).count,
+            genesis_key_id: signer.signing_id
+          }
+        end
+
+        def named
+          "name = #{connection.quote(ChainRecord::LEDGER)}"
+        end
+      end
+    end
+  end
+end

data/lib/reeve/audit/chain/keyring.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Reeve
+  module Audit
+    class Chain
+      # The keys one chained write or one verification may use (specs/002 FR-011, R7).
+      #
+      # One key signs; any number verify. Rows name their key by fingerprint, so lookup is
+      # by fingerprint too, and a key the ring does not hold is simply absent — the verifier
+      # turns that into "unverifiable", never into "tampered".
+      #
+      #   Keyring.from(Reeve.config)                        # what the recorder signs with
+      #   Keyring.new(signing: nil, retired: [auditor_key]) # verify-only, e.g. an auditor
+      class Keyring
+        def self.from(config)
+          new(signing: config.ledger_key, retired: config.retired_ledger_keys)
+        end
+
+        def initialize(signing:, retired: [])
+          @signing_secret = signing
+          @by_id = {}
+          [signing, *retired].compact.each do |secret|
+            @by_id[LedgerKey.fingerprint(secret)] ||= secret
+          end
+          @by_id.freeze
+          freeze
+        end
+
+        attr_reader :signing_secret
+
+        def signing_id
+          signing_secret && LedgerKey.fingerprint(signing_secret)
+        end
+
+        def signing?
+          !signing_secret.nil?
+        end
+
+        def for(key_id)
+          @by_id[key_id]
+        end
+
+        def empty?
+          @by_id.empty?
+        end
+
+        def inspect
+          "#<#{self.class.name} signing=#{signing_id.inspect} keys=#{@by_id.keys.inspect}>"
+        end
+      end
+    end
+  end
+end

data/lib/reeve/audit/chain/normalizer.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+module Reeve
+  module Audit
+    class Chain
+      # Puts a row into the form the database will hand back, before it is written
+      # (specs/002 FR-003, research R2 step 1).
+      #
+      # The digest is computed at write time and recomputed at verification from what the
+      # database returns. The two only agree if nothing is left for the database to change,
+      # so anything it would change is changed here first:
+      #
+      # * JSON values go through the column's own ActiveRecord type, encode and decode — the
+      #   exact path a write takes. Symbol keys become strings, a Time inside the arguments
+      #   becomes the string ActiveRecord stores, NaN becomes nil.
+      # * `occurred_at` is floored to the precision the column keeps, so the database has
+      #   no fraction to round (MySQL rounds; it does not truncate), and moved to UTC.
+      #
+      # What remains — JSON key order and how a number is spelled — is the database's to
+      # change, and Canonical makes it irrelevant.
+      #
+      # It cannot recover a value the database alters for good: an integer beyond double
+      # precision in a MySQL JSON column is stored rounded. That shows up as a false break,
+      # never a false pass, and is documented.
+      class Normalizer
+        JSON_FIELDS = %i[arguments record_ids metadata].freeze
+        TIME_FIELD = :occurred_at
+
+        # Undeclared precision: MySQL's DATETIME is whole seconds, the others keep
+        # microseconds.
+        WHOLE_SECOND_ADAPTERS = /mysql|trilogy/i
+        DEFAULT_PRECISION = 6
+
+        def self.effective_precision(column, adapter_name)
+          return column.precision unless column.precision.nil?
+
+          adapter_name.to_s.match?(WHOLE_SECOND_ADAPTERS) ? 0 : DEFAULT_PRECISION
+        end
+
+        def initialize(entry_class)
+          @entry_class = entry_class
+        end
+
+        # A new hash; the row passed in is not modified.
+        def call(row)
+          # Symbol keys throughout, whatever the caller built: a host recorder (FR-007) may use
+          # strings, and every lookup below — and the replay check after it — is by symbol.
+          normalized = row.to_h.transform_keys(&:to_sym)
+          JSON_FIELDS.each do |field|
+            next unless normalized.key?(field)
+
+            normalized[field] = round_trip(field, normalized[field])
+          end
+          normalized[TIME_FIELD] = floor(normalized[TIME_FIELD]) if normalized[TIME_FIELD]
+          normalized
+        end
+
+        private
+
+        attr_reader :entry_class
+
+        def round_trip(field, value)
+          return nil if value.nil?
+
+          type = json_types[field]
+          type.deserialize(type.serialize(value))
+        end
+
+        def floor(time)
+          time.to_time.getutc.floor(precision)
+        end
+
+        def json_types
+          @json_types ||= JSON_FIELDS.to_h do |field|
+            [field, entry_class.type_for_attribute(field.to_s)]
+          end
+        end
+
+        def precision
+          @precision ||= self.class.effective_precision(
+            entry_class.columns_hash.fetch(TIME_FIELD.to_s),
+            entry_class.connection_db_config.adapter
+          )
+        end
+      end
+    end
+  end
+end

data/lib/reeve/audit/chain/report.rb

@@ -0,0 +1,134 @@
+# frozen_string_literal: true
+
+module Reeve
+  module Audit
+    class Chain
+      # A Verification in words (contracts/ledger-chain.md "Output").
+      #
+      # Every summary line carries its own coverage. A reader who copies only the first line
+      # into a ticket still copies how much of the ledger the word in it covers — "INTACT"
+      # never travels without "sequences 1–n" and the count of what was not covered.
+      #
+      # Unknown-key breaks are summarized, not listed: there is one per row a missing key
+      # signed, and the summary already names the key and the count.
+      class Report
+        LABEL_WIDTH = 14
+        KIND_WIDTH = 17
+        PREFIX = "reeve ledger:"
+
+        def self.number(value)
+          digits = value.abs.to_s.reverse.scan(/\d{1,3}/).join(",").reverse
+          value.negative? ? "-#{digits}" : digits
+        end
+
+        def initialize(verification)
+          @v = verification
+        end
+
+        def to_s(limit:)
+          listed = @v.breaks.reject { |found| found.kind == :unknown_key }
+          return summary if listed.empty?
+
+          lines = listed.first(limit).map { |found| line_for(found) }
+          [summary, totals, *lines, more(listed, limit)].compact.join("\n")
+        end
+
+        private
+
+        def summary
+          case @v.status
+          when :intact then intact
+          when :broken then broken
+          when :no_chain then no_chain
+          else @v.reason == :no_key ? no_key : unverifiable
+          end
+        end
+
+        def intact
+          "#{PREFIX} INTACT — sequences #{range(@v.covered)} verified " \
+            "(head at start: #{number(@v.head_at_start)}); #{uncovered}"
+        end
+
+        def broken
+          count = @v.breaks.size
+          where = @v.covered ? "in sequences #{range(@v.covered)}" : "in the ledger"
+          "#{PREFIX} BROKEN — #{number(count)} break#{'s' unless count == 1} #{where}; #{uncovered}"
+        end
+
+        def no_chain
+          "#{PREFIX} NO CHAIN — #{number(@v.uncovered_now + @v.unchained_now)} entries, " \
+            "none chained; nothing was verified"
+        end
+
+        def no_key
+          "#{PREFIX} UNVERIFIABLE — no ledger key configured; nothing was verified"
+        end
+
+        def unverifiable
+          unknown = @v.breaks.select { |found| found.kind == :unknown_key }
+          rows = unknown.count { |found| entry_position?(found.sequence) }
+          checked = @v.covered ? @v.covered.size - rows : 0
+          "#{PREFIX} UNVERIFIABLE — #{number(rows)} entries signed by #{missing_keys(unknown)} " \
+            "not configured; the other #{number(checked)} in #{range(@v.covered)} show no " \
+            "breaks, which is not a pass; #{number(@v.uncovered_now)} earlier entries not covered"
+        end
+
+        # The genesis is position 0 and is not an entry.
+        def entry_position?(sequence)
+          sequence.is_a?(Integer) && sequence.positive?
+        end
+
+        def missing_keys(unknown)
+          keys = unknown.map(&:key_id).compact.uniq
+          keys.one? ? "key #{keys.first}, which is" : "keys #{keys.join(', ')}, which are"
+        end
+
+        def uncovered
+          parts = ["#{number(@v.uncovered_now)} earlier entries not covered"]
+          at_genesis = @v.uncovered_at_genesis
+          parts << " (#{number(at_genesis)} when the chain started)" if at_genesis
+          unless @v.unchained_at_genesis.to_i.zero?
+            parts << "; #{number(@v.unchained_at_genesis)} unchained entries from before the " \
+                     "chain started, not covered"
+          end
+          parts.join
+        end
+
+        def totals
+          "  totals: #{@v.totals.map { |kind, count| "#{kind} #{count}" }.join(', ')}"
+        end
+
+        def more(listed, limit)
+          remaining = listed.size - limit
+          return nil unless limit.positive? && remaining.positive?
+
+          "  … and #{number(remaining)} more (the totals above are complete)"
+        end
+
+        def line_for(found)
+          kind = found.kind.to_s.ljust(KIND_WIDTH)
+          "  #{label(found).ljust(LABEL_WIDTH)}#{kind}#{found.detail}".rstrip
+        end
+
+        def label(found)
+          sequence = found.sequence
+          return "invocation #{found.invocation_id}" if found.invocation_id
+          return "—" if sequence.nil?
+          return "##{range(sequence)}" if sequence.is_a?(Range)
+
+          sequence.zero? ? "genesis" : "##{number(sequence)}"
+        end
+
+        def range(value)
+          return "—" if value.nil?
+
+          "#{number(value.first)}–#{number(value.last)}"
+        end
+
+        def number(value)
+          self.class.number(value.to_i)
+        end
+      end
+    end
+  end
+end

data/lib/reeve/audit/chain/verification.rb

@@ -0,0 +1,110 @@
+# frozen_string_literal: true
+
+module Reeve
+  module Audit
+    class Chain
+      # What a verification established, and — as importantly — what it did not
+      # (specs/002 FR-012–FR-014, data-model "Verification result").
+      #
+      # The status says whether the chain held; the coverage says over what. They are
+      # separate fields so that no reader can take one for the other: an intact chain that
+      # covers the last 300 rows of a ledger of 5,300 is intact *for those 300*.
+      class Verification
+        STATUSES = %i[intact broken unverifiable no_chain].freeze
+
+        # `sequence` is an Integer or a Range (a gap), or nil for a break that has no
+        # position; an unchained row has none and is named by `invocation_id` instead.
+        # `key_id` is set for unknown_key: the fingerprint of the key the verifier lacked.
+        Break = Struct.new(:kind, :sequence, :detail, :invocation_id, :key_id, keyword_init: true)
+
+        DEFAULT_LIMIT = 100
+
+        KINDS = %i[
+          digest_mismatch missing duplicate unchained head_mismatch beyond_head
+          genesis_altered chain_record_missing unknown_key
+        ].freeze
+
+        # The status rules (data-model), for a verification that had a key and found a
+        # chain; the no-key and no-chain answers are given before a walk ever starts. Any
+        # break but an unknown key is a broken chain. Rows only an absent key could check
+        # leave it unverifiable — never intact, and never called tampered.
+        def self.status_for(breaks)
+          kinds = breaks.map(&:kind)
+          return :broken if (kinds - [:unknown_key]).any?
+          return :unverifiable if kinds.include?(:unknown_key)
+
+          :intact
+        end
+
+        attr_reader :status, :covered, :breaks, :head_at_start, :head_at_end,
+                    :uncovered_now, :uncovered_at_genesis, :unchained_now, :unchained_at_genesis,
+                    :reason
+
+        def initialize(status:, covered: nil, breaks: [], head_at_start: nil, head_at_end: nil,
+                       uncovered_now: 0, uncovered_at_genesis: nil, unchained_now: 0,
+                       unchained_at_genesis: nil, reason: nil)
+          raise ArgumentError, "unknown status #{status.inspect}" unless STATUSES.include?(status)
+
+          @status = status
+          @covered = covered
+          @breaks = breaks.freeze
+          @head_at_start = head_at_start
+          @head_at_end = head_at_end
+          @uncovered_now = uncovered_now
+          @uncovered_at_genesis = uncovered_at_genesis
+          @unchained_now = unchained_now
+          @unchained_at_genesis = unchained_at_genesis
+          @reason = reason
+          freeze
+        end
+
+        def intact?
+          status == :intact
+        end
+
+        # Never capped, whatever the display does with the list.
+        def totals
+          breaks.each_with_object(Hash.new(0)) { |b, counts| counts[b.kind] += 1 }
+        end
+
+        def unverifiable_count
+          totals[:unknown_key]
+        end
+
+        # The report a person reads (contracts/ledger-chain.md "Output"). The first line
+        # states the status and what it covers together; no line says "intact" or "pass"
+        # without the covered range and the uncovered count beside it (SC-006). `limit`
+        # caps the breaks listed, never the totals.
+        def to_s(limit: DEFAULT_LIMIT)
+          Report.new(self).to_s(limit: limit)
+        end
+
+        def as_json(*)
+          {
+            "status" => status.to_s, "covered" => range_json(covered),
+            "head_at_start" => head_at_start, "head_at_end" => head_at_end,
+            "uncovered_now" => uncovered_now, "uncovered_at_genesis" => uncovered_at_genesis,
+            "unchained_now" => unchained_now, "unchained_at_genesis" => unchained_at_genesis,
+            "unverifiable_count" => unverifiable_count,
+            "totals" => totals.transform_keys(&:to_s), "reason" => reason&.to_s,
+            "breaks" => breaks.map { |b| break_json(b) }
+          }
+        end
+
+        private
+
+        def break_json(found)
+          {
+            "kind" => found.kind.to_s, "sequence" => range_json(found.sequence),
+            "detail" => found.detail, "invocation_id" => found.invocation_id,
+            "key_id" => found.key_id
+          }
+        end
+
+        def range_json(value)
+          value.is_a?(Range) ? { "first" => value.first, "last" => value.last } : value
+        end
+      end
+    end
+  end
+end

data/lib/reeve/audit/chain/verifier.rb

@@ -0,0 +1,160 @@
+# frozen_string_literal: true
+
+module Reeve
+  module Audit
+    class Chain
+      # Checks the chain and reports every place it no longer holds (specs/002 FR-012–FR-015,
+      # research R6).
+      #
+      # The chain record is read first; its head is where the walk (Walk) stops. After the
+      # walk the record is checked against what was found — the genesis against its digest,
+      # the head against the last row — and the head is read again. Rows between the two
+      # reads are writes that committed during the walk: legitimate, and left for the next
+      # run. A row above the second read cannot be legitimate, because reeve commits a row
+      # and its head move together, so it is reported. Without that second read, a row
+      # forged past the head would never be examined by any run.
+      #
+      # == What it cannot say
+      #
+      # A failing digest means the row's content, its position or its predecessor changed;
+      # which one is not knowable from the digest, and the report does not guess. Rows
+      # written before the chain began are counted, never verified.
+      class Verifier
+        BATCH_SIZE = 1_000
+
+        RECORD_MISSING = "chained rows exist but the chain record does not"
+        UNCHAINED_ROW = "a contract-3 row outside the chain, written after it started (by clock)"
+
+        def initialize(entry_class:, keyring:, batch_size: BATCH_SIZE)
+          @entry_class = entry_class
+          @keyring = keyring
+          @batch_size = batch_size
+        end
+
+        def call
+          return unwalked(:unverifiable, :no_key) if keyring.empty?
+
+          ChainRecord.require_same_database!(entry_class)
+
+          record = read_head
+          return unwalked(:no_chain) unless record&.head_sequence&.positive? || chained_rows?
+
+          verify_chain(record)
+        end
+
+        private
+
+        attr_reader :entry_class, :keyring, :batch_size, :breaks
+
+        def verify_chain(record)
+          @breaks = []
+          walk = walk(record)
+          check_record(record, walk)
+          head_at_end = record && read_head
+          check_beyond_head(head_at_end || record)
+          check_unchained(record)
+
+          result(record, head_at_end, walk.last_sequence)
+        end
+
+        def walk(record)
+          Walk.new(entry_class: entry_class, keyring: keyring, batch_size: batch_size,
+                   link: record&.genesis_digest, report: method(:add))
+              .call(record&.head_sequence)
+        end
+
+        # A seam as well as a read: the specs wrap it to land a write mid-walk.
+        def read_head
+          ChainRecord.uncached { ChainRecord.find_by(name: ChainRecord::LEDGER) }
+        end
+
+        def check_record(record, walk)
+          return add(:chain_record_missing, nil, RECORD_MISSING) if record.nil?
+
+          check_genesis(record)
+          head = record.head_sequence
+          last_digest = walk.last_sequence.zero? ? record.genesis_digest : walk.last_digest
+          return if walk.last_sequence == head && last_digest == record.head_digest
+
+          add(:head_mismatch, head,
+              "the head is at #{head}, the last chained row found is #{walk.last_sequence}")
+        end
+
+        def check_genesis(record)
+          key_id = record.genesis_key_id
+          secret = keyring.for(key_id)
+          if secret.nil?
+            return add(:unknown_key, 0, "the genesis is signed by key #{key_id}", nil, key_id)
+          end
+          return if Canonical.genesis_digest(secret, record) == record.genesis_digest
+
+          add(:genesis_altered, 0, "the genesis does not match its digest")
+        end
+
+        def check_beyond_head(record)
+          return if record.nil?
+
+          candidates = entry_class.where("chain_sequence > ?", record.head_sequence)
+                                  .order(:chain_sequence).pluck(:chain_sequence)
+          return if candidates.empty?
+
+          # Read the head once more, after the search. A row committed between the last read
+          # and the search moved the head with it, so the head now covers it; a row above
+          # this read cannot have been written by reeve.
+          head = read_head&.head_sequence || record.head_sequence
+          detail = "numbered past the head (#{Report.number(head)} at the end of the walk); " \
+                   "reeve did not write it"
+          candidates.select { |sequence| sequence > head }
+                    .each { |sequence| add(:beyond_head, sequence, detail) }
+        end
+
+        # Contract-3 rows with no digest, beyond the number that already existed when the
+        # chain began (research R5). Which ones are new is known only by clock, and the
+        # detail says so.
+        def check_unchained(record)
+          return if count_unchained <= (record ? record.pre_chain_unchained_count : 0)
+
+          scope = entry_class.where(UNCHAINED)
+          scope = scope.where("occurred_at > ?", record.genesis_at) if record
+          scope.order(:occurred_at).pluck(:invocation_id).each do |invocation_id|
+            add(:unchained, nil, UNCHAINED_ROW, invocation_id)
+          end
+        end
+
+        def add(kind, sequence, detail = nil, invocation_id = nil, key_id = nil)
+          breaks << Verification::Break.new(kind: kind, sequence: sequence, detail: detail,
+                                            invocation_id: invocation_id, key_id: key_id)
+        end
+
+        def result(record, head_at_end, last_sequence)
+          Verification.new(
+            status: Verification.status_for(breaks), breaks: breaks,
+            covered: covered(record, last_sequence),
+            head_at_start: record&.head_sequence, head_at_end: head_at_end&.head_sequence,
+            uncovered_now: count_uncovered, uncovered_at_genesis: record&.pre_chain_count,
+            unchained_now: count_unchained, unchained_at_genesis: record&.pre_chain_unchained_count
+          )
+        end
+
+        def covered(record, last_sequence)
+          last = record ? record.head_sequence : last_sequence
+          last.positive? ? 1..last : nil
+        end
+
+        # An answer given before any walk: no key to walk with, or no chain to walk.
+        def unwalked(status, reason = nil)
+          Verification.new(status: status, reason: reason,
+                           uncovered_now: count_uncovered, unchained_now: count_unchained)
+        end
+
+        def chained_rows?
+          entry_class.where.not(chain_sequence: nil).exists?
+        end
+
+        def count_uncovered = (@count_uncovered ||= entry_class.where("contract_version < 3").count)
+
+        def count_unchained = (@count_unchained ||= entry_class.where(UNCHAINED).count)
+      end
+    end
+  end
+end