reeve 0.4.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +176 -1
- data/README.md +202 -12
- data/lib/generators/reeve/install/templates/create_audit_entries.rb.tt +41 -3
- data/lib/generators/reeve/install/templates/initializer.rb.tt +20 -0
- data/lib/generators/reeve/upgrade/templates/add_ledger_chain_to_reeve_audit_entries.rb.tt +51 -0
- data/lib/generators/reeve/upgrade/upgrade_generator.rb +40 -2
- data/lib/reeve/audit/chain/canonical.rb +129 -0
- data/lib/reeve/audit/chain/head.rb +94 -0
- data/lib/reeve/audit/chain/keyring.rb +53 -0
- data/lib/reeve/audit/chain/normalizer.rb +88 -0
- data/lib/reeve/audit/chain/report.rb +134 -0
- data/lib/reeve/audit/chain/verification.rb +110 -0
- data/lib/reeve/audit/chain/verifier.rb +160 -0
- data/lib/reeve/audit/chain/walk.rb +123 -0
- data/lib/reeve/audit/chain.rb +168 -0
- data/lib/reeve/audit/chain_record.rb +61 -0
- data/lib/reeve/audit/isolated_recorder.rb +5 -0
- data/lib/reeve/audit/recorder.rb +32 -19
- data/lib/reeve/audit.rb +24 -4
- data/lib/reeve/authorization.rb +1 -11
- data/lib/reeve/configuration.rb +90 -2
- data/lib/reeve/decision.rb +6 -1
- data/lib/reeve/errors.rb +28 -0
- data/lib/reeve/inventory/report.rb +146 -0
- data/lib/reeve/inventory.rb +212 -0
- data/lib/reeve/invocation.rb +3 -1
- data/lib/reeve/ledger_key.rb +47 -0
- data/lib/reeve/railtie.rb +15 -0
- data/lib/reeve/tasks.rb +62 -0
- data/lib/reeve/testing/assertions.rb +2 -2
- data/lib/reeve/testing/checks/contract_version.rb +2 -1
- data/lib/reeve/testing/checks/endpoint_coverage.rb +133 -0
- data/lib/reeve/testing/checks/ledger_chain.rb +116 -0
- data/lib/reeve/testing/checks.rb +43 -7
- data/lib/reeve/testing/compliance_assertions.rb +4 -1
- data/lib/reeve/testing/compliance_suite.rb +4 -1
- data/lib/reeve/testing/ledger.rb +11 -1
- data/lib/reeve/testing/matchers/pass_reeve_check.rb +1 -1
- data/lib/reeve/testing/report.rb +30 -6
- data/lib/reeve/testing/result.rb +28 -8
- data/lib/reeve/testing.rb +20 -4
- data/lib/reeve/version.rb +1 -1
- data/lib/reeve.rb +3 -0
- metadata +20 -2
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Reeve
|
|
4
|
+
module Audit
|
|
5
|
+
class Chain
|
|
6
|
+
# Checks the chain and reports every place it no longer holds (specs/002 FR-012–FR-015,
|
|
7
|
+
# research R6).
|
|
8
|
+
#
|
|
9
|
+
# The chain record is read first; its head is where the walk (Walk) stops. After the
|
|
10
|
+
# walk the record is checked against what was found — the genesis against its digest,
|
|
11
|
+
# the head against the last row — and the head is read again. Rows between the two
|
|
12
|
+
# reads are writes that committed during the walk: legitimate, and left for the next
|
|
13
|
+
# run. A row above the second read cannot be legitimate, because reeve commits a row
|
|
14
|
+
# and its head move together, so it is reported. Without that second read, a row
|
|
15
|
+
# forged past the head would never be examined by any run.
|
|
16
|
+
#
|
|
17
|
+
# == What it cannot say
|
|
18
|
+
#
|
|
19
|
+
# A failing digest means the row's content, its position or its predecessor changed;
|
|
20
|
+
# which one is not knowable from the digest, and the report does not guess. Rows
|
|
21
|
+
# written before the chain began are counted, never verified.
|
|
22
|
+
class Verifier
|
|
23
|
+
BATCH_SIZE = 1_000
|
|
24
|
+
|
|
25
|
+
RECORD_MISSING = "chained rows exist but the chain record does not"
|
|
26
|
+
UNCHAINED_ROW = "a contract-3 row outside the chain, written after it started (by clock)"
|
|
27
|
+
|
|
28
|
+
def initialize(entry_class:, keyring:, batch_size: BATCH_SIZE)
|
|
29
|
+
@entry_class = entry_class
|
|
30
|
+
@keyring = keyring
|
|
31
|
+
@batch_size = batch_size
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def call
|
|
35
|
+
return unwalked(:unverifiable, :no_key) if keyring.empty?
|
|
36
|
+
|
|
37
|
+
ChainRecord.require_same_database!(entry_class)
|
|
38
|
+
|
|
39
|
+
record = read_head
|
|
40
|
+
return unwalked(:no_chain) unless record&.head_sequence&.positive? || chained_rows?
|
|
41
|
+
|
|
42
|
+
verify_chain(record)
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
private
|
|
46
|
+
|
|
47
|
+
attr_reader :entry_class, :keyring, :batch_size, :breaks
|
|
48
|
+
|
|
49
|
+
def verify_chain(record)
|
|
50
|
+
@breaks = []
|
|
51
|
+
walk = walk(record)
|
|
52
|
+
check_record(record, walk)
|
|
53
|
+
head_at_end = record && read_head
|
|
54
|
+
check_beyond_head(head_at_end || record)
|
|
55
|
+
check_unchained(record)
|
|
56
|
+
|
|
57
|
+
result(record, head_at_end, walk.last_sequence)
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
def walk(record)
|
|
61
|
+
Walk.new(entry_class: entry_class, keyring: keyring, batch_size: batch_size,
|
|
62
|
+
link: record&.genesis_digest, report: method(:add))
|
|
63
|
+
.call(record&.head_sequence)
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
# A seam as well as a read: the specs wrap it to land a write mid-walk.
|
|
67
|
+
def read_head
|
|
68
|
+
ChainRecord.uncached { ChainRecord.find_by(name: ChainRecord::LEDGER) }
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
def check_record(record, walk)
|
|
72
|
+
return add(:chain_record_missing, nil, RECORD_MISSING) if record.nil?
|
|
73
|
+
|
|
74
|
+
check_genesis(record)
|
|
75
|
+
head = record.head_sequence
|
|
76
|
+
last_digest = walk.last_sequence.zero? ? record.genesis_digest : walk.last_digest
|
|
77
|
+
return if walk.last_sequence == head && last_digest == record.head_digest
|
|
78
|
+
|
|
79
|
+
add(:head_mismatch, head,
|
|
80
|
+
"the head is at #{head}, the last chained row found is #{walk.last_sequence}")
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
def check_genesis(record)
|
|
84
|
+
key_id = record.genesis_key_id
|
|
85
|
+
secret = keyring.for(key_id)
|
|
86
|
+
if secret.nil?
|
|
87
|
+
return add(:unknown_key, 0, "the genesis is signed by key #{key_id}", nil, key_id)
|
|
88
|
+
end
|
|
89
|
+
return if Canonical.genesis_digest(secret, record) == record.genesis_digest
|
|
90
|
+
|
|
91
|
+
add(:genesis_altered, 0, "the genesis does not match its digest")
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
def check_beyond_head(record)
|
|
95
|
+
return if record.nil?
|
|
96
|
+
|
|
97
|
+
candidates = entry_class.where("chain_sequence > ?", record.head_sequence)
|
|
98
|
+
.order(:chain_sequence).pluck(:chain_sequence)
|
|
99
|
+
return if candidates.empty?
|
|
100
|
+
|
|
101
|
+
# Read the head once more, after the search. A row committed between the last read
|
|
102
|
+
# and the search moved the head with it, so the head now covers it; a row above
|
|
103
|
+
# this read cannot have been written by reeve.
|
|
104
|
+
head = read_head&.head_sequence || record.head_sequence
|
|
105
|
+
detail = "numbered past the head (#{Report.number(head)} at the end of the walk); " \
|
|
106
|
+
"reeve did not write it"
|
|
107
|
+
candidates.select { |sequence| sequence > head }
|
|
108
|
+
.each { |sequence| add(:beyond_head, sequence, detail) }
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
# Contract-3 rows with no digest, beyond the number that already existed when the
|
|
112
|
+
# chain began (research R5). Which ones are new is known only by clock, and the
|
|
113
|
+
# detail says so.
|
|
114
|
+
def check_unchained(record)
|
|
115
|
+
return if count_unchained <= (record ? record.pre_chain_unchained_count : 0)
|
|
116
|
+
|
|
117
|
+
scope = entry_class.where(UNCHAINED)
|
|
118
|
+
scope = scope.where("occurred_at > ?", record.genesis_at) if record
|
|
119
|
+
scope.order(:occurred_at).pluck(:invocation_id).each do |invocation_id|
|
|
120
|
+
add(:unchained, nil, UNCHAINED_ROW, invocation_id)
|
|
121
|
+
end
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
def add(kind, sequence, detail = nil, invocation_id = nil, key_id = nil)
|
|
125
|
+
breaks << Verification::Break.new(kind: kind, sequence: sequence, detail: detail,
|
|
126
|
+
invocation_id: invocation_id, key_id: key_id)
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
def result(record, head_at_end, last_sequence)
|
|
130
|
+
Verification.new(
|
|
131
|
+
status: Verification.status_for(breaks), breaks: breaks,
|
|
132
|
+
covered: covered(record, last_sequence),
|
|
133
|
+
head_at_start: record&.head_sequence, head_at_end: head_at_end&.head_sequence,
|
|
134
|
+
uncovered_now: count_uncovered, uncovered_at_genesis: record&.pre_chain_count,
|
|
135
|
+
unchained_now: count_unchained, unchained_at_genesis: record&.pre_chain_unchained_count
|
|
136
|
+
)
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
def covered(record, last_sequence)
|
|
140
|
+
last = record ? record.head_sequence : last_sequence
|
|
141
|
+
last.positive? ? 1..last : nil
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
# An answer given before any walk: no key to walk with, or no chain to walk.
|
|
145
|
+
def unwalked(status, reason = nil)
|
|
146
|
+
Verification.new(status: status, reason: reason,
|
|
147
|
+
uncovered_now: count_uncovered, unchained_now: count_unchained)
|
|
148
|
+
end
|
|
149
|
+
|
|
150
|
+
def chained_rows?
|
|
151
|
+
entry_class.where.not(chain_sequence: nil).exists?
|
|
152
|
+
end
|
|
153
|
+
|
|
154
|
+
def count_uncovered = (@count_uncovered ||= entry_class.where("contract_version < 3").count)
|
|
155
|
+
|
|
156
|
+
def count_unchained = (@count_unchained ||= entry_class.where(UNCHAINED).count)
|
|
157
|
+
end
|
|
158
|
+
end
|
|
159
|
+
end
|
|
160
|
+
end
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Reeve
|
|
4
|
+
module Audit
|
|
5
|
+
class Chain
|
|
6
|
+
# The verifier's row-by-row pass (research R6): every chained row up to a sequence, in
|
|
7
|
+
# order, a batch at a time.
|
|
8
|
+
#
|
|
9
|
+
# Each row's digest is recomputed against its predecessor's *recorded* digest, not a
|
|
10
|
+
# recomputed one, so one edited row is one break rather than a cascade to the end of
|
|
11
|
+
# the ledger. Gaps and repeated sequences are reported where they occur.
|
|
12
|
+
#
|
|
13
|
+
# Keyset pagination on (chain_sequence, id), never OFFSET, and never inside a
|
|
14
|
+
# transaction: memory stays flat whatever the ledger's size, and no writer waits.
|
|
15
|
+
class Walk
|
|
16
|
+
MISMATCH = "does not verify at its position — its content, its sequence number or " \
|
|
17
|
+
"its predecessor changed"
|
|
18
|
+
|
|
19
|
+
attr_reader :last_sequence, :last_digest
|
|
20
|
+
|
|
21
|
+
# `link` is the digest the first row must link to (the genesis), or nil when there
|
|
22
|
+
# is no chain record to take it from. `report` receives (kind, sequence, detail).
|
|
23
|
+
def initialize(entry_class:, keyring:, batch_size:, link:, report:)
|
|
24
|
+
@entry_class = entry_class
|
|
25
|
+
@keyring = keyring
|
|
26
|
+
@batch_size = batch_size
|
|
27
|
+
@report = report
|
|
28
|
+
@expected = 1
|
|
29
|
+
@last_sequence = 0
|
|
30
|
+
@link = link
|
|
31
|
+
@verify_first_link = !link.nil?
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
# Walks up to `limit` (the head read at the start), or every chained row if nil.
|
|
35
|
+
def call(limit)
|
|
36
|
+
each_chained(limit) { |entry| visit(entry) }
|
|
37
|
+
self
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
private
|
|
41
|
+
|
|
42
|
+
attr_reader :entry_class, :keyring, :batch_size
|
|
43
|
+
|
|
44
|
+
def visit(entry)
|
|
45
|
+
sequence = entry.chain_sequence
|
|
46
|
+
return revisit(entry) if sequence == @last_sequence
|
|
47
|
+
|
|
48
|
+
report_gap(@expected..(sequence - 1)) if sequence > @expected
|
|
49
|
+
verify(entry, @link)
|
|
50
|
+
advance(entry)
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def report_gap(range)
|
|
54
|
+
count = range.size
|
|
55
|
+
@report.call(:missing, range, "#{count} #{count == 1 ? 'entry' : 'entries'} not found")
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
# A second row claiming the sequence just seen: reported, and checked against the
|
|
59
|
+
# same predecessor the first one was.
|
|
60
|
+
def revisit(entry)
|
|
61
|
+
@report.call(:duplicate, entry.chain_sequence)
|
|
62
|
+
verify(entry, @previous_link)
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
def advance(entry)
|
|
66
|
+
@previous_link = @link
|
|
67
|
+
@link = entry.chain_digest
|
|
68
|
+
@last_sequence = entry.chain_sequence
|
|
69
|
+
@last_digest = entry.chain_digest
|
|
70
|
+
@expected = entry.chain_sequence + 1
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
def verify(entry, previous)
|
|
74
|
+
# With no chain record there is no genesis for the first row to link to. That is
|
|
75
|
+
# reported once, as chain_record_missing, not again here as a mismatch.
|
|
76
|
+
return if !@verify_first_link && entry.chain_sequence == 1
|
|
77
|
+
|
|
78
|
+
key_id = entry.chain_key_id
|
|
79
|
+
sequence = entry.chain_sequence
|
|
80
|
+
secret = keyring.for(key_id)
|
|
81
|
+
if secret.nil?
|
|
82
|
+
return @report.call(:unknown_key, sequence, "signed by key #{key_id}", nil, key_id)
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
expected = Canonical.entry_digest(secret, entry, previous_digest: previous)
|
|
86
|
+
@report.call(:digest_mismatch, sequence, MISMATCH) unless expected == entry.chain_digest
|
|
87
|
+
rescue ArgumentError => e
|
|
88
|
+
@report.call(:digest_mismatch, entry.chain_sequence, e.message)
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
def each_chained(limit, &block)
|
|
92
|
+
after = nil
|
|
93
|
+
loop do
|
|
94
|
+
batch = batch_after(after, limit)
|
|
95
|
+
batch.each(&block)
|
|
96
|
+
break if batch.size < batch_size
|
|
97
|
+
|
|
98
|
+
after = [batch.last.chain_sequence, batch.last.id]
|
|
99
|
+
end
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
def batch_after(after, limit)
|
|
103
|
+
entry_class.uncached { batch_scope(after, limit).to_a }
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
# Resumes after (sequence, id). The `>=` is what the index can use; the OR beneath it
|
|
107
|
+
# only breaks ties between rows sharing that sequence. Written as the OR alone, the
|
|
108
|
+
# condition is the same set of rows, but PostgreSQL applied it as a filter and scanned
|
|
109
|
+
# every batch from the first row: the walk was quadratic, and a million rows took 331 s
|
|
110
|
+
# against SC-004's 300. SQLite derived the bound on its own, which hid it there.
|
|
111
|
+
def batch_scope(after, limit)
|
|
112
|
+
scope = entry_class.where.not(chain_sequence: nil)
|
|
113
|
+
scope = scope.where("chain_sequence <= ?", limit) if limit
|
|
114
|
+
if after
|
|
115
|
+
scope = scope.where("chain_sequence >= ?", after[0])
|
|
116
|
+
.where("chain_sequence > ? OR id > ?", after[0], after[1])
|
|
117
|
+
end
|
|
118
|
+
scope.order(:chain_sequence, :id).limit(batch_size)
|
|
119
|
+
end
|
|
120
|
+
end
|
|
121
|
+
end
|
|
122
|
+
end
|
|
123
|
+
end
|
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Reeve
|
|
4
|
+
module Audit
|
|
5
|
+
# The ledger's chain (specs/002): every row reeve writes is linked to the one before it
|
|
6
|
+
# by a keyed digest, so a row changed, removed or reordered outside reeve is reported by
|
|
7
|
+
# verification.
|
|
8
|
+
#
|
|
9
|
+
# Reeve::Audit::Chain.new(entry_class: Reeve::Audit::Entry).append!(row) # write
|
|
10
|
+
# Reeve::Audit::Chain.verify # check
|
|
11
|
+
#
|
|
12
|
+
# `append!` is the only door a chained row comes through. Recorder and IsolatedRecorder
|
|
13
|
+
# use it; a host with its own recorder uses it too (FR-007), or its rows are reported as
|
|
14
|
+
# unchained.
|
|
15
|
+
#
|
|
16
|
+
# == How a write is serialized (research R3)
|
|
17
|
+
#
|
|
18
|
+
# UPDATE the chain record's head_sequence by one -- the lock, taken first
|
|
19
|
+
# SELECT the new head_sequence and the old head_digest
|
|
20
|
+
# INSERT the row with its sequence, key id and digest
|
|
21
|
+
# UPDATE the head_digest to the row's digest
|
|
22
|
+
#
|
|
23
|
+
# all inside one savepoint on the entry class's own connection. The first statement is
|
|
24
|
+
# a write, so on PostgreSQL and MySQL it takes the row lock and on SQLite the database
|
|
25
|
+
# write lock: every writer queues at the same point with the same SQL. Reading first
|
|
26
|
+
# would be refused outright by SQLite (a shared lock cannot be upgraded while another
|
|
27
|
+
# writer waits), and SELECT ... FOR UPDATE is ignored there.
|
|
28
|
+
#
|
|
29
|
+
# Because the row and the head move commit together, a rolled-back write consumes no
|
|
30
|
+
# sequence — and a committed row can never sit beyond a committed head, which is what
|
|
31
|
+
# lets the verifier call such a row forged.
|
|
32
|
+
#
|
|
33
|
+
# == The cost, stated
|
|
34
|
+
#
|
|
35
|
+
# Under the default Recorder inside a transaction the host opened, the chain record
|
|
36
|
+
# stays locked until that transaction ends, so every other guarded call waits behind it.
|
|
37
|
+
# IsolatedRecorder holds it for one insert.
|
|
38
|
+
class Chain
|
|
39
|
+
TABLE = "reeve_audit_chains"
|
|
40
|
+
|
|
41
|
+
COLUMNS = %w[chain_sequence chain_key_id chain_digest].freeze
|
|
42
|
+
|
|
43
|
+
# A contract-3 row outside the chain: the walk only reads rows with a sequence, so a
|
|
44
|
+
# row without one is counted here or nowhere. Written once and used by both the
|
|
45
|
+
# genesis (the baseline it records) and the verifier (the count it compares), because
|
|
46
|
+
# the baseline means something only while the two count the same rows. It keys on the
|
|
47
|
+
# missing sequence, not the missing digest: a forged row with a digest and no sequence
|
|
48
|
+
# once sat outside every count.
|
|
49
|
+
UNCHAINED = "contract_version >= 3 AND chain_sequence IS NULL"
|
|
50
|
+
|
|
51
|
+
# Verifies the whole ledger. With `keys:`, verifies with those keys alone — how an
|
|
52
|
+
# auditor checks a ledger with their own copy of the key.
|
|
53
|
+
def self.verify(keys: nil, entry_class: Entry, batch_size: Verifier::BATCH_SIZE)
|
|
54
|
+
keyring = keys ? Keyring.new(signing: nil, retired: keys) : Keyring.from(Reeve.config)
|
|
55
|
+
Verifier.new(entry_class: entry_class, keyring: keyring, batch_size: batch_size).call
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
def initialize(entry_class: Entry, keyring: nil)
|
|
59
|
+
@entry_class = entry_class
|
|
60
|
+
@keyring = keyring
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
# Writes `row` as the next link and returns the entry. The row carries every column
|
|
64
|
+
# but the three chain ones; it is normalized here (research R2).
|
|
65
|
+
#
|
|
66
|
+
# Raises LedgerKeyError with no signing key, and whatever the database raises if the
|
|
67
|
+
# write fails. A replayed invocation_id returns the existing entry and consumes
|
|
68
|
+
# nothing. What a failure means for the caller — fail the call or warn — is the
|
|
69
|
+
# caller's decision, not this method's.
|
|
70
|
+
def append!(row)
|
|
71
|
+
require_columns!
|
|
72
|
+
signer = keyring
|
|
73
|
+
raise LedgerKeyError unless signer.signing?
|
|
74
|
+
|
|
75
|
+
normalized = normalizer.call(row)
|
|
76
|
+
with_connection do |connection|
|
|
77
|
+
insert_once(normalized) do
|
|
78
|
+
head = Head.new(connection: connection, entry_class: entry_class)
|
|
79
|
+
head.start!(signer)
|
|
80
|
+
entry_class.transaction(requires_new: true) { link!(head, normalized, signer) }
|
|
81
|
+
end
|
|
82
|
+
end
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
# Writes `row` at its contract version with no chain columns, around the chain: no lock,
|
|
86
|
+
# no sequence. For a caller that has decided to keep running without a key (Recorder
|
|
87
|
+
# under audit_failure_mode :warn). Verification reports such a row as unchained.
|
|
88
|
+
def append_unchained!(row)
|
|
89
|
+
require_columns!
|
|
90
|
+
normalized = normalizer.call(row)
|
|
91
|
+
with_connection do
|
|
92
|
+
insert_once(normalized) do
|
|
93
|
+
entry_class.transaction(requires_new: true) { entry_class.create!(normalized) }
|
|
94
|
+
end
|
|
95
|
+
end
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
private
|
|
99
|
+
|
|
100
|
+
attr_reader :entry_class
|
|
101
|
+
|
|
102
|
+
def keyring
|
|
103
|
+
@keyring || Keyring.from(Reeve.config)
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
def normalizer
|
|
107
|
+
@normalizer ||= Chain::Normalizer.new(entry_class)
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
# The digest is computed over the model instance, not the hash: ActiveRecord fills in
|
|
111
|
+
# column defaults (`truncated: false`) and casts on assignment (an Integer principal_id
|
|
112
|
+
# becomes a String), and the instance is exactly what the database will be given.
|
|
113
|
+
# Digesting the hash a caller passed would sign values that are never stored.
|
|
114
|
+
def link!(head, normalized, signer)
|
|
115
|
+
sequence, previous = head.advance!
|
|
116
|
+
entry = entry_class.new(normalized.merge(chain_sequence: sequence,
|
|
117
|
+
chain_key_id: signer.signing_id))
|
|
118
|
+
entry.chain_digest = Canonical.entry_digest(signer.signing_secret, entry,
|
|
119
|
+
previous_digest: previous)
|
|
120
|
+
entry.save!
|
|
121
|
+
head.point_at!(entry.chain_digest)
|
|
122
|
+
entry
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
# A replayed invocation_id returns the row already written, whether the pre-check sees
|
|
126
|
+
# it or the insert collides with it.
|
|
127
|
+
def insert_once(normalized)
|
|
128
|
+
existing = replayed(normalized)
|
|
129
|
+
return existing if existing
|
|
130
|
+
|
|
131
|
+
yield
|
|
132
|
+
rescue ActiveRecord::RecordNotUnique, ActiveRecord::RecordInvalid => e
|
|
133
|
+
replayed(normalized) || raise(e)
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
# A table one contract behind the gem fails here, naming the fix, rather than as an
|
|
137
|
+
# "unknown attribute" or "no such column" from deep inside the write.
|
|
138
|
+
def require_columns!
|
|
139
|
+
missing = COLUMNS - entry_class.column_names
|
|
140
|
+
return if missing.empty?
|
|
141
|
+
|
|
142
|
+
raise ConfigurationError,
|
|
143
|
+
"#{entry_class.table_name} is missing #{missing.join(', ')}: the ledger is at an " \
|
|
144
|
+
"older audit-entry contract than this reeve writes (#{CONTRACT_VERSION}). Run " \
|
|
145
|
+
"`bin/rails generate reeve:upgrade` and `bin/rails db:migrate`."
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
def replayed(normalized)
|
|
149
|
+
id = normalized[:invocation_id]
|
|
150
|
+
id && entry_class.find_by(invocation_id: id)
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
# Checked out for one append and returned, never leased to the thread: from Rails 7.2
|
|
154
|
+
# `entry_class.connection` pins a connection to the thread for good, which would undo
|
|
155
|
+
# IsolatedRecorder's promise to hold none between writes. ConnectionPool#with_connection
|
|
156
|
+
# exists on every Rails this gem supports, and reuses a connection the thread already
|
|
157
|
+
# holds — so the transaction and the INSERT run on the same one.
|
|
158
|
+
#
|
|
159
|
+
# The connection is passed down as an argument, never kept on the instance. A Chain
|
|
160
|
+
# holds no per-call state, so one instance can be shared by every thread that writes:
|
|
161
|
+
# an earlier version stored it in an instance variable, and two threads appending
|
|
162
|
+
# through one Chain drove a single PostgreSQL connection at once.
|
|
163
|
+
def with_connection(&block)
|
|
164
|
+
entry_class.connection_pool.with_connection(&block)
|
|
165
|
+
end
|
|
166
|
+
end
|
|
167
|
+
end
|
|
168
|
+
end
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Reeve
|
|
4
|
+
module Audit
|
|
5
|
+
# The ledger's chain record: one row holding the genesis and the head (specs/002
|
|
6
|
+
# data-model "reeve_audit_chains").
|
|
7
|
+
#
|
|
8
|
+
# The genesis is link 0 — when the chain started, how many rows preceded it, and the
|
|
9
|
+
# digest the first chained row links to. It is written once, with the first chained
|
|
10
|
+
# row, and the model refuses to change it afterwards. The head is the latest sequence
|
|
11
|
+
# and digest; Chain advances it with SQL of its own, inside the same transaction as
|
|
12
|
+
# the row it points at, so it is never updated through this model.
|
|
13
|
+
#
|
|
14
|
+
# As with Entry, this protects against the library, not against raw SQL. A role that
|
|
15
|
+
# can UPDATE this table can rewind the head; that is one of the stated limits of the
|
|
16
|
+
# chain, and the verifier's job is to report what it can.
|
|
17
|
+
class ChainRecord < ActiveRecord::Base
|
|
18
|
+
self.table_name = "reeve_audit_chains"
|
|
19
|
+
|
|
20
|
+
# Always this. A name rather than a primary key, because ids change on restore.
|
|
21
|
+
LEDGER = "ledger"
|
|
22
|
+
|
|
23
|
+
GENESIS_COLUMNS = %w[
|
|
24
|
+
name genesis_at pre_chain_count pre_chain_unchained_count genesis_key_id genesis_digest
|
|
25
|
+
].freeze
|
|
26
|
+
|
|
27
|
+
attr_readonly(*GENESIS_COLUMNS)
|
|
28
|
+
|
|
29
|
+
# attr_readonly ignores the change silently on Rails 7.0 and raises from 7.1; this
|
|
30
|
+
# makes every version refuse it out loud.
|
|
31
|
+
before_update do
|
|
32
|
+
changed = changed_attribute_names_to_save & GENESIS_COLUMNS
|
|
33
|
+
unless changed.empty?
|
|
34
|
+
raise ActiveRecord::ReadOnlyRecord,
|
|
35
|
+
"the genesis of the ledger chain is written once; refused to change " \
|
|
36
|
+
"#{changed.join(', ')}"
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
before_destroy { throw :abort }
|
|
41
|
+
|
|
42
|
+
def self.ledger
|
|
43
|
+
find_by(name: LEDGER)
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
# The record is written through the ledger's own connection (Chain::Head) and read
|
|
47
|
+
# through this model's. A host that moves the ledger to a database of its own and
|
|
48
|
+
# leaves this model behind would have verification read the wrong record and report
|
|
49
|
+
# tampering on every run; refusing, with the fix, is the honest answer.
|
|
50
|
+
def self.require_same_database!(entry_class)
|
|
51
|
+
ledger_database = entry_class.connection_db_config.configuration_hash
|
|
52
|
+
return if connection_db_config.configuration_hash == ledger_database
|
|
53
|
+
|
|
54
|
+
raise ConfigurationError,
|
|
55
|
+
"Reeve::Audit::ChainRecord is connected to a different database than " \
|
|
56
|
+
"#{entry_class.name}. Connect ChainRecord the same way as the ledger (for " \
|
|
57
|
+
"example with the same connects_to) so the chain is read where it is written."
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
end
|
|
@@ -34,6 +34,11 @@ module Reeve
|
|
|
34
34
|
# and it also survives a rollback of an invocation the host *meant* to undo entirely.
|
|
35
35
|
# A ledger of what was attempted is the intended reading of Constitution II, but it is
|
|
36
36
|
# a choice, and it should be made rather than inherited.
|
|
37
|
+
#
|
|
38
|
+
# It is also the answer to the chain's one real cost (specs/002). Every chained write
|
|
39
|
+
# locks the ledger's chain record; under Recorder inside a host transaction that lock is
|
|
40
|
+
# held until the host commits, so every other guarded call waits behind it. Here the
|
|
41
|
+
# chain record is written on this connection too, and the lock is held for one insert.
|
|
37
42
|
class IsolatedRecorder < Recorder
|
|
38
43
|
UNSUPPORTED_ADAPTERS = %w[sqlite sqlite3].freeze
|
|
39
44
|
|
data/lib/reeve/audit/recorder.rb
CHANGED
|
@@ -10,9 +10,10 @@ module Reeve
|
|
|
10
10
|
# ActiveJob. An asynchronous ledger cannot be relied on and would make FR-012
|
|
11
11
|
# unenforceable.
|
|
12
12
|
#
|
|
13
|
-
#
|
|
14
|
-
#
|
|
15
|
-
#
|
|
13
|
+
# Every row goes through Chain#append! (specs/002), which links it to the row before it
|
|
14
|
+
# and runs the insert in `requires_new: true` — what makes the trace survive a tool body
|
|
15
|
+
# that opens a transaction and rolls it back (R5), the case this was built for and the
|
|
16
|
+
# one it does solve.
|
|
16
17
|
#
|
|
17
18
|
# What it does **not** do, corrected after review: `requires_new` is a SAVEPOINT, not
|
|
18
19
|
# an independent transaction. If the host has already opened a transaction *around*
|
|
@@ -54,20 +55,28 @@ module Reeve
|
|
|
54
55
|
|
|
55
56
|
# Returns the entry. Raises if the row could not be written, so the envelope can
|
|
56
57
|
# fail the invocation.
|
|
58
|
+
#
|
|
59
|
+
# A replayed invocation is the same invocation: one row per invocation_id means a
|
|
60
|
+
# retry returns the existing row and never writes a second (contracts/audit-entry.md).
|
|
61
|
+
# Chain#append! owns that, because it must also consume no sequence number.
|
|
62
|
+
#
|
|
63
|
+
# With no ledger key (specs/002 FR-010): under :fail, Chain raises LedgerKeyError and
|
|
64
|
+
# the envelope fails the call like any other ledger failure. Under :warn the row is
|
|
65
|
+
# still written, at contract 3 and unchained, and the log says so every time —
|
|
66
|
+
# :warn keeps calls running, it does not drop traces, and verification reports the
|
|
67
|
+
# row as unchained rather than letting it pass unseen.
|
|
57
68
|
def record(attributes)
|
|
58
69
|
row = row_for(attributes)
|
|
59
70
|
warn_about_enclosing_transaction(row[:invocation_id])
|
|
60
71
|
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
rescue ActiveRecord::RecordNotUnique, ActiveRecord::RecordInvalid => e
|
|
65
|
-
# A replayed invocation is the same invocation. One row per invocation_id means
|
|
66
|
-
# a retry is a no-op insert, never a second row (contracts/audit-entry.md).
|
|
67
|
-
existing = entry_class.find_by(invocation_id: row[:invocation_id])
|
|
68
|
-
raise e if existing.nil?
|
|
72
|
+
keyring = Chain::Keyring.from(config)
|
|
73
|
+
chain = Chain.new(entry_class: entry_class, keyring: keyring)
|
|
74
|
+
return chain.append!(row) if keyring.signing? || config.audit_failure_mode == :fail
|
|
69
75
|
|
|
70
|
-
|
|
76
|
+
warn("reeve: invocation #{row[:invocation_id]} was recorded unchained: no ledger key is " \
|
|
77
|
+
"configured and audit_failure_mode is :warn. Verification will report the row as " \
|
|
78
|
+
"unchained. Set config.ledger_key to chain the ledger.")
|
|
79
|
+
chain.append_unchained!(row)
|
|
71
80
|
end
|
|
72
81
|
|
|
73
82
|
private
|
|
@@ -78,6 +87,11 @@ module Reeve
|
|
|
78
87
|
@config || Reeve.config
|
|
79
88
|
end
|
|
80
89
|
|
|
90
|
+
def warn(message)
|
|
91
|
+
logger = config.logger
|
|
92
|
+
logger ? logger.warn(message) : Kernel.warn(message)
|
|
93
|
+
end
|
|
94
|
+
|
|
81
95
|
def row_for(attributes)
|
|
82
96
|
ids, count, truncated = identifiers(attributes)
|
|
83
97
|
|
|
@@ -163,13 +177,12 @@ module Reeve
|
|
|
163
177
|
def warn_about_enclosing_transaction(invocation_id)
|
|
164
178
|
return unless enclosing_transaction?
|
|
165
179
|
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
logger ? logger.warn(message) : Kernel.warn(message)
|
|
180
|
+
warn("reeve: invocation #{invocation_id} was recorded inside a transaction " \
|
|
181
|
+
"the host opened around it, so the ledger row will be rolled back with " \
|
|
182
|
+
"it, and the ledger chain stays locked until it ends, so every other " \
|
|
183
|
+
"guarded call waits behind it. Set Reeve.config.audit_recorder = " \
|
|
184
|
+
"Reeve::Audit::IsolatedRecorder to write on a connection the transaction " \
|
|
185
|
+
"cannot reach (needs a database with concurrent writers; not SQLite).")
|
|
173
186
|
end
|
|
174
187
|
|
|
175
188
|
def enclosing_transaction?
|
data/lib/reeve/audit.rb
CHANGED
|
@@ -44,9 +44,14 @@ module Reeve
|
|
|
44
44
|
# documents the `GRANT INSERT, SELECT` that enforces the rest where it can actually be
|
|
45
45
|
# enforced, and the gem makes no stronger claim than that.
|
|
46
46
|
# * No retention, rotation or archival yet — the table is host-owned and the host's
|
|
47
|
-
# existing policies apply.
|
|
48
|
-
# *
|
|
49
|
-
#
|
|
47
|
+
# existing policies apply. Deleting rows breaks the chain, by design.
|
|
48
|
+
# * Tamper-evidence, not tamper-proofing. Every row is chained to the one before it
|
|
49
|
+
# (Reeve::Audit::Chain, contract 3), so a change made outside the library is reported
|
|
50
|
+
# by verification — after the fact; it prevents nothing. It does not detect a change
|
|
51
|
+
# made by someone holding the ledger key, the newest rows deleted by a role that can
|
|
52
|
+
# also rewind the chain record's head, the whole ledger deleted, or anything about
|
|
53
|
+
# rows written before contract 3. specs/001-guardrails-core/contracts/audit-entry.md,
|
|
54
|
+
# guarantee 9, is the authoritative statement.
|
|
50
55
|
#
|
|
51
56
|
# ("yet" rather than "in v1": the gem version and the audit-entry contract version are
|
|
52
57
|
# different numbers that move independently, and writing v1 for one of them read as the
|
|
@@ -57,11 +62,16 @@ module Reeve
|
|
|
57
62
|
# column is a MINOR change and leaves this alone; removing or renaming a column, or
|
|
58
63
|
# changing what a value means, is MAJOR and bumps it.
|
|
59
64
|
#
|
|
65
|
+
# 3 — entries are chained (specs/002). The three chain columns are nullable, so the
|
|
66
|
+
# shape change is additive, but what a NULL digest means changed: on a contract-3 row
|
|
67
|
+
# it is a row written unchained, which verification reports, where on an older row it
|
|
68
|
+
# only means the row predates chaining.
|
|
69
|
+
#
|
|
60
70
|
# 2 — `metadata` carries the transport detail the caller passed. Through version 1 it
|
|
61
71
|
# was written NULL on every row regardless of what was passed, so anything mapping
|
|
62
72
|
# version 1 rows could reasonably have read the column as "always empty". The shape
|
|
63
73
|
# did not change; what a value means did.
|
|
64
|
-
CONTRACT_VERSION =
|
|
74
|
+
CONTRACT_VERSION = 3
|
|
65
75
|
|
|
66
76
|
TABLE_NAME = "reeve_audit_entries"
|
|
67
77
|
|
|
@@ -94,5 +104,15 @@ require_relative "audit/entry"
|
|
|
94
104
|
require_relative "audit/recorder"
|
|
95
105
|
require_relative "audit/isolated_recorder"
|
|
96
106
|
require_relative "audit/query"
|
|
107
|
+
require_relative "audit/chain/keyring"
|
|
108
|
+
require_relative "audit/chain/normalizer"
|
|
109
|
+
require_relative "audit/chain/canonical"
|
|
110
|
+
require_relative "audit/chain_record"
|
|
111
|
+
require_relative "audit/chain/verification"
|
|
112
|
+
require_relative "audit/chain/report"
|
|
113
|
+
require_relative "audit/chain/head"
|
|
114
|
+
require_relative "audit/chain/walk"
|
|
115
|
+
require_relative "audit/chain/verifier"
|
|
116
|
+
require_relative "audit/chain"
|
|
97
117
|
|
|
98
118
|
Reeve::Audit.install!
|