reeve 0.7.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +117 -0
- data/README.md +236 -9
- data/lib/generators/reeve/install/templates/create_audit_entries.rb.tt +29 -4
- data/lib/generators/reeve/install/templates/initializer.rb.tt +10 -0
- data/lib/generators/reeve/upgrade/templates/add_ledger_retention_to_reeve_audit_entries.rb.tt +37 -0
- data/lib/generators/reeve/upgrade/upgrade_generator.rb +10 -1
- data/lib/generators/reeve/upgrade/upgrade_notes.rb +15 -4
- data/lib/reeve/audit/chain/anchoring.rb +91 -0
- data/lib/reeve/audit/chain/canonical.rb +17 -1
- data/lib/reeve/audit/chain/checkpoint.rb +228 -0
- data/lib/reeve/audit/chain/checkpoint_check.rb +156 -0
- data/lib/reeve/audit/chain/entrypoints.rb +29 -0
- data/lib/reeve/audit/chain/epochs.rb +10 -4
- data/lib/reeve/audit/chain/head.rb +26 -1
- data/lib/reeve/audit/chain/keyring.rb +6 -0
- data/lib/reeve/audit/chain/report.rb +188 -31
- data/lib/reeve/audit/chain/retention/archive.rb +98 -0
- data/lib/reeve/audit/chain/retention/archive_check.rb +179 -0
- data/lib/reeve/audit/chain/retention/check.rb +94 -0
- data/lib/reeve/audit/chain/retention/connection.rb +65 -0
- data/lib/reeve/audit/chain/retention/digest.rb +29 -0
- data/lib/reeve/audit/chain/retention/plan.rb +100 -0
- data/lib/reeve/audit/chain/retention/preflight.rb +48 -0
- data/lib/reeve/audit/chain/retention/purge.rb +155 -0
- data/lib/reeve/audit/chain/retention/purge_result.rb +134 -0
- data/lib/reeve/audit/chain/retention/records.rb +23 -0
- data/lib/reeve/audit/chain/verification.rb +31 -9
- data/lib/reeve/audit/chain/verifier.rb +64 -19
- data/lib/reeve/audit/chain/walk.rb +9 -3
- data/lib/reeve/audit/chain.rb +12 -3
- data/lib/reeve/audit/chain_record.rb +3 -2
- data/lib/reeve/audit/file_anchor.rb +49 -0
- data/lib/reeve/audit/retention_record.rb +24 -0
- data/lib/reeve/audit.rb +16 -0
- data/lib/reeve/configuration.rb +25 -0
- data/lib/reeve/errors.rb +15 -0
- data/lib/reeve/railtie.rb +2 -2
- data/lib/reeve/tasks/descriptions.rb +22 -0
- data/lib/reeve/tasks/purge_options.rb +54 -0
- data/lib/reeve/tasks.rb +82 -14
- data/lib/reeve/testing/checks/ledger_chain.rb +56 -2
- data/lib/reeve/testing/ledger.rb +19 -3
- data/lib/reeve/version.rb +1 -1
- metadata +21 -2
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require "time"
|
|
4
|
+
|
|
3
5
|
module Reeve
|
|
4
6
|
module Audit
|
|
5
7
|
class Chain
|
|
@@ -15,6 +17,7 @@ module Reeve
|
|
|
15
17
|
LABEL_WIDTH = 14
|
|
16
18
|
KIND_WIDTH = 17
|
|
17
19
|
PREFIX = "reeve ledger:"
|
|
20
|
+
NOTHING_AFTER_PURGE = "no entries after the purge to verify"
|
|
18
21
|
|
|
19
22
|
def self.number(value)
|
|
20
23
|
digits = value.abs.to_s.reverse.scan(/\d{1,3}/).join(",").reverse
|
|
@@ -46,48 +49,50 @@ module Reeve
|
|
|
46
49
|
end
|
|
47
50
|
|
|
48
51
|
def intact
|
|
49
|
-
|
|
50
|
-
"
|
|
52
|
+
if @v.walked_from
|
|
53
|
+
return "#{PREFIX} INTACT — #{anchor_text.walked_from}; #{uncovered}; #{anchor_text}"
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
walked = @v.covered ? "sequences #{range(@v.covered)} verified" : NOTHING_AFTER_PURGE
|
|
57
|
+
"#{PREFIX} INTACT — #{walked} (head at start: #{number(@v.head_at_start)}); " \
|
|
58
|
+
"#{uncovered}; #{anchor_text}"
|
|
51
59
|
end
|
|
52
60
|
|
|
53
61
|
def broken
|
|
54
62
|
count = @v.breaks.size
|
|
63
|
+
breaks = "#{number(count)} break#{'s' unless count == 1}"
|
|
64
|
+
if anchor_text.emptied?
|
|
65
|
+
return "#{PREFIX} BROKEN — #{breaks} in the ledger; #{anchor_text.emptied}"
|
|
66
|
+
end
|
|
67
|
+
|
|
55
68
|
where = @v.covered ? "in sequences #{range(@v.covered)}" : "in the ledger"
|
|
56
|
-
"#{PREFIX} BROKEN — #{
|
|
69
|
+
"#{PREFIX} BROKEN — #{breaks} #{where}; #{uncovered}; #{anchor_text}"
|
|
57
70
|
end
|
|
58
71
|
|
|
72
|
+
def anchor_text = (@anchor_text ||= AnchorText.new(@v))
|
|
73
|
+
|
|
59
74
|
def no_chain
|
|
60
75
|
"#{PREFIX} NO CHAIN — #{number(@v.uncovered_now + @v.unchained_now)} entries, " \
|
|
61
76
|
"none chained; nothing was verified"
|
|
62
77
|
end
|
|
63
78
|
|
|
64
|
-
def no_key
|
|
65
|
-
"#{PREFIX} UNVERIFIABLE — no ledger key configured; nothing was verified"
|
|
66
|
-
end
|
|
79
|
+
def no_key = "#{PREFIX} UNVERIFIABLE — no ledger key configured; nothing was verified"
|
|
67
80
|
|
|
81
|
+
# The purge clause leads here too (FR-011), but the uncovered count stays as 0.6.0 wrote it.
|
|
68
82
|
def unverifiable
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
checked = @v.covered ? @v.covered.size - rows : 0
|
|
72
|
-
"#{PREFIX} UNVERIFIABLE — #{number(rows)} entries signed by #{missing_keys(unknown)} " \
|
|
73
|
-
"not configured; the other #{number(checked)} in #{range(@v.covered)} show no " \
|
|
74
|
-
"breaks, which is not a pass; #{number(@v.uncovered_now)} earlier entries not covered"
|
|
75
|
-
end
|
|
76
|
-
|
|
77
|
-
# The genesis is position 0 and is not an entry.
|
|
78
|
-
def entry_position?(sequence)
|
|
79
|
-
sequence.is_a?(Integer) && sequence.positive?
|
|
80
|
-
end
|
|
81
|
-
|
|
82
|
-
def missing_keys(unknown)
|
|
83
|
-
keys = unknown.map(&:key_id).compact.uniq
|
|
84
|
-
keys.one? ? "key #{keys.first}, which is" : "keys #{keys.join(', ')}, which are"
|
|
83
|
+
"#{PREFIX} UNVERIFIABLE — #{UnverifiableText.new(@v)}; #{RetentionText.new(@v).clause}" \
|
|
84
|
+
"#{number(@v.uncovered_now)} earlier entries not covered; #{anchor_text}"
|
|
85
85
|
end
|
|
86
86
|
|
|
87
|
+
# After a purge, what it removed comes first (specs/005 FR-011): no line says what was
|
|
88
|
+
# verified without saying what was purged before it.
|
|
87
89
|
def uncovered
|
|
88
|
-
|
|
90
|
+
retention = RetentionText.new(@v)
|
|
91
|
+
parts = [retention.clause, "#{number(@v.uncovered_now)} earlier entries not covered"]
|
|
89
92
|
at_genesis = @v.uncovered_at_genesis
|
|
90
|
-
|
|
93
|
+
if at_genesis
|
|
94
|
+
parts << " (#{number(at_genesis)} when the chain started#{retention.pre_chain})"
|
|
95
|
+
end
|
|
91
96
|
unless @v.unchained_at_genesis.to_i.zero?
|
|
92
97
|
parts << "; #{number(@v.unchained_at_genesis)} unchained entries from before the " \
|
|
93
98
|
"chain started, not covered"
|
|
@@ -95,9 +100,7 @@ module Reeve
|
|
|
95
100
|
parts.join
|
|
96
101
|
end
|
|
97
102
|
|
|
98
|
-
def totals
|
|
99
|
-
" totals: #{@v.totals.map { |kind, count| "#{kind} #{count}" }.join(', ')}"
|
|
100
|
-
end
|
|
103
|
+
def totals = " totals: #{@v.totals.map { |kind, count| "#{kind} #{count}" }.join(', ')}"
|
|
101
104
|
|
|
102
105
|
def more(listed, limit)
|
|
103
106
|
remaining = listed.size - limit
|
|
@@ -106,15 +109,24 @@ module Reeve
|
|
|
106
109
|
" … and #{number(remaining)} more (the totals above are complete)"
|
|
107
110
|
end
|
|
108
111
|
|
|
112
|
+
# One space at least between columns, so a label or kind longer than its column
|
|
113
|
+
# (`checkpoint #48,000`, `checkpoint_not_reached`) never runs into the next.
|
|
109
114
|
def line_for(found)
|
|
110
|
-
|
|
111
|
-
|
|
115
|
+
label = label(found).ljust(LABEL_WIDTH - 1)
|
|
116
|
+
kind = found.kind.to_s.ljust(KIND_WIDTH - 1)
|
|
117
|
+
" #{label} #{kind} #{found.detail}".rstrip
|
|
112
118
|
end
|
|
113
119
|
|
|
114
120
|
def label(found)
|
|
115
|
-
sequence = found.sequence
|
|
116
121
|
return "invocation #{found.invocation_id}" if found.invocation_id
|
|
117
|
-
return "
|
|
122
|
+
return "checkpoint ##{number(found.checkpoint)}" if found.checkpoint
|
|
123
|
+
return "retention #{found.retention}" if found.retention
|
|
124
|
+
return "epoch #{found.epoch}" if found.sequence.nil? && found.epoch
|
|
125
|
+
|
|
126
|
+
position(found.sequence)
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
def position(sequence)
|
|
118
130
|
return "—" if sequence.nil?
|
|
119
131
|
return "##{range(sequence)}" if sequence.is_a?(Range)
|
|
120
132
|
|
|
@@ -132,6 +144,151 @@ module Reeve
|
|
|
132
144
|
end
|
|
133
145
|
end
|
|
134
146
|
|
|
147
|
+
# What an unverifiable run could not check, and what it did (the UNVERIFIABLE line). Rows
|
|
148
|
+
# signed by a key that is not configured are counted; a checkpoint whose key is missing is
|
|
149
|
+
# named apart from them and is not one (specs/004).
|
|
150
|
+
class UnverifiableText
|
|
151
|
+
def initialize(verification)
|
|
152
|
+
@v = verification
|
|
153
|
+
unknown = verification.breaks.select { |found| found.kind == :unknown_key }
|
|
154
|
+
@checkpoints, unknown = unknown.partition(&:checkpoint)
|
|
155
|
+
@records, @signed = unknown.partition(&:retention)
|
|
156
|
+
end
|
|
157
|
+
|
|
158
|
+
def to_s
|
|
159
|
+
rows = @signed.count { |found| entry?(found.sequence) }
|
|
160
|
+
checked = @v.covered ? @v.covered.size - rows : 0
|
|
161
|
+
"#{unchecked(rows)}; the other #{Report.number(checked)} in #{range} show no breaks, " \
|
|
162
|
+
"which is not a pass"
|
|
163
|
+
end
|
|
164
|
+
|
|
165
|
+
private
|
|
166
|
+
|
|
167
|
+
def unchecked(rows)
|
|
168
|
+
parts = []
|
|
169
|
+
parts << "#{Report.number(rows)} entries signed by #{missing(@signed)}" if @signed.any?
|
|
170
|
+
parts << other("checkpoint", @checkpoints) if @checkpoints.any?
|
|
171
|
+
parts << other("retention record", @records) if @records.any?
|
|
172
|
+
parts.join("; ")
|
|
173
|
+
end
|
|
174
|
+
|
|
175
|
+
# Not entries, so named apart and not counted as rows.
|
|
176
|
+
def other(noun, found)
|
|
177
|
+
count = found.size
|
|
178
|
+
"#{Report.number(count)} #{noun}#{'s' unless count == 1} signed by #{missing(found)}"
|
|
179
|
+
end
|
|
180
|
+
|
|
181
|
+
# The genesis is position 0 and is not an entry.
|
|
182
|
+
def entry?(sequence)
|
|
183
|
+
sequence.is_a?(Integer) && sequence.positive?
|
|
184
|
+
end
|
|
185
|
+
|
|
186
|
+
def missing(unknown)
|
|
187
|
+
keys = unknown.map(&:key_id).compact.uniq
|
|
188
|
+
named = keys.one? ? "key #{keys.first}, which is" : "keys #{keys.join(', ')}, which are"
|
|
189
|
+
"#{named} not configured"
|
|
190
|
+
end
|
|
191
|
+
|
|
192
|
+
def range
|
|
193
|
+
return "—" if @v.covered.nil?
|
|
194
|
+
|
|
195
|
+
"#{Report.number(@v.covered.first)}–#{Report.number(@v.covered.last)}"
|
|
196
|
+
end
|
|
197
|
+
end
|
|
198
|
+
|
|
199
|
+
# What the ledger's purges removed, in words (specs/005 FR-011): the clause that leads the
|
|
200
|
+
# coverage on every walked summary line, and the pre-chain rows a purge removed.
|
|
201
|
+
class RetentionText
|
|
202
|
+
def initialize(verification)
|
|
203
|
+
@retention = verification.retention
|
|
204
|
+
@awaiting = verification.awaiting_purge.to_i
|
|
205
|
+
end
|
|
206
|
+
|
|
207
|
+
def clause
|
|
208
|
+
boundary = @retention&.boundary
|
|
209
|
+
return "" if boundary.nil?
|
|
210
|
+
|
|
211
|
+
text = "1–#{Report.number(boundary.through_sequence)} purged (retention record " \
|
|
212
|
+
"#{boundary.purge_number}, #{boundary.purged_at.utc.strftime('%Y-%m-%d')}); "
|
|
213
|
+
text += "#{Report.number(@awaiting)} entries awaiting purge; " if @awaiting.positive?
|
|
214
|
+
text
|
|
215
|
+
end
|
|
216
|
+
|
|
217
|
+
def pre_chain
|
|
218
|
+
purged = if @retention
|
|
219
|
+
@retention.summaries.reject { |found| found.status == :altered }
|
|
220
|
+
.sum(&:removed_pre_chain)
|
|
221
|
+
else
|
|
222
|
+
0
|
|
223
|
+
end
|
|
224
|
+
purged.positive? ? ", #{Report.number(purged)} purged" : ""
|
|
225
|
+
end
|
|
226
|
+
end
|
|
227
|
+
|
|
228
|
+
# What a run was anchored by, in words (specs/004 FR-014, FR-015): the clause every walked
|
|
229
|
+
# summary line ends with, and the wording for a walk from a checkpoint and for an emptied
|
|
230
|
+
# ledger that checkpoints saw.
|
|
231
|
+
class AnchorText
|
|
232
|
+
# A checkpoint's time for people: "2026-10-25 14:00 UTC". A time that does not parse is
|
|
233
|
+
# shown as written; verification never depends on it.
|
|
234
|
+
def self.made_on(made_at)
|
|
235
|
+
Time.iso8601(made_at).utc.strftime("%Y-%m-%d %H:%M UTC")
|
|
236
|
+
rescue ArgumentError
|
|
237
|
+
made_at
|
|
238
|
+
end
|
|
239
|
+
|
|
240
|
+
def initialize(verification)
|
|
241
|
+
@v = verification
|
|
242
|
+
end
|
|
243
|
+
|
|
244
|
+
# The highest checkpoint that checked, and how many entries it does not cover.
|
|
245
|
+
def to_s
|
|
246
|
+
anchor = @v.anchored_at
|
|
247
|
+
return "not anchored" if anchor.nil?
|
|
248
|
+
|
|
249
|
+
after = @v.beyond_anchor.to_i
|
|
250
|
+
"anchored at ##{number(anchor.head_sequence)} (checkpoint #{made_on(anchor)}), " \
|
|
251
|
+
"#{number(after)} #{after == 1 ? 'entry' : 'entries'} after it"
|
|
252
|
+
end
|
|
253
|
+
|
|
254
|
+
def walked_from
|
|
255
|
+
start = @v.walked_from
|
|
256
|
+
from = "checkpoint ##{number(start.head_sequence)} (#{made_on(start)})"
|
|
257
|
+
walked = if @v.covered
|
|
258
|
+
"sequences #{range} verified from #{from}"
|
|
259
|
+
else
|
|
260
|
+
"no entries after #{from}"
|
|
261
|
+
end
|
|
262
|
+
"#{walked}; #{number(start.head_sequence)} earlier chained entries not walked by " \
|
|
263
|
+
"this run"
|
|
264
|
+
end
|
|
265
|
+
|
|
266
|
+
# No walk, and checkpoints that saw a chain.
|
|
267
|
+
def emptied?
|
|
268
|
+
@v.head_at_start.nil? && @v.checkpoints.any?
|
|
269
|
+
end
|
|
270
|
+
|
|
271
|
+
def emptied
|
|
272
|
+
count = @v.checkpoints.size
|
|
273
|
+
"the ledger has no chain, but #{number(count)} checkpoint#{'s' unless count == 1} " \
|
|
274
|
+
"name#{'s' if count == 1} one"
|
|
275
|
+
end
|
|
276
|
+
|
|
277
|
+
private
|
|
278
|
+
|
|
279
|
+
def number(value)
|
|
280
|
+
Report.number(value)
|
|
281
|
+
end
|
|
282
|
+
|
|
283
|
+
def made_on(checkpoint)
|
|
284
|
+
AnchorText.made_on(checkpoint.made_at)
|
|
285
|
+
end
|
|
286
|
+
|
|
287
|
+
def range
|
|
288
|
+
"#{number(@v.covered.first)}–#{number(@v.covered.last)}"
|
|
289
|
+
end
|
|
290
|
+
end
|
|
291
|
+
|
|
135
292
|
# The key epochs, one line each (specs/003 FR-013), shown once they say something the
|
|
136
293
|
# summary does not: more than one, one that is not verified, or rows chained before the
|
|
137
294
|
# first. A ledger with one verified epoch and nothing before it reads as 0.6.0's did.
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "digest"
|
|
4
|
+
require "fileutils"
|
|
5
|
+
require "json"
|
|
6
|
+
|
|
7
|
+
module Reeve
|
|
8
|
+
module Audit
|
|
9
|
+
class Chain
|
|
10
|
+
module Retention
|
|
11
|
+
# The archive a purge writes before it removes anything (specs/005 FR-007,
|
|
12
|
+
# contracts/archive-v1.md): one JSON line of header, then every purged row with every
|
|
13
|
+
# column, chain columns included, so the range can be verified on its own.
|
|
14
|
+
#
|
|
15
|
+
# Written to `path.tmp`, flushed and renamed, so a partial archive is never at `path`.
|
|
16
|
+
module Archive
|
|
17
|
+
FORMAT = "reeve-ledger-archive-v1"
|
|
18
|
+
BATCH = 1_000
|
|
19
|
+
|
|
20
|
+
module_function
|
|
21
|
+
|
|
22
|
+
# Writes the archive for `plan` and returns its SHA-256.
|
|
23
|
+
def write(path, plan, entry_class:)
|
|
24
|
+
partial = "#{path}.tmp"
|
|
25
|
+
File.open(partial, File::WRONLY | File::CREAT | File::EXCL, 0o644) do |file|
|
|
26
|
+
file.puts(JSON.generate(header(plan)))
|
|
27
|
+
each_row(plan, entry_class) { |row| file.puts(JSON.generate(line(row))) }
|
|
28
|
+
file.flush
|
|
29
|
+
file.fsync
|
|
30
|
+
end
|
|
31
|
+
File.rename(partial, path)
|
|
32
|
+
::Digest::SHA256.file(path).hexdigest
|
|
33
|
+
rescue StandardError
|
|
34
|
+
FileUtils.rm_f(partial)
|
|
35
|
+
raise
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def header(plan)
|
|
39
|
+
{
|
|
40
|
+
"format" => FORMAT, "ledger" => ChainRecord::LEDGER, "purge" => plan.purge,
|
|
41
|
+
"from_sequence" => plan.from, "through_sequence" => plan.through,
|
|
42
|
+
"through_checkpoint" => plan.checkpoint.to_line, "start" => plan.start || genesis,
|
|
43
|
+
"chained" => plan.removed_chained, "pre_chain" => plan.removed_pre_chain,
|
|
44
|
+
"reeve_version" => Reeve::VERSION
|
|
45
|
+
}.sort.to_h
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
# Purge 0 starts at the genesis, so the archive carries it.
|
|
49
|
+
def genesis
|
|
50
|
+
record = ChainRecord.ledger
|
|
51
|
+
fields = Canonical::GENESIS_FIELDS + ["genesis_digest"]
|
|
52
|
+
{ "genesis" => fields.to_h { |field| [field, value(record[field])] } }
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
# Pre-chain rows first, then the chained range in sequence order, a batch at a time.
|
|
56
|
+
def each_row(plan, entry_class, &block)
|
|
57
|
+
entry_class.where("contract_version < 3").order(:id).find_each(batch_size: BATCH,
|
|
58
|
+
&block)
|
|
59
|
+
after = plan.from - 1
|
|
60
|
+
loop do
|
|
61
|
+
batch = entry_class.where(chain_sequence: (after + 1)..plan.through)
|
|
62
|
+
.order(:chain_sequence).limit(BATCH).to_a
|
|
63
|
+
batch.each(&block)
|
|
64
|
+
break if batch.size < BATCH
|
|
65
|
+
|
|
66
|
+
after = batch.last.chain_sequence
|
|
67
|
+
end
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
# The header of the archive at `path`, refusing one in another format: a verifier
|
|
71
|
+
# never guesses at a format it does not know.
|
|
72
|
+
def read_header(path)
|
|
73
|
+
header = JSON.parse(File.foreach(path).first.to_s)
|
|
74
|
+
return header if header.is_a?(Hash) && header["format"] == FORMAT
|
|
75
|
+
|
|
76
|
+
found = header.is_a?(Hash) ? header["format"].inspect : "unreadable"
|
|
77
|
+
raise ArgumentError, "#{path} is not a #{FORMAT} archive (format #{found})"
|
|
78
|
+
rescue JSON::ParserError
|
|
79
|
+
raise ArgumentError, "#{path} is not a #{FORMAT} archive (its first line is not JSON)"
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
# The header's genesis, with its time as a Time again, for Canonical.genesis_digest.
|
|
83
|
+
def genesis_values(genesis)
|
|
84
|
+
genesis.merge("genesis_at" => Time.iso8601(genesis["genesis_at"]))
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
def line(row)
|
|
88
|
+
row.attributes.except("id").transform_values { |item| value(item) }.sort.to_h
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
def value(item)
|
|
92
|
+
item.respond_to?(:utc) ? item.utc.iso8601(6) : item
|
|
93
|
+
end
|
|
94
|
+
end
|
|
95
|
+
end
|
|
96
|
+
end
|
|
97
|
+
end
|
|
98
|
+
end
|
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
|
|
5
|
+
module Reeve
|
|
6
|
+
module Audit
|
|
7
|
+
class Chain
|
|
8
|
+
module Retention
|
|
9
|
+
# Verifies an archive on its own (specs/005 FR-012, contracts/archive-v1.md): its boundary
|
|
10
|
+
# checkpoint is genuine, its start link is the genesis (or the boundary before it), and
|
|
11
|
+
# every chained row, rebuilt as an unsaved entry so it casts as the database did,
|
|
12
|
+
# recomputes its digest against the row before it, ending at the checkpoint's digest.
|
|
13
|
+
#
|
|
14
|
+
# Needs the entries table's columns, for the casts, but none of its rows.
|
|
15
|
+
class ArchiveCheck
|
|
16
|
+
BOUNDARY = "its boundary checkpoint"
|
|
17
|
+
|
|
18
|
+
def initialize(path, keyring:, entry_class:)
|
|
19
|
+
@path = path
|
|
20
|
+
@keyring = keyring
|
|
21
|
+
@entry_class = entry_class
|
|
22
|
+
@breaks = []
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def call
|
|
26
|
+
header = Archive.read_header(@path)
|
|
27
|
+
checkpoint = Checkpoint.parse(header.fetch("through_checkpoint"), position: 1)
|
|
28
|
+
return result(header, checkpoint) unless genuine?(checkpoint)
|
|
29
|
+
|
|
30
|
+
check_end(header, checkpoint, walk(header, start_link(header, checkpoint)))
|
|
31
|
+
result(header, checkpoint)
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
private
|
|
35
|
+
|
|
36
|
+
attr_reader :keyring, :breaks
|
|
37
|
+
|
|
38
|
+
def genuine?(checkpoint)
|
|
39
|
+
signed = checkpoint.signed_by(keyring)
|
|
40
|
+
key = checkpoint.key_id
|
|
41
|
+
add(:unknown_key, nil, "#{BOUNDARY} is signed by key #{key}", key_id: key) if
|
|
42
|
+
signed == :key_not_configured
|
|
43
|
+
if signed == :altered
|
|
44
|
+
add(:checkpoint_altered, nil,
|
|
45
|
+
"#{BOUNDARY} does not match its signature")
|
|
46
|
+
end
|
|
47
|
+
signed == :valid
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
# Purge 0 starts at the genesis, which is recomputed under its own key and must be the
|
|
51
|
+
# one the checkpoint names; a later purge starts at the previous boundary's digest,
|
|
52
|
+
# which the first row's own digest then binds.
|
|
53
|
+
def start_link(header, checkpoint)
|
|
54
|
+
genesis = header.dig("start", "genesis")
|
|
55
|
+
return header.dig("start", "through_digest") unless genesis
|
|
56
|
+
|
|
57
|
+
digest = genesis["genesis_digest"]
|
|
58
|
+
secret = keyring.for(genesis["genesis_key_id"])
|
|
59
|
+
if secret.nil?
|
|
60
|
+
add(:unknown_key, 0, "the genesis is signed by key #{genesis['genesis_key_id']}",
|
|
61
|
+
key_id: genesis["genesis_key_id"])
|
|
62
|
+
elsif Canonical.genesis_digest(secret, Archive.genesis_values(genesis)) != digest ||
|
|
63
|
+
checkpoint.genesis_digest != digest
|
|
64
|
+
add(:genesis_altered, 0,
|
|
65
|
+
"the archive's genesis does not match its digest or its checkpoint")
|
|
66
|
+
end
|
|
67
|
+
digest
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
# Every row after the header: pre-chain rows counted, chained rows checked in order.
|
|
71
|
+
def walk(header, link)
|
|
72
|
+
expected = header["from_sequence"]
|
|
73
|
+
last = nil
|
|
74
|
+
@chained = @pre_chain = 0
|
|
75
|
+
File.foreach(@path).drop(1).each do |line|
|
|
76
|
+
entry = @entry_class.new(JSON.parse(line))
|
|
77
|
+
next @pre_chain += 1 if entry.chain_sequence.nil?
|
|
78
|
+
|
|
79
|
+
expected, link = visit(entry, expected, link)
|
|
80
|
+
last = entry
|
|
81
|
+
end
|
|
82
|
+
last
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
def visit(entry, expected, link)
|
|
86
|
+
@chained += 1
|
|
87
|
+
sequence = entry.chain_sequence
|
|
88
|
+
if sequence != expected
|
|
89
|
+
kind = sequence < expected ? :duplicate : :missing
|
|
90
|
+
add(kind, sequence < expected ? sequence : expected..(sequence - 1),
|
|
91
|
+
"out of order in the archive")
|
|
92
|
+
end
|
|
93
|
+
check_digest(entry, link)
|
|
94
|
+
[sequence + 1, entry.chain_digest]
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
def check_digest(entry, link)
|
|
98
|
+
secret = keyring.for(entry.chain_key_id)
|
|
99
|
+
if secret.nil?
|
|
100
|
+
return add(:unknown_key, entry.chain_sequence, "signed by key #{entry.chain_key_id}",
|
|
101
|
+
key_id: entry.chain_key_id)
|
|
102
|
+
end
|
|
103
|
+
return if Canonical.entry_digest(secret, entry,
|
|
104
|
+
previous_digest: link) == entry.chain_digest
|
|
105
|
+
|
|
106
|
+
add(:digest_mismatch, entry.chain_sequence, Walk::MISMATCH)
|
|
107
|
+
rescue ArgumentError => e
|
|
108
|
+
add(:digest_mismatch, entry.chain_sequence, e.message)
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
def check_end(header, checkpoint, last)
|
|
112
|
+
reached = last&.chain_sequence == header["through_sequence"] &&
|
|
113
|
+
last&.chain_digest == checkpoint.head_digest
|
|
114
|
+
unless reached
|
|
115
|
+
add(:checkpoint_not_reached, nil, "the archive does not end at its boundary " \
|
|
116
|
+
"checkpoint " \
|
|
117
|
+
"##{Report.number(checkpoint.head_sequence)}")
|
|
118
|
+
end
|
|
119
|
+
return if @chained == header["chained"] && @pre_chain == header["pre_chain"]
|
|
120
|
+
|
|
121
|
+
add(:missing, nil, "the header counts #{header['chained']} chained and " \
|
|
122
|
+
"#{header['pre_chain']} " \
|
|
123
|
+
"pre-chain rows; the archive holds #{@chained} and #{@pre_chain}")
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
def add(kind, sequence, detail, key_id: nil)
|
|
127
|
+
breaks << Verification::Break.new(kind: kind, sequence: sequence, detail: detail,
|
|
128
|
+
key_id: key_id)
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
def result(header, checkpoint)
|
|
132
|
+
ArchiveResult.new(status: Verification.status_for(breaks), breaks: breaks,
|
|
133
|
+
covered: header["from_sequence"]..header["through_sequence"],
|
|
134
|
+
checkpoint: checkpoint, pre_chain: @pre_chain.to_i)
|
|
135
|
+
end
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
# What verifying an archive found, in words (`bin/rails reeve:verify_archive`).
|
|
139
|
+
ArchiveResult = Struct.new(:status, :breaks, :covered, :checkpoint, :pre_chain,
|
|
140
|
+
keyword_init: true) do
|
|
141
|
+
def intact? = status == :intact
|
|
142
|
+
|
|
143
|
+
def to_s
|
|
144
|
+
listed = breaks.reject { |found| found.kind == :unknown_key }
|
|
145
|
+
[summary, *listed.map { |found| line(found) }].join("\n")
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
private
|
|
149
|
+
|
|
150
|
+
def summary
|
|
151
|
+
word, found =
|
|
152
|
+
case status
|
|
153
|
+
when :intact then ["INTACT", "sequences #{range} verified, ending at #{ending}"]
|
|
154
|
+
when :broken then ["BROKEN", "#{n(breaks.size)} breaks in sequences #{range}"]
|
|
155
|
+
else ["UNVERIFIABLE", "signed by a key not configured; nothing in #{range} verified"]
|
|
156
|
+
end
|
|
157
|
+
"reeve archive: #{word} — #{found}; #{counted}"
|
|
158
|
+
end
|
|
159
|
+
|
|
160
|
+
def range = "#{n(covered.first)}–#{n(covered.last)}"
|
|
161
|
+
|
|
162
|
+
def ending
|
|
163
|
+
"checkpoint ##{n(checkpoint.head_sequence)} (#{AnchorText.made_on(checkpoint.made_at)})"
|
|
164
|
+
end
|
|
165
|
+
|
|
166
|
+
def counted = "#{n(pre_chain)} pre-chain rows counted, not verified"
|
|
167
|
+
|
|
168
|
+
def line(found)
|
|
169
|
+
sequence = found.sequence
|
|
170
|
+
where = sequence.is_a?(Range) ? "##{sequence.first}–#{sequence.last}" : "##{sequence}"
|
|
171
|
+
" #{sequence.nil? ? '—' : where} #{found.kind} #{found.detail}"
|
|
172
|
+
end
|
|
173
|
+
|
|
174
|
+
def n(value) = Report.number(value.to_i)
|
|
175
|
+
end
|
|
176
|
+
end
|
|
177
|
+
end
|
|
178
|
+
end
|
|
179
|
+
end
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Reeve
|
|
4
|
+
module Audit
|
|
5
|
+
class Chain
|
|
6
|
+
module Retention
|
|
7
|
+
# Checks the retention records (specs/005 FR-009): numbered 0 to n, each signed under a
|
|
8
|
+
# key this verifier holds, each linked to the one before and record 0 to the genesis,
|
|
9
|
+
# each boundary following on from the last. Leaves the boundary the walk starts after:
|
|
10
|
+
# the last record of the unbroken run from record 0.
|
|
11
|
+
#
|
|
12
|
+
# A record whose key is not held cannot be checked; it still sets the boundary (the
|
|
13
|
+
# rows it accounts for are gone either way) and makes the run unverifiable, never
|
|
14
|
+
# intact and never broken. A record that fails ends the run: the walk starts at the one
|
|
15
|
+
# before it, so the rows the failing record claims to account for are reported missing.
|
|
16
|
+
class Check
|
|
17
|
+
Summary = Struct.new(:purge, :through_sequence, :purged_at, :removed_chained,
|
|
18
|
+
:removed_pre_chain, :archived, :status, keyword_init: true) do
|
|
19
|
+
def as_json(*)
|
|
20
|
+
to_h.transform_keys(&:to_s).merge("status" => status.to_s)
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
Result = Struct.new(:boundary, :summaries, keyword_init: true) do
|
|
25
|
+
def as_json(*)
|
|
26
|
+
{ "through" => boundary&.through_sequence, "records" => summaries.map(&:as_json) }
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
# `report` receives (kind, sequence, detail, invocation_id, key_id, retention:).
|
|
31
|
+
def initialize(keyring:, report:)
|
|
32
|
+
@keyring = keyring
|
|
33
|
+
@report = report
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def call(records, genesis_digest:)
|
|
37
|
+
boundary = nil
|
|
38
|
+
previous = nil
|
|
39
|
+
summaries = records.each_with_index.map do |record, index|
|
|
40
|
+
status = judge(record, index, previous,
|
|
41
|
+
previous ? previous.retention_digest : genesis_digest)
|
|
42
|
+
boundary = record if boundary.equal?(previous) && status != :altered
|
|
43
|
+
previous = record
|
|
44
|
+
summary(record, status)
|
|
45
|
+
end
|
|
46
|
+
Result.new(boundary: boundary, summaries: summaries)
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
private
|
|
50
|
+
|
|
51
|
+
attr_reader :keyring
|
|
52
|
+
|
|
53
|
+
def judge(record, index, previous, link)
|
|
54
|
+
if record.purge_number != index
|
|
55
|
+
return altered(record,
|
|
56
|
+
"is numbered #{record.purge_number}, not #{index}")
|
|
57
|
+
end
|
|
58
|
+
return altered(record, "does not follow on from the purge before it") unless follows?(
|
|
59
|
+
record, previous
|
|
60
|
+
)
|
|
61
|
+
|
|
62
|
+
secret = keyring.for(record.key_id)
|
|
63
|
+
if secret.nil?
|
|
64
|
+
@report.call(:unknown_key, nil, "signed by key #{record.key_id}", nil, record.key_id,
|
|
65
|
+
retention: record.purge_number)
|
|
66
|
+
return :key_not_configured
|
|
67
|
+
end
|
|
68
|
+
return :verified if Digest.call(secret, record, previous_digest: link) ==
|
|
69
|
+
record.retention_digest
|
|
70
|
+
|
|
71
|
+
altered(record, "does not match its digest; it was edited, or a record before it was")
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
def follows?(record, previous)
|
|
75
|
+
through = previous ? previous.through_sequence : 0
|
|
76
|
+
record.previous_through == through && record.through_sequence > through
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
def altered(record, detail)
|
|
80
|
+
@report.call(:retention_altered, nil, detail, nil, nil, retention: record.purge_number)
|
|
81
|
+
:altered
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
def summary(record, status)
|
|
85
|
+
Summary.new(purge: record.purge_number, through_sequence: record.through_sequence,
|
|
86
|
+
purged_at: record.purged_at, removed_chained: record.removed_chained,
|
|
87
|
+
removed_pre_chain: record.removed_pre_chain, archived: record.archived,
|
|
88
|
+
status: status)
|
|
89
|
+
end
|
|
90
|
+
end
|
|
91
|
+
end
|
|
92
|
+
end
|
|
93
|
+
end
|
|
94
|
+
end
|