reeve 0.7.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +117 -0
- data/README.md +236 -9
- data/lib/generators/reeve/install/templates/create_audit_entries.rb.tt +29 -4
- data/lib/generators/reeve/install/templates/initializer.rb.tt +10 -0
- data/lib/generators/reeve/upgrade/templates/add_ledger_retention_to_reeve_audit_entries.rb.tt +37 -0
- data/lib/generators/reeve/upgrade/upgrade_generator.rb +10 -1
- data/lib/generators/reeve/upgrade/upgrade_notes.rb +15 -4
- data/lib/reeve/audit/chain/anchoring.rb +91 -0
- data/lib/reeve/audit/chain/canonical.rb +17 -1
- data/lib/reeve/audit/chain/checkpoint.rb +228 -0
- data/lib/reeve/audit/chain/checkpoint_check.rb +156 -0
- data/lib/reeve/audit/chain/entrypoints.rb +29 -0
- data/lib/reeve/audit/chain/epochs.rb +10 -4
- data/lib/reeve/audit/chain/head.rb +26 -1
- data/lib/reeve/audit/chain/keyring.rb +6 -0
- data/lib/reeve/audit/chain/report.rb +188 -31
- data/lib/reeve/audit/chain/retention/archive.rb +98 -0
- data/lib/reeve/audit/chain/retention/archive_check.rb +179 -0
- data/lib/reeve/audit/chain/retention/check.rb +94 -0
- data/lib/reeve/audit/chain/retention/connection.rb +65 -0
- data/lib/reeve/audit/chain/retention/digest.rb +29 -0
- data/lib/reeve/audit/chain/retention/plan.rb +100 -0
- data/lib/reeve/audit/chain/retention/preflight.rb +48 -0
- data/lib/reeve/audit/chain/retention/purge.rb +155 -0
- data/lib/reeve/audit/chain/retention/purge_result.rb +134 -0
- data/lib/reeve/audit/chain/retention/records.rb +23 -0
- data/lib/reeve/audit/chain/verification.rb +31 -9
- data/lib/reeve/audit/chain/verifier.rb +64 -19
- data/lib/reeve/audit/chain/walk.rb +9 -3
- data/lib/reeve/audit/chain.rb +12 -3
- data/lib/reeve/audit/chain_record.rb +3 -2
- data/lib/reeve/audit/file_anchor.rb +49 -0
- data/lib/reeve/audit/retention_record.rb +24 -0
- data/lib/reeve/audit.rb +16 -0
- data/lib/reeve/configuration.rb +25 -0
- data/lib/reeve/errors.rb +15 -0
- data/lib/reeve/railtie.rb +2 -2
- data/lib/reeve/tasks/descriptions.rb +22 -0
- data/lib/reeve/tasks/purge_options.rb +54 -0
- data/lib/reeve/tasks.rb +82 -14
- data/lib/reeve/testing/checks/ledger_chain.rb +56 -2
- data/lib/reeve/testing/ledger.rb +19 -3
- data/lib/reeve/version.rb +1 -1
- metadata +21 -2
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Reeve
|
|
4
|
+
module Audit
|
|
5
|
+
class Chain
|
|
6
|
+
# The verifier's checkpoints (specs/004): what it checks the ledger against, and the one,
|
|
7
|
+
# if any, it walks from instead of the genesis.
|
|
8
|
+
#
|
|
9
|
+
# Checkpoints come from outside the database (FR-010). With none, the verifier is exactly
|
|
10
|
+
# 0.7.0's and its report says the run was not anchored.
|
|
11
|
+
class Anchoring
|
|
12
|
+
attr_reader :checkpoints, :from
|
|
13
|
+
|
|
14
|
+
# `from` is :latest, a Checkpoint, or nil to walk from the genesis (Checkpoint.start).
|
|
15
|
+
def initialize(entry_class:, keyring:, checkpoints:, from: nil)
|
|
16
|
+
@entry_class = entry_class
|
|
17
|
+
@keyring = keyring
|
|
18
|
+
@checkpoints = checkpoints
|
|
19
|
+
@from = Checkpoint.start(from, checkpoints)
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def any?
|
|
23
|
+
checkpoints.any?
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
# The latest retention record that holds (specs/005), or nil: the walk never starts
|
|
27
|
+
# below it, and a `from` at or below it is moot.
|
|
28
|
+
attr_reader :floor
|
|
29
|
+
|
|
30
|
+
def floor=(record)
|
|
31
|
+
@floor = record
|
|
32
|
+
@from = nil if record && from && from.head_sequence <= record.through_sequence
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
# Where the walk starts, and the digest its first row links to.
|
|
36
|
+
def start
|
|
37
|
+
return from.head_sequence if from
|
|
38
|
+
return floor.through_sequence if floor
|
|
39
|
+
|
|
40
|
+
0
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def link(record)
|
|
44
|
+
return from.head_digest if from
|
|
45
|
+
return floor.through_digest if floor
|
|
46
|
+
|
|
47
|
+
record&.genesis_digest
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
# Every checkpoint, after the walk, against the head read after it. The checked
|
|
51
|
+
# checkpoint with the highest sequence anchors the run (data-model).
|
|
52
|
+
def check(record, head, epoch_records, report:)
|
|
53
|
+
results = check_with(report).call(checkpoints, record: record, head: head,
|
|
54
|
+
epochs: epoch_records, floor: floor)
|
|
55
|
+
anchor = results.select { |found| found.status == :checked }.max_by(&:head_sequence)
|
|
56
|
+
beyond = anchor && record && [record.head_sequence - anchor.head_sequence, 0].max
|
|
57
|
+
{ checkpoints: results, anchored_at: anchor, beyond_anchor: beyond }
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
# A ledger with no chain, which every checkpoint saw reach an entry: emptied, not never
|
|
61
|
+
# chained (FR-013).
|
|
62
|
+
def emptied(record, report:)
|
|
63
|
+
check_with(report).call(checkpoints, record: record, head: record, epochs: [],
|
|
64
|
+
chained: false)
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
# The starting checkpoint, checked alone before the walk relies on it (FR-015). Returns
|
|
68
|
+
# its results when it does not hold, reporting its breaks; nil when it holds, so its
|
|
69
|
+
# breaks are left to the full check after the walk.
|
|
70
|
+
def failed_start(record, epoch_records, report:)
|
|
71
|
+
return nil if from.nil?
|
|
72
|
+
|
|
73
|
+
found = []
|
|
74
|
+
collect = ->(*args, **options) { found << [args, options] }
|
|
75
|
+
results = check_with(collect).call([from], record: record, head: record,
|
|
76
|
+
epochs: epoch_records, floor: floor)
|
|
77
|
+
return nil if results.first.status == :checked
|
|
78
|
+
|
|
79
|
+
found.each { |args, options| report.call(*args, **options) }
|
|
80
|
+
results
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
private
|
|
84
|
+
|
|
85
|
+
def check_with(report)
|
|
86
|
+
CheckpointCheck.new(entry_class: @entry_class, keyring: @keyring, report: report)
|
|
87
|
+
end
|
|
88
|
+
end
|
|
89
|
+
end
|
|
90
|
+
end
|
|
91
|
+
end
|
|
@@ -33,6 +33,8 @@ module Reeve
|
|
|
33
33
|
|
|
34
34
|
EPOCH_TAG = "reeve-ledger-epoch-v1"
|
|
35
35
|
|
|
36
|
+
CHECKPOINT_TAG = "reeve-ledger-checkpoint-v1"
|
|
37
|
+
|
|
36
38
|
CONTRACT_3_FIELDS = %w[
|
|
37
39
|
chain_sequence chain_key_id contract_version
|
|
38
40
|
invocation_id occurred_at agent_id agent_name principal_type principal_id
|
|
@@ -51,7 +53,14 @@ module Reeve
|
|
|
51
53
|
|
|
52
54
|
EPOCH_FIELDS = %w[epoch key_id first_sequence opened_at].freeze
|
|
53
55
|
|
|
54
|
-
|
|
56
|
+
# A checkpoint's signed fields (specs/004 contracts/checkpoint-v1.md), in this order.
|
|
57
|
+
CHECKPOINT_FIELDS = %w[
|
|
58
|
+
format ledger genesis_digest pre_chain_count pre_chain_unchained_count
|
|
59
|
+
head_sequence head_digest epoch epoch_digest key_id made_at reeve_version
|
|
60
|
+
].freeze
|
|
61
|
+
|
|
62
|
+
# `purged_at` is a retention record's (Retention::Digest).
|
|
63
|
+
TIME_FIELDS = %w[occurred_at genesis_at opened_at purged_at].freeze
|
|
55
64
|
|
|
56
65
|
module_function
|
|
57
66
|
|
|
@@ -71,6 +80,13 @@ module Reeve
|
|
|
71
80
|
hmac(secret, EPOCH_TAG, dump([["previous_digest", previous_digest], *pairs]))
|
|
72
81
|
end
|
|
73
82
|
|
|
83
|
+
# A checkpoint (specs/004). `made_at` is signed as the string it is stored as, so a line
|
|
84
|
+
# verifies exactly as written, whatever its reader does with times.
|
|
85
|
+
def checkpoint_signature(secret, values)
|
|
86
|
+
pairs = CHECKPOINT_FIELDS.map { |field| [field, values.fetch(field)] }
|
|
87
|
+
hmac(secret, CHECKPOINT_TAG, dump(pairs))
|
|
88
|
+
end
|
|
89
|
+
|
|
74
90
|
def entry_message_body(row, previous_digest)
|
|
75
91
|
version = value(row, "contract_version")
|
|
76
92
|
fields = ENTRY_FIELDS.fetch(version) do
|
|
@@ -0,0 +1,228 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
|
|
5
|
+
module Reeve
|
|
6
|
+
module Audit
|
|
7
|
+
class Chain
|
|
8
|
+
# A signed statement of where the chain reached (specs/004, contracts/checkpoint-v1.md):
|
|
9
|
+
# the genesis, the head, and the latest key epoch, which pins every earlier epoch because
|
|
10
|
+
# epoch digests are chained to each other and to the genesis.
|
|
11
|
+
#
|
|
12
|
+
# Kept outside the database by the host's anchor, so that removing entries and rewinding
|
|
13
|
+
# the head, or emptying the ledger, leaves a witness the database cannot change. It
|
|
14
|
+
# asserts where the chain reached, not that the chain was intact, and its `made_at` is
|
|
15
|
+
# the producing host's clock: nothing in verification depends on it.
|
|
16
|
+
#
|
|
17
|
+
# It holds digests, counts, a key fingerprint, a time and versions — never entry content,
|
|
18
|
+
# never a key — so it can be stored where the ledger's data must not be.
|
|
19
|
+
class Checkpoint
|
|
20
|
+
FORMAT = "reeve-ledger-checkpoint-v1"
|
|
21
|
+
|
|
22
|
+
TIME_FORMAT = "%Y-%m-%dT%H:%M:%S.%6NZ"
|
|
23
|
+
|
|
24
|
+
INTEGER = [Integer].freeze
|
|
25
|
+
STRING = [String].freeze
|
|
26
|
+
OPTIONAL_INTEGER = [Integer, NilClass].freeze
|
|
27
|
+
OPTIONAL_STRING = [String, NilClass].freeze
|
|
28
|
+
|
|
29
|
+
TYPES = {
|
|
30
|
+
"format" => STRING, "ledger" => STRING, "genesis_digest" => STRING,
|
|
31
|
+
"pre_chain_count" => INTEGER, "pre_chain_unchained_count" => INTEGER,
|
|
32
|
+
"head_sequence" => INTEGER, "head_digest" => STRING,
|
|
33
|
+
"epoch" => OPTIONAL_INTEGER, "epoch_digest" => OPTIONAL_STRING,
|
|
34
|
+
"key_id" => STRING, "made_at" => STRING, "reeve_version" => STRING,
|
|
35
|
+
"signature" => STRING
|
|
36
|
+
}.freeze
|
|
37
|
+
|
|
38
|
+
TYPES.each_key { |field| define_method(field) { @values[field] } }
|
|
39
|
+
|
|
40
|
+
# `state` is the chain record's columns as one read returned them; `epoch` the latest
|
|
41
|
+
# epoch's number and digest, or nil for a ledger with none.
|
|
42
|
+
def self.build(state:, epoch:, keyring:)
|
|
43
|
+
values = {
|
|
44
|
+
"format" => FORMAT, "ledger" => ChainRecord::LEDGER,
|
|
45
|
+
"genesis_digest" => state["genesis_digest"],
|
|
46
|
+
"pre_chain_count" => Integer(state["pre_chain_count"]),
|
|
47
|
+
"pre_chain_unchained_count" => Integer(state["pre_chain_unchained_count"]),
|
|
48
|
+
"head_sequence" => Integer(state["head_sequence"]),
|
|
49
|
+
"head_digest" => state["head_digest"],
|
|
50
|
+
"epoch" => epoch && epoch[:epoch], "epoch_digest" => epoch && epoch[:epoch_digest],
|
|
51
|
+
"key_id" => keyring.signing_id, "made_at" => Time.now.utc.strftime(TIME_FORMAT),
|
|
52
|
+
"reeve_version" => Reeve::VERSION
|
|
53
|
+
}
|
|
54
|
+
signature = Canonical.checkpoint_signature(keyring.signing_secret, values)
|
|
55
|
+
new(values.merge("signature" => signature))
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
# `position` is the line's place in its list, from 1, for the error that names it.
|
|
59
|
+
def self.parse(line, position:)
|
|
60
|
+
values = JSON.parse(line.to_s)
|
|
61
|
+
reason = problem(values)
|
|
62
|
+
raise CheckpointError.new(reason, position: position) if reason
|
|
63
|
+
|
|
64
|
+
new(values)
|
|
65
|
+
rescue JSON::ParserError
|
|
66
|
+
raise CheckpointError.new("not JSON", position: position)
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
# Checkpoint lines or Checkpoints, as Checkpoints; nil means the configured anchor's.
|
|
70
|
+
def self.read_all(given)
|
|
71
|
+
given = Reeve.config.ledger_anchor&.checkpoints || [] if given.nil?
|
|
72
|
+
given.each_with_index.map do |checkpoint, index|
|
|
73
|
+
checkpoint.is_a?(Checkpoint) ? checkpoint : parse(checkpoint, position: index + 1)
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
# Where verification walks from (FR-015): nil for the genesis, :latest for the highest
|
|
78
|
+
# of `checkpoints`, or a Checkpoint given as itself.
|
|
79
|
+
def self.start(from, checkpoints)
|
|
80
|
+
return from if from.nil? || from.is_a?(Checkpoint)
|
|
81
|
+
unless from == :latest
|
|
82
|
+
raise ArgumentError,
|
|
83
|
+
"from: must be :latest or a Checkpoint, got #{from.inspect}"
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
checkpoints.max_by(&:head_sequence) ||
|
|
87
|
+
raise(CheckpointError,
|
|
88
|
+
"FROM=latest needs a checkpoint to start from, and none was given")
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
def self.problem(values)
|
|
92
|
+
return "not a JSON object" unless values.is_a?(Hash)
|
|
93
|
+
|
|
94
|
+
shape_problem(values) || value_problem(values)
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
def self.shape_problem(values)
|
|
98
|
+
missing = TYPES.keys - values.keys
|
|
99
|
+
extra = values.keys - TYPES.keys
|
|
100
|
+
return "missing #{missing.join(', ')}" if missing.any?
|
|
101
|
+
return "unexpected #{extra.join(', ')}" if extra.any?
|
|
102
|
+
|
|
103
|
+
"format #{values['format'].inspect} is not one this version reads (#{FORMAT})" unless
|
|
104
|
+
values["format"] == FORMAT
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
def self.value_problem(values)
|
|
108
|
+
wrong = TYPES.find { |field, types| types.none? { |type| values[field].is_a?(type) } }
|
|
109
|
+
if wrong
|
|
110
|
+
return "#{wrong[0]} is not #{wrong[1].first == Integer ? 'a whole number' : 'a string'}"
|
|
111
|
+
end
|
|
112
|
+
return "epoch and epoch_digest must both be set or both be null" if
|
|
113
|
+
values["epoch"].nil? != values["epoch_digest"].nil?
|
|
114
|
+
|
|
115
|
+
"head_sequence must be at least 1" unless values["head_sequence"].positive?
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
private_class_method :problem, :shape_problem, :value_problem
|
|
119
|
+
|
|
120
|
+
def initialize(values)
|
|
121
|
+
@values = values.dup.freeze
|
|
122
|
+
freeze
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
# One line of JSON, keys sorted (contracts/checkpoint-v1.md).
|
|
126
|
+
def to_line
|
|
127
|
+
JSON.generate(@values.sort.to_h)
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
# :valid, :altered, or :key_not_configured when `keyring` lacks the key that signed it.
|
|
131
|
+
def signed_by(keyring)
|
|
132
|
+
secret = keyring.for(key_id)
|
|
133
|
+
return :key_not_configured if secret.nil?
|
|
134
|
+
|
|
135
|
+
Canonical.checkpoint_signature(secret, @values) == signature ? :valid : :altered
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
def as_json(*)
|
|
139
|
+
{ "head_sequence" => head_sequence, "made_at" => made_at }
|
|
140
|
+
end
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
# What `bin/rails reeve:checkpoint_ledger` did (specs/004 data-model "Checkpoint result").
|
|
144
|
+
#
|
|
145
|
+
# :made — `checkpoint` was made; `published` says whether the anchor took it, and
|
|
146
|
+
# `error` why it did not, when one is configured
|
|
147
|
+
# :refused — nothing made; `reason` is :no_key or :no_chain
|
|
148
|
+
class CheckpointResult
|
|
149
|
+
PREFIX = "reeve ledger checkpoint:"
|
|
150
|
+
|
|
151
|
+
attr_reader :status, :reason, :checkpoint, :error
|
|
152
|
+
|
|
153
|
+
def self.refused(reason)
|
|
154
|
+
new(status: :refused, reason: reason)
|
|
155
|
+
end
|
|
156
|
+
|
|
157
|
+
# Makes a checkpoint on the ledger's own connection and hands it to `anchor`. The
|
|
158
|
+
# reading happens under the head's lock (Head#checkpoint); publishing after it is
|
|
159
|
+
# released, so a slow store never holds a writer (FR-009).
|
|
160
|
+
def self.make!(entry_class:, anchor:, keyring: Keyring.from(Reeve.config))
|
|
161
|
+
return refused(:no_key) unless keyring.signing?
|
|
162
|
+
|
|
163
|
+
Schema.new(entry_class).require_columns!
|
|
164
|
+
checkpoint = entry_class.connection_pool.with_connection do |connection|
|
|
165
|
+
Head.new(connection: connection, entry_class: entry_class).checkpoint(keyring)
|
|
166
|
+
end
|
|
167
|
+
checkpoint ? publish(checkpoint, anchor) : refused(:no_chain)
|
|
168
|
+
end
|
|
169
|
+
|
|
170
|
+
# Hands the line to `anchor`, if there is one. A failing anchor is reported, not raised:
|
|
171
|
+
# the checkpoint was made, and its line can still be stored by hand.
|
|
172
|
+
def self.publish(checkpoint, anchor)
|
|
173
|
+
return new(status: :made, checkpoint: checkpoint) if anchor.nil?
|
|
174
|
+
|
|
175
|
+
anchor.publish(checkpoint.to_line)
|
|
176
|
+
new(status: :made, checkpoint: checkpoint, published: true)
|
|
177
|
+
rescue StandardError => e
|
|
178
|
+
new(status: :made, checkpoint: checkpoint, error: "#{e.message} (#{e.class})")
|
|
179
|
+
end
|
|
180
|
+
|
|
181
|
+
def initialize(status:, reason: nil, checkpoint: nil, published: false, error: nil)
|
|
182
|
+
@status = status
|
|
183
|
+
@reason = reason
|
|
184
|
+
@checkpoint = checkpoint
|
|
185
|
+
@published = published
|
|
186
|
+
@error = error
|
|
187
|
+
freeze
|
|
188
|
+
end
|
|
189
|
+
|
|
190
|
+
def made? = status == :made
|
|
191
|
+
def refused? = status == :refused
|
|
192
|
+
def published? = @published
|
|
193
|
+
def anchor_failed? = !error.nil?
|
|
194
|
+
|
|
195
|
+
def to_s
|
|
196
|
+
return "#{PREFIX} REFUSED — #{refusal}; nothing was made" if refused?
|
|
197
|
+
|
|
198
|
+
[first_line, checkpoint.to_line].join("\n")
|
|
199
|
+
end
|
|
200
|
+
|
|
201
|
+
private
|
|
202
|
+
|
|
203
|
+
def first_line
|
|
204
|
+
"#{PREFIX} MADE — head ##{Report.number(checkpoint.head_sequence)}#{epoch_text}, key " \
|
|
205
|
+
"#{checkpoint.key_id}; #{where}"
|
|
206
|
+
end
|
|
207
|
+
|
|
208
|
+
def epoch_text
|
|
209
|
+
checkpoint.epoch.nil? ? "" : ", epoch #{checkpoint.epoch}"
|
|
210
|
+
end
|
|
211
|
+
|
|
212
|
+
def where
|
|
213
|
+
return "published to the anchor" if published?
|
|
214
|
+
return "the anchor failed (#{error}), store the line below yourself" if anchor_failed?
|
|
215
|
+
|
|
216
|
+
"no anchor configured, store the line below yourself"
|
|
217
|
+
end
|
|
218
|
+
|
|
219
|
+
def refusal
|
|
220
|
+
case reason
|
|
221
|
+
when :no_key then "no ledger key configured (config.ledger_key)"
|
|
222
|
+
else "the ledger has no chain yet; nothing to checkpoint"
|
|
223
|
+
end
|
|
224
|
+
end
|
|
225
|
+
end
|
|
226
|
+
end
|
|
227
|
+
end
|
|
228
|
+
end
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Reeve
|
|
4
|
+
module Audit
|
|
5
|
+
class Chain
|
|
6
|
+
# Checks each checkpoint against the ledger (specs/004 FR-011, FR-012): that it is
|
|
7
|
+
# genuine, that the chain it names is this one, that the epoch it names is still there,
|
|
8
|
+
# and that the ledger still reaches its entry with the digest it names.
|
|
9
|
+
#
|
|
10
|
+
# Each checkpoint is checked on its own; none is compared with another. A rule about
|
|
11
|
+
# their order would need a trusted clock or a trusted order, and nothing supplies either;
|
|
12
|
+
# every case such a rule would catch already fails here (spec amendment 2).
|
|
13
|
+
#
|
|
14
|
+
# A checkpoint signed by a key this verifier lacks is reported as unknown_key and used for
|
|
15
|
+
# nothing else: an unchecked checkpoint must not anchor anything.
|
|
16
|
+
class CheckpointCheck
|
|
17
|
+
LOOKUP_BATCH = 1_000
|
|
18
|
+
|
|
19
|
+
Result = Struct.new(:head_sequence, :made_at, :key_id, :status, keyword_init: true) do
|
|
20
|
+
def as_json(*) = to_h.transform_keys(&:to_s).merge("status" => status.to_s)
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
# The break each failing status is reported as.
|
|
24
|
+
KINDS = { key_not_configured: :unknown_key, altered: :checkpoint_altered,
|
|
25
|
+
mismatch: :checkpoint_mismatch, not_reached: :checkpoint_not_reached }.freeze
|
|
26
|
+
|
|
27
|
+
ALTERED = "does not match its signature; it was edited, or is not reeve's"
|
|
28
|
+
NO_CHAIN = "the ledger has no chain, but this checkpoint saw it reach %s"
|
|
29
|
+
NO_RECORD = "its genesis cannot be checked: the chain record is missing"
|
|
30
|
+
OTHER_CHAIN = "names another chain: its genesis is not this ledger's"
|
|
31
|
+
EPOCH = "names epoch %d, %s"
|
|
32
|
+
CHANGED = "whose record is not the one it saw"
|
|
33
|
+
GONE = "which the ledger no longer has"
|
|
34
|
+
REMOVED = "the head is at %s; entries the checkpoint saw were removed"
|
|
35
|
+
BOUNDARY = "is at the purge boundary, but names another digest than the retention record"
|
|
36
|
+
|
|
37
|
+
# `report` receives (kind, sequence, detail, invocation_id, key_id, checkpoint:).
|
|
38
|
+
def initialize(entry_class:, keyring:, report:)
|
|
39
|
+
@entry_class = entry_class
|
|
40
|
+
@keyring = keyring
|
|
41
|
+
@report = report
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
# `record` is the chain record read at the start, `head` the one read after the walk
|
|
45
|
+
# (the later read, so an entry written during the walk is not "not reached"); `epochs`
|
|
46
|
+
# the epoch records the verifier loaded. `chained` says whether the ledger has a chain
|
|
47
|
+
# at all. Returns one Result per checkpoint, in the order given.
|
|
48
|
+
#
|
|
49
|
+
# `floor` is the latest retention record (specs/005): a checkpoint below it saw entries
|
|
50
|
+
# a purge removed, and is `purged`; the one at it is checked against the record.
|
|
51
|
+
def call(checkpoints, record:, head:, epochs:, chained: true, floor: nil)
|
|
52
|
+
genuine = checkpoints.select { |checkpoint| checkpoint.signed_by(keyring) == :valid }
|
|
53
|
+
digests = chained ? digests_at(genuine.map(&:head_sequence)) : {}
|
|
54
|
+
context = { record: record, head: head, epochs: epochs.to_h { |e| [e.epoch, e] },
|
|
55
|
+
digests: digests, chained: chained, floor: floor }
|
|
56
|
+
checkpoints.map do |checkpoint|
|
|
57
|
+
Result.new(head_sequence: checkpoint.head_sequence, made_at: checkpoint.made_at,
|
|
58
|
+
key_id: checkpoint.key_id, status: judge(checkpoint, **context))
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
private
|
|
63
|
+
|
|
64
|
+
attr_reader :entry_class, :keyring
|
|
65
|
+
|
|
66
|
+
def judge(checkpoint, record:, head:, epochs:, digests:, chained:, floor:)
|
|
67
|
+
signature(checkpoint) ||
|
|
68
|
+
chain(checkpoint, record, chained) ||
|
|
69
|
+
epoch(checkpoint, epochs) ||
|
|
70
|
+
purged(checkpoint, floor) ||
|
|
71
|
+
position(checkpoint, head, digests) ||
|
|
72
|
+
:checked
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
# At or below the latest purge's boundary (specs/005 amendment 4).
|
|
76
|
+
def purged(checkpoint, floor)
|
|
77
|
+
return if floor.nil? || checkpoint.head_sequence > floor.through_sequence
|
|
78
|
+
return :purged if checkpoint.head_sequence < floor.through_sequence
|
|
79
|
+
|
|
80
|
+
return :checked if checkpoint.head_digest == floor.through_digest
|
|
81
|
+
|
|
82
|
+
flag(:mismatch, checkpoint, BOUNDARY)
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
def signature(checkpoint)
|
|
86
|
+
case checkpoint.signed_by(keyring)
|
|
87
|
+
when :key_not_configured
|
|
88
|
+
flag(:key_not_configured, checkpoint, "signed by key #{checkpoint.key_id}",
|
|
89
|
+
key_id: checkpoint.key_id)
|
|
90
|
+
when :altered then flag(:altered, checkpoint, ALTERED)
|
|
91
|
+
end
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
def chain(checkpoint, record, chained)
|
|
95
|
+
return flag(:not_reached, checkpoint, format(NO_CHAIN, reach(checkpoint))) unless chained
|
|
96
|
+
return flag(:mismatch, checkpoint, NO_RECORD) if record.nil?
|
|
97
|
+
return if same_genesis?(checkpoint, record)
|
|
98
|
+
|
|
99
|
+
flag(:mismatch, checkpoint, OTHER_CHAIN)
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
def same_genesis?(checkpoint, record)
|
|
103
|
+
checkpoint.genesis_digest == record.genesis_digest &&
|
|
104
|
+
checkpoint.pre_chain_count == record.pre_chain_count &&
|
|
105
|
+
checkpoint.pre_chain_unchained_count == record.pre_chain_unchained_count
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
def epoch(checkpoint, epochs)
|
|
109
|
+
number = checkpoint.epoch
|
|
110
|
+
return if number.nil? || epochs[number]&.epoch_digest == checkpoint.epoch_digest
|
|
111
|
+
|
|
112
|
+
flag(:mismatch, checkpoint, format(EPOCH, number, epochs[number] ? CHANGED : GONE))
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
def position(checkpoint, head, digests)
|
|
116
|
+
sequence = checkpoint.head_sequence
|
|
117
|
+
head_sequence = head&.head_sequence.to_i
|
|
118
|
+
if sequence > head_sequence
|
|
119
|
+
return flag(:not_reached, checkpoint, format(REMOVED, Report.number(head_sequence)))
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
found = digests.fetch(sequence, [])
|
|
123
|
+
return flag(:not_reached, checkpoint, "entry #{reach(checkpoint)} is gone") if found.none?
|
|
124
|
+
return if found.include?(checkpoint.head_digest)
|
|
125
|
+
|
|
126
|
+
flag(:mismatch, checkpoint, "entry #{reach(checkpoint)} is not the one it saw")
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
def reach(checkpoint)
|
|
130
|
+
"##{Report.number(checkpoint.head_sequence)}"
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
# Reports the break that goes with `status` and returns the status.
|
|
134
|
+
def flag(status, checkpoint, detail, key_id: nil)
|
|
135
|
+
@report.call(KINDS.fetch(status), nil, detail, nil, key_id,
|
|
136
|
+
checkpoint: checkpoint.head_sequence)
|
|
137
|
+
status
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
# Every digest at each sequence, a batch of sequences per query (FR-016). A list, because
|
|
141
|
+
# a duplicated sequence is a break of its own and must not hide the entry the
|
|
142
|
+
# checkpoint saw.
|
|
143
|
+
def digests_at(sequences)
|
|
144
|
+
found = Hash.new { |hash, sequence| hash[sequence] = [] }
|
|
145
|
+
sequences.uniq.each_slice(LOOKUP_BATCH) do |slice|
|
|
146
|
+
rows = entry_class.uncached do
|
|
147
|
+
entry_class.where(chain_sequence: slice).pluck(:chain_sequence, :chain_digest)
|
|
148
|
+
end
|
|
149
|
+
rows.each { |sequence, digest| found[sequence] << digest }
|
|
150
|
+
end
|
|
151
|
+
found
|
|
152
|
+
end
|
|
153
|
+
end
|
|
154
|
+
end
|
|
155
|
+
end
|
|
156
|
+
end
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Reeve
|
|
4
|
+
module Audit
|
|
5
|
+
class Chain
|
|
6
|
+
# Chain.checkpoint! (specs/004), Chain.purge! and Chain.verify_archive (specs/005), kept
|
|
7
|
+
# beside the code they enter rather than in the class that appends.
|
|
8
|
+
module Entrypoints
|
|
9
|
+
# Makes a checkpoint of where the chain has reached and hands it to `anchor`; returns a
|
|
10
|
+
# CheckpointResult. `bin/rails reeve:checkpoint_ledger` runs it.
|
|
11
|
+
def checkpoint!(entry_class: Entry, anchor: Reeve.config.ledger_anchor)
|
|
12
|
+
CheckpointResult.make!(entry_class: entry_class, anchor: anchor)
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
# Purges the chain's oldest entries up to a checkpoint and returns a PurgeResult.
|
|
16
|
+
# `bin/rails reeve:purge_ledger` runs it.
|
|
17
|
+
def purge!(entry_class: Entry, **options)
|
|
18
|
+
Retention::Purge.new(entry_class: entry_class, **options).call
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
# Verifies an archive a purge wrote, on its own. With `keys:`, with those keys alone.
|
|
22
|
+
def verify_archive(path, keys: nil, entry_class: Entry)
|
|
23
|
+
Retention::ArchiveCheck.new(path, keyring: Keyring.verifying(keys),
|
|
24
|
+
entry_class: entry_class).call
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
end
|
|
@@ -44,15 +44,21 @@ module Reeve
|
|
|
44
44
|
# latest epoch's digest, or to the genesis for epoch 0, so the chain must have
|
|
45
45
|
# started.
|
|
46
46
|
def open!(signer, first_sequence:)
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
epoch = { epoch: latest ? Integer(latest["epoch"]) + 1 : 0, key_id: signer.signing_id,
|
|
47
|
+
last = latest
|
|
48
|
+
epoch = { epoch: last ? last[:epoch] + 1 : 0, key_id: signer.signing_id,
|
|
50
49
|
first_sequence: first_sequence, opened_at: Time.now.utc.floor(6) }
|
|
51
|
-
previous =
|
|
50
|
+
previous = last ? last[:epoch_digest] : genesis_digest
|
|
52
51
|
digest = Canonical.epoch_digest(signer.signing_secret, epoch, previous_digest: previous)
|
|
53
52
|
epoch.merge(epoch_digest: digest).tap { |values| insert(values) }
|
|
54
53
|
end
|
|
55
54
|
|
|
55
|
+
# The newest epoch's number and digest, or nil when no key has opened one.
|
|
56
|
+
def latest
|
|
57
|
+
sql = "SELECT epoch, epoch_digest FROM #{EPOCHS_TABLE} ORDER BY epoch DESC LIMIT 1"
|
|
58
|
+
row = connection.select_one(current(sql))
|
|
59
|
+
row && { epoch: Integer(row["epoch"]), epoch_digest: row["epoch_digest"] }
|
|
60
|
+
end
|
|
61
|
+
|
|
56
62
|
private
|
|
57
63
|
|
|
58
64
|
attr_reader :connection
|
|
@@ -46,7 +46,7 @@ module Reeve
|
|
|
46
46
|
# So a started chain can exist with nothing in it yet (head 0) — if the first chained
|
|
47
47
|
# write then fails, say. The verifier reports that as no chain, not as a pass.
|
|
48
48
|
def start!(signer)
|
|
49
|
-
return if
|
|
49
|
+
return if started?
|
|
50
50
|
|
|
51
51
|
genesis = genesis_for(signer)
|
|
52
52
|
digest = Canonical.genesis_digest(signer.signing_secret, genesis)
|
|
@@ -86,6 +86,31 @@ module Reeve
|
|
|
86
86
|
Integer(connection.select_value(Head.current(connection, sql)))
|
|
87
87
|
end
|
|
88
88
|
|
|
89
|
+
# A Checkpoint of the head and the latest epoch, read under #lock!, which a write holds
|
|
90
|
+
# while it moves the head and opens its epoch, so the two are one committed state
|
|
91
|
+
# (specs/004 FR-005, research R2). Nil when the chain has not started or holds nothing.
|
|
92
|
+
def checkpoint(signer)
|
|
93
|
+
return nil unless started?
|
|
94
|
+
|
|
95
|
+
entry_class.transaction(requires_new: true) do
|
|
96
|
+
next nil unless lock!.positive?
|
|
97
|
+
|
|
98
|
+
Checkpoint.build(state: state, epoch: Epochs.new(connection: connection).latest,
|
|
99
|
+
keyring: signer)
|
|
100
|
+
end
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
# The chain record's genesis and head as one current read.
|
|
104
|
+
def state
|
|
105
|
+
sql = "SELECT genesis_digest, pre_chain_count, pre_chain_unchained_count, " \
|
|
106
|
+
"head_sequence, head_digest FROM #{TABLE} WHERE #{named}"
|
|
107
|
+
connection.select_one(Head.current(connection, sql))
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
def started?
|
|
111
|
+
!connection.select_value("SELECT 1 FROM #{TABLE} WHERE #{named}").nil?
|
|
112
|
+
end
|
|
113
|
+
|
|
89
114
|
def point_at!(digest)
|
|
90
115
|
connection.update("UPDATE #{TABLE} SET head_digest = #{connection.quote(digest)} " \
|
|
91
116
|
"WHERE #{named}")
|
|
@@ -16,6 +16,12 @@ module Reeve
|
|
|
16
16
|
new(signing: config.ledger_key, retired: config.retired_ledger_keys)
|
|
17
17
|
end
|
|
18
18
|
|
|
19
|
+
# The configured keys, or only `keys` when given: how an auditor verifies with their
|
|
20
|
+
# own copy.
|
|
21
|
+
def self.verifying(keys)
|
|
22
|
+
keys ? new(signing: nil, retired: keys) : from(Reeve.config)
|
|
23
|
+
end
|
|
24
|
+
|
|
19
25
|
def initialize(signing:, retired: [])
|
|
20
26
|
@signing_secret = signing
|
|
21
27
|
@by_id = {}
|