reeve 0.6.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +293 -3
- data/README.md +276 -11
- data/lib/generators/reeve/install/templates/create_audit_entries.rb.tt +52 -5
- data/lib/generators/reeve/install/templates/initializer.rb.tt +10 -0
- data/lib/generators/reeve/upgrade/templates/add_key_epochs_to_reeve_audit_entries.rb.tt +35 -0
- data/lib/generators/reeve/upgrade/templates/add_ledger_retention_to_reeve_audit_entries.rb.tt +37 -0
- data/lib/generators/reeve/upgrade/upgrade_generator.rb +19 -25
- data/lib/generators/reeve/upgrade/upgrade_notes.rb +62 -0
- data/lib/reeve/audit/chain/anchoring.rb +91 -0
- data/lib/reeve/audit/chain/canonical.rb +38 -7
- data/lib/reeve/audit/chain/checkpoint.rb +228 -0
- data/lib/reeve/audit/chain/checkpoint_check.rb +156 -0
- data/lib/reeve/audit/chain/entrypoints.rb +29 -0
- data/lib/reeve/audit/chain/epoch_rules.rb +159 -0
- data/lib/reeve/audit/chain/epochs.rb +83 -0
- data/lib/reeve/audit/chain/head.rb +55 -2
- data/lib/reeve/audit/chain/keyring.rb +6 -0
- data/lib/reeve/audit/chain/report.rb +232 -32
- data/lib/reeve/audit/chain/retention/archive.rb +98 -0
- data/lib/reeve/audit/chain/retention/archive_check.rb +179 -0
- data/lib/reeve/audit/chain/retention/check.rb +94 -0
- data/lib/reeve/audit/chain/retention/connection.rb +65 -0
- data/lib/reeve/audit/chain/retention/digest.rb +29 -0
- data/lib/reeve/audit/chain/retention/plan.rb +100 -0
- data/lib/reeve/audit/chain/retention/preflight.rb +48 -0
- data/lib/reeve/audit/chain/retention/purge.rb +155 -0
- data/lib/reeve/audit/chain/retention/purge_result.rb +134 -0
- data/lib/reeve/audit/chain/retention/records.rb +23 -0
- data/lib/reeve/audit/chain/rotation.rb +124 -0
- data/lib/reeve/audit/chain/schema.rb +59 -0
- data/lib/reeve/audit/chain/verification.rb +53 -10
- data/lib/reeve/audit/chain/verifier.rb +75 -22
- data/lib/reeve/audit/chain/walk.rb +24 -4
- data/lib/reeve/audit/chain.rb +61 -13
- data/lib/reeve/audit/chain_epoch.rb +24 -0
- data/lib/reeve/audit/chain_record.rb +14 -8
- data/lib/reeve/audit/file_anchor.rb +49 -0
- data/lib/reeve/audit/retention_record.rb +24 -0
- data/lib/reeve/audit.rb +30 -3
- data/lib/reeve/configuration.rb +25 -0
- data/lib/reeve/errors.rb +15 -0
- data/lib/reeve/railtie.rb +2 -1
- data/lib/reeve/tasks/descriptions.rb +22 -0
- data/lib/reeve/tasks/purge_options.rb +54 -0
- data/lib/reeve/tasks.rb +104 -7
- data/lib/reeve/testing/checks/contract_version.rb +2 -2
- data/lib/reeve/testing/checks/ledger_chain.rb +56 -2
- data/lib/reeve/testing/ledger.rb +19 -3
- data/lib/reeve/version.rb +1 -1
- metadata +28 -2
|
@@ -0,0 +1,228 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
|
|
5
|
+
module Reeve
|
|
6
|
+
module Audit
|
|
7
|
+
class Chain
|
|
8
|
+
# A signed statement of where the chain reached (specs/004, contracts/checkpoint-v1.md):
|
|
9
|
+
# the genesis, the head, and the latest key epoch, which pins every earlier epoch because
|
|
10
|
+
# epoch digests are chained to each other and to the genesis.
|
|
11
|
+
#
|
|
12
|
+
# Kept outside the database by the host's anchor, so that removing entries and rewinding
|
|
13
|
+
# the head, or emptying the ledger, leaves a witness the database cannot change. It
|
|
14
|
+
# asserts where the chain reached, not that the chain was intact, and its `made_at` is
|
|
15
|
+
# the producing host's clock: nothing in verification depends on it.
|
|
16
|
+
#
|
|
17
|
+
# It holds digests, counts, a key fingerprint, a time and versions — never entry content,
|
|
18
|
+
# never a key — so it can be stored where the ledger's data must not be.
|
|
19
|
+
class Checkpoint
|
|
20
|
+
FORMAT = "reeve-ledger-checkpoint-v1"
|
|
21
|
+
|
|
22
|
+
TIME_FORMAT = "%Y-%m-%dT%H:%M:%S.%6NZ"
|
|
23
|
+
|
|
24
|
+
INTEGER = [Integer].freeze
|
|
25
|
+
STRING = [String].freeze
|
|
26
|
+
OPTIONAL_INTEGER = [Integer, NilClass].freeze
|
|
27
|
+
OPTIONAL_STRING = [String, NilClass].freeze
|
|
28
|
+
|
|
29
|
+
TYPES = {
|
|
30
|
+
"format" => STRING, "ledger" => STRING, "genesis_digest" => STRING,
|
|
31
|
+
"pre_chain_count" => INTEGER, "pre_chain_unchained_count" => INTEGER,
|
|
32
|
+
"head_sequence" => INTEGER, "head_digest" => STRING,
|
|
33
|
+
"epoch" => OPTIONAL_INTEGER, "epoch_digest" => OPTIONAL_STRING,
|
|
34
|
+
"key_id" => STRING, "made_at" => STRING, "reeve_version" => STRING,
|
|
35
|
+
"signature" => STRING
|
|
36
|
+
}.freeze
|
|
37
|
+
|
|
38
|
+
TYPES.each_key { |field| define_method(field) { @values[field] } }
|
|
39
|
+
|
|
40
|
+
# `state` is the chain record's columns as one read returned them; `epoch` the latest
|
|
41
|
+
# epoch's number and digest, or nil for a ledger with none.
|
|
42
|
+
def self.build(state:, epoch:, keyring:)
|
|
43
|
+
values = {
|
|
44
|
+
"format" => FORMAT, "ledger" => ChainRecord::LEDGER,
|
|
45
|
+
"genesis_digest" => state["genesis_digest"],
|
|
46
|
+
"pre_chain_count" => Integer(state["pre_chain_count"]),
|
|
47
|
+
"pre_chain_unchained_count" => Integer(state["pre_chain_unchained_count"]),
|
|
48
|
+
"head_sequence" => Integer(state["head_sequence"]),
|
|
49
|
+
"head_digest" => state["head_digest"],
|
|
50
|
+
"epoch" => epoch && epoch[:epoch], "epoch_digest" => epoch && epoch[:epoch_digest],
|
|
51
|
+
"key_id" => keyring.signing_id, "made_at" => Time.now.utc.strftime(TIME_FORMAT),
|
|
52
|
+
"reeve_version" => Reeve::VERSION
|
|
53
|
+
}
|
|
54
|
+
signature = Canonical.checkpoint_signature(keyring.signing_secret, values)
|
|
55
|
+
new(values.merge("signature" => signature))
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
# `position` is the line's place in its list, from 1, for the error that names it.
|
|
59
|
+
def self.parse(line, position:)
|
|
60
|
+
values = JSON.parse(line.to_s)
|
|
61
|
+
reason = problem(values)
|
|
62
|
+
raise CheckpointError.new(reason, position: position) if reason
|
|
63
|
+
|
|
64
|
+
new(values)
|
|
65
|
+
rescue JSON::ParserError
|
|
66
|
+
raise CheckpointError.new("not JSON", position: position)
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
# Checkpoint lines or Checkpoints, as Checkpoints; nil means the configured anchor's.
|
|
70
|
+
def self.read_all(given)
|
|
71
|
+
given = Reeve.config.ledger_anchor&.checkpoints || [] if given.nil?
|
|
72
|
+
given.each_with_index.map do |checkpoint, index|
|
|
73
|
+
checkpoint.is_a?(Checkpoint) ? checkpoint : parse(checkpoint, position: index + 1)
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
# Where verification walks from (FR-015): nil for the genesis, :latest for the highest
|
|
78
|
+
# of `checkpoints`, or a Checkpoint given as itself.
|
|
79
|
+
def self.start(from, checkpoints)
|
|
80
|
+
return from if from.nil? || from.is_a?(Checkpoint)
|
|
81
|
+
unless from == :latest
|
|
82
|
+
raise ArgumentError,
|
|
83
|
+
"from: must be :latest or a Checkpoint, got #{from.inspect}"
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
checkpoints.max_by(&:head_sequence) ||
|
|
87
|
+
raise(CheckpointError,
|
|
88
|
+
"FROM=latest needs a checkpoint to start from, and none was given")
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
def self.problem(values)
|
|
92
|
+
return "not a JSON object" unless values.is_a?(Hash)
|
|
93
|
+
|
|
94
|
+
shape_problem(values) || value_problem(values)
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
def self.shape_problem(values)
|
|
98
|
+
missing = TYPES.keys - values.keys
|
|
99
|
+
extra = values.keys - TYPES.keys
|
|
100
|
+
return "missing #{missing.join(', ')}" if missing.any?
|
|
101
|
+
return "unexpected #{extra.join(', ')}" if extra.any?
|
|
102
|
+
|
|
103
|
+
"format #{values['format'].inspect} is not one this version reads (#{FORMAT})" unless
|
|
104
|
+
values["format"] == FORMAT
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
def self.value_problem(values)
|
|
108
|
+
wrong = TYPES.find { |field, types| types.none? { |type| values[field].is_a?(type) } }
|
|
109
|
+
if wrong
|
|
110
|
+
return "#{wrong[0]} is not #{wrong[1].first == Integer ? 'a whole number' : 'a string'}"
|
|
111
|
+
end
|
|
112
|
+
return "epoch and epoch_digest must both be set or both be null" if
|
|
113
|
+
values["epoch"].nil? != values["epoch_digest"].nil?
|
|
114
|
+
|
|
115
|
+
"head_sequence must be at least 1" unless values["head_sequence"].positive?
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
private_class_method :problem, :shape_problem, :value_problem
|
|
119
|
+
|
|
120
|
+
def initialize(values)
|
|
121
|
+
@values = values.dup.freeze
|
|
122
|
+
freeze
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
# One line of JSON, keys sorted (contracts/checkpoint-v1.md).
|
|
126
|
+
def to_line
|
|
127
|
+
JSON.generate(@values.sort.to_h)
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
# :valid, :altered, or :key_not_configured when `keyring` lacks the key that signed it.
|
|
131
|
+
def signed_by(keyring)
|
|
132
|
+
secret = keyring.for(key_id)
|
|
133
|
+
return :key_not_configured if secret.nil?
|
|
134
|
+
|
|
135
|
+
Canonical.checkpoint_signature(secret, @values) == signature ? :valid : :altered
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
def as_json(*)
|
|
139
|
+
{ "head_sequence" => head_sequence, "made_at" => made_at }
|
|
140
|
+
end
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
# What `bin/rails reeve:checkpoint_ledger` did (specs/004 data-model "Checkpoint result").
|
|
144
|
+
#
|
|
145
|
+
# :made — `checkpoint` was made; `published` says whether the anchor took it, and
|
|
146
|
+
# `error` why it did not, when one is configured
|
|
147
|
+
# :refused — nothing made; `reason` is :no_key or :no_chain
|
|
148
|
+
class CheckpointResult
|
|
149
|
+
PREFIX = "reeve ledger checkpoint:"
|
|
150
|
+
|
|
151
|
+
attr_reader :status, :reason, :checkpoint, :error
|
|
152
|
+
|
|
153
|
+
def self.refused(reason)
|
|
154
|
+
new(status: :refused, reason: reason)
|
|
155
|
+
end
|
|
156
|
+
|
|
157
|
+
# Makes a checkpoint on the ledger's own connection and hands it to `anchor`. The
|
|
158
|
+
# reading happens under the head's lock (Head#checkpoint); publishing after it is
|
|
159
|
+
# released, so a slow store never holds a writer (FR-009).
|
|
160
|
+
def self.make!(entry_class:, anchor:, keyring: Keyring.from(Reeve.config))
|
|
161
|
+
return refused(:no_key) unless keyring.signing?
|
|
162
|
+
|
|
163
|
+
Schema.new(entry_class).require_columns!
|
|
164
|
+
checkpoint = entry_class.connection_pool.with_connection do |connection|
|
|
165
|
+
Head.new(connection: connection, entry_class: entry_class).checkpoint(keyring)
|
|
166
|
+
end
|
|
167
|
+
checkpoint ? publish(checkpoint, anchor) : refused(:no_chain)
|
|
168
|
+
end
|
|
169
|
+
|
|
170
|
+
# Hands the line to `anchor`, if there is one. A failing anchor is reported, not raised:
|
|
171
|
+
# the checkpoint was made, and its line can still be stored by hand.
|
|
172
|
+
def self.publish(checkpoint, anchor)
|
|
173
|
+
return new(status: :made, checkpoint: checkpoint) if anchor.nil?
|
|
174
|
+
|
|
175
|
+
anchor.publish(checkpoint.to_line)
|
|
176
|
+
new(status: :made, checkpoint: checkpoint, published: true)
|
|
177
|
+
rescue StandardError => e
|
|
178
|
+
new(status: :made, checkpoint: checkpoint, error: "#{e.message} (#{e.class})")
|
|
179
|
+
end
|
|
180
|
+
|
|
181
|
+
def initialize(status:, reason: nil, checkpoint: nil, published: false, error: nil)
|
|
182
|
+
@status = status
|
|
183
|
+
@reason = reason
|
|
184
|
+
@checkpoint = checkpoint
|
|
185
|
+
@published = published
|
|
186
|
+
@error = error
|
|
187
|
+
freeze
|
|
188
|
+
end
|
|
189
|
+
|
|
190
|
+
def made? = status == :made
|
|
191
|
+
def refused? = status == :refused
|
|
192
|
+
def published? = @published
|
|
193
|
+
def anchor_failed? = !error.nil?
|
|
194
|
+
|
|
195
|
+
def to_s
|
|
196
|
+
return "#{PREFIX} REFUSED — #{refusal}; nothing was made" if refused?
|
|
197
|
+
|
|
198
|
+
[first_line, checkpoint.to_line].join("\n")
|
|
199
|
+
end
|
|
200
|
+
|
|
201
|
+
private
|
|
202
|
+
|
|
203
|
+
def first_line
|
|
204
|
+
"#{PREFIX} MADE — head ##{Report.number(checkpoint.head_sequence)}#{epoch_text}, key " \
|
|
205
|
+
"#{checkpoint.key_id}; #{where}"
|
|
206
|
+
end
|
|
207
|
+
|
|
208
|
+
def epoch_text
|
|
209
|
+
checkpoint.epoch.nil? ? "" : ", epoch #{checkpoint.epoch}"
|
|
210
|
+
end
|
|
211
|
+
|
|
212
|
+
def where
|
|
213
|
+
return "published to the anchor" if published?
|
|
214
|
+
return "the anchor failed (#{error}), store the line below yourself" if anchor_failed?
|
|
215
|
+
|
|
216
|
+
"no anchor configured, store the line below yourself"
|
|
217
|
+
end
|
|
218
|
+
|
|
219
|
+
def refusal
|
|
220
|
+
case reason
|
|
221
|
+
when :no_key then "no ledger key configured (config.ledger_key)"
|
|
222
|
+
else "the ledger has no chain yet; nothing to checkpoint"
|
|
223
|
+
end
|
|
224
|
+
end
|
|
225
|
+
end
|
|
226
|
+
end
|
|
227
|
+
end
|
|
228
|
+
end
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Reeve
|
|
4
|
+
module Audit
|
|
5
|
+
class Chain
|
|
6
|
+
# Checks each checkpoint against the ledger (specs/004 FR-011, FR-012): that it is
|
|
7
|
+
# genuine, that the chain it names is this one, that the epoch it names is still there,
|
|
8
|
+
# and that the ledger still reaches its entry with the digest it names.
|
|
9
|
+
#
|
|
10
|
+
# Each checkpoint is checked on its own; none is compared with another. A rule about
|
|
11
|
+
# their order would need a trusted clock or a trusted order, and nothing supplies either;
|
|
12
|
+
# every case such a rule would catch already fails here (spec amendment 2).
|
|
13
|
+
#
|
|
14
|
+
# A checkpoint signed by a key this verifier lacks is reported as unknown_key and used for
|
|
15
|
+
# nothing else: an unchecked checkpoint must not anchor anything.
|
|
16
|
+
class CheckpointCheck
|
|
17
|
+
LOOKUP_BATCH = 1_000
|
|
18
|
+
|
|
19
|
+
Result = Struct.new(:head_sequence, :made_at, :key_id, :status, keyword_init: true) do
|
|
20
|
+
def as_json(*) = to_h.transform_keys(&:to_s).merge("status" => status.to_s)
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
# The break each failing status is reported as.
|
|
24
|
+
KINDS = { key_not_configured: :unknown_key, altered: :checkpoint_altered,
|
|
25
|
+
mismatch: :checkpoint_mismatch, not_reached: :checkpoint_not_reached }.freeze
|
|
26
|
+
|
|
27
|
+
ALTERED = "does not match its signature; it was edited, or is not reeve's"
|
|
28
|
+
NO_CHAIN = "the ledger has no chain, but this checkpoint saw it reach %s"
|
|
29
|
+
NO_RECORD = "its genesis cannot be checked: the chain record is missing"
|
|
30
|
+
OTHER_CHAIN = "names another chain: its genesis is not this ledger's"
|
|
31
|
+
EPOCH = "names epoch %d, %s"
|
|
32
|
+
CHANGED = "whose record is not the one it saw"
|
|
33
|
+
GONE = "which the ledger no longer has"
|
|
34
|
+
REMOVED = "the head is at %s; entries the checkpoint saw were removed"
|
|
35
|
+
BOUNDARY = "is at the purge boundary, but names another digest than the retention record"
|
|
36
|
+
|
|
37
|
+
# `report` receives (kind, sequence, detail, invocation_id, key_id, checkpoint:).
|
|
38
|
+
def initialize(entry_class:, keyring:, report:)
|
|
39
|
+
@entry_class = entry_class
|
|
40
|
+
@keyring = keyring
|
|
41
|
+
@report = report
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
# `record` is the chain record read at the start, `head` the one read after the walk
|
|
45
|
+
# (the later read, so an entry written during the walk is not "not reached"); `epochs`
|
|
46
|
+
# the epoch records the verifier loaded. `chained` says whether the ledger has a chain
|
|
47
|
+
# at all. Returns one Result per checkpoint, in the order given.
|
|
48
|
+
#
|
|
49
|
+
# `floor` is the latest retention record (specs/005): a checkpoint below it saw entries
|
|
50
|
+
# a purge removed, and is `purged`; the one at it is checked against the record.
|
|
51
|
+
def call(checkpoints, record:, head:, epochs:, chained: true, floor: nil)
|
|
52
|
+
genuine = checkpoints.select { |checkpoint| checkpoint.signed_by(keyring) == :valid }
|
|
53
|
+
digests = chained ? digests_at(genuine.map(&:head_sequence)) : {}
|
|
54
|
+
context = { record: record, head: head, epochs: epochs.to_h { |e| [e.epoch, e] },
|
|
55
|
+
digests: digests, chained: chained, floor: floor }
|
|
56
|
+
checkpoints.map do |checkpoint|
|
|
57
|
+
Result.new(head_sequence: checkpoint.head_sequence, made_at: checkpoint.made_at,
|
|
58
|
+
key_id: checkpoint.key_id, status: judge(checkpoint, **context))
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
private
|
|
63
|
+
|
|
64
|
+
attr_reader :entry_class, :keyring
|
|
65
|
+
|
|
66
|
+
def judge(checkpoint, record:, head:, epochs:, digests:, chained:, floor:)
|
|
67
|
+
signature(checkpoint) ||
|
|
68
|
+
chain(checkpoint, record, chained) ||
|
|
69
|
+
epoch(checkpoint, epochs) ||
|
|
70
|
+
purged(checkpoint, floor) ||
|
|
71
|
+
position(checkpoint, head, digests) ||
|
|
72
|
+
:checked
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
# At or below the latest purge's boundary (specs/005 amendment 4).
|
|
76
|
+
def purged(checkpoint, floor)
|
|
77
|
+
return if floor.nil? || checkpoint.head_sequence > floor.through_sequence
|
|
78
|
+
return :purged if checkpoint.head_sequence < floor.through_sequence
|
|
79
|
+
|
|
80
|
+
return :checked if checkpoint.head_digest == floor.through_digest
|
|
81
|
+
|
|
82
|
+
flag(:mismatch, checkpoint, BOUNDARY)
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
def signature(checkpoint)
|
|
86
|
+
case checkpoint.signed_by(keyring)
|
|
87
|
+
when :key_not_configured
|
|
88
|
+
flag(:key_not_configured, checkpoint, "signed by key #{checkpoint.key_id}",
|
|
89
|
+
key_id: checkpoint.key_id)
|
|
90
|
+
when :altered then flag(:altered, checkpoint, ALTERED)
|
|
91
|
+
end
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
def chain(checkpoint, record, chained)
|
|
95
|
+
return flag(:not_reached, checkpoint, format(NO_CHAIN, reach(checkpoint))) unless chained
|
|
96
|
+
return flag(:mismatch, checkpoint, NO_RECORD) if record.nil?
|
|
97
|
+
return if same_genesis?(checkpoint, record)
|
|
98
|
+
|
|
99
|
+
flag(:mismatch, checkpoint, OTHER_CHAIN)
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
def same_genesis?(checkpoint, record)
|
|
103
|
+
checkpoint.genesis_digest == record.genesis_digest &&
|
|
104
|
+
checkpoint.pre_chain_count == record.pre_chain_count &&
|
|
105
|
+
checkpoint.pre_chain_unchained_count == record.pre_chain_unchained_count
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
def epoch(checkpoint, epochs)
|
|
109
|
+
number = checkpoint.epoch
|
|
110
|
+
return if number.nil? || epochs[number]&.epoch_digest == checkpoint.epoch_digest
|
|
111
|
+
|
|
112
|
+
flag(:mismatch, checkpoint, format(EPOCH, number, epochs[number] ? CHANGED : GONE))
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
def position(checkpoint, head, digests)
|
|
116
|
+
sequence = checkpoint.head_sequence
|
|
117
|
+
head_sequence = head&.head_sequence.to_i
|
|
118
|
+
if sequence > head_sequence
|
|
119
|
+
return flag(:not_reached, checkpoint, format(REMOVED, Report.number(head_sequence)))
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
found = digests.fetch(sequence, [])
|
|
123
|
+
return flag(:not_reached, checkpoint, "entry #{reach(checkpoint)} is gone") if found.none?
|
|
124
|
+
return if found.include?(checkpoint.head_digest)
|
|
125
|
+
|
|
126
|
+
flag(:mismatch, checkpoint, "entry #{reach(checkpoint)} is not the one it saw")
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
def reach(checkpoint)
|
|
130
|
+
"##{Report.number(checkpoint.head_sequence)}"
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
# Reports the break that goes with `status` and returns the status.
|
|
134
|
+
def flag(status, checkpoint, detail, key_id: nil)
|
|
135
|
+
@report.call(KINDS.fetch(status), nil, detail, nil, key_id,
|
|
136
|
+
checkpoint: checkpoint.head_sequence)
|
|
137
|
+
status
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
# Every digest at each sequence, a batch of sequences per query (FR-016). A list, because
|
|
141
|
+
# a duplicated sequence is a break of its own and must not hide the entry the
|
|
142
|
+
# checkpoint saw.
|
|
143
|
+
def digests_at(sequences)
|
|
144
|
+
found = Hash.new { |hash, sequence| hash[sequence] = [] }
|
|
145
|
+
sequences.uniq.each_slice(LOOKUP_BATCH) do |slice|
|
|
146
|
+
rows = entry_class.uncached do
|
|
147
|
+
entry_class.where(chain_sequence: slice).pluck(:chain_sequence, :chain_digest)
|
|
148
|
+
end
|
|
149
|
+
rows.each { |sequence, digest| found[sequence] << digest }
|
|
150
|
+
end
|
|
151
|
+
found
|
|
152
|
+
end
|
|
153
|
+
end
|
|
154
|
+
end
|
|
155
|
+
end
|
|
156
|
+
end
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Reeve
|
|
4
|
+
module Audit
|
|
5
|
+
class Chain
|
|
6
|
+
# Chain.checkpoint! (specs/004), Chain.purge! and Chain.verify_archive (specs/005), kept
|
|
7
|
+
# beside the code they enter rather than in the class that appends.
|
|
8
|
+
module Entrypoints
|
|
9
|
+
# Makes a checkpoint of where the chain has reached and hands it to `anchor`; returns a
|
|
10
|
+
# CheckpointResult. `bin/rails reeve:checkpoint_ledger` runs it.
|
|
11
|
+
def checkpoint!(entry_class: Entry, anchor: Reeve.config.ledger_anchor)
|
|
12
|
+
CheckpointResult.make!(entry_class: entry_class, anchor: anchor)
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
# Purges the chain's oldest entries up to a checkpoint and returns a PurgeResult.
|
|
16
|
+
# `bin/rails reeve:purge_ledger` runs it.
|
|
17
|
+
def purge!(entry_class: Entry, **options)
|
|
18
|
+
Retention::Purge.new(entry_class: entry_class, **options).call
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
# Verifies an archive a purge wrote, on its own. With `keys:`, with those keys alone.
|
|
22
|
+
def verify_archive(path, keys: nil, entry_class: Entry)
|
|
23
|
+
Retention::ArchiveCheck.new(path, keyring: Keyring.verifying(keys),
|
|
24
|
+
entry_class: entry_class).call
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
end
|
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Reeve
|
|
4
|
+
module Audit
|
|
5
|
+
class Chain
|
|
6
|
+
# The key epochs as verification sees them (specs/003 FR-011): read once, then asked
|
|
7
|
+
# about every row the walk visits. Counts, as it goes, the rows that named each epoch and
|
|
8
|
+
# the rows chained before the first, for the report.
|
|
9
|
+
#
|
|
10
|
+
# A row at contract 4 names its epoch, and must carry that epoch's key and not precede
|
|
11
|
+
# its start. A contract-3 row names none and is checked exactly as 0.6.0 checked it,
|
|
12
|
+
# wherever it sits: one written by a 0.6.0 server after the first epoch opened may carry
|
|
13
|
+
# any key that server had, and a verifier without that key cannot tell it from a
|
|
14
|
+
# rewritten one — calling it tampered would report a rolling deploy as an attack.
|
|
15
|
+
class EpochRules
|
|
16
|
+
attr_reader :records, :counts, :before_first_epoch, :summaries
|
|
17
|
+
|
|
18
|
+
# Every epoch record, in one query whatever the ledger's size (FR-014). A ledger whose
|
|
19
|
+
# gem was upgraded before its table was migrated has none — and no row that could
|
|
20
|
+
# name one, since writes refuse until it is migrated (FR-018).
|
|
21
|
+
def self.read(keyring:, entry_class:)
|
|
22
|
+
present = entry_class.connection_pool.with_connection do |connection|
|
|
23
|
+
connection.table_exists?(EPOCHS_TABLE)
|
|
24
|
+
end
|
|
25
|
+
new(present ? ChainEpoch.uncached { ChainEpoch.all.to_a } : [], keyring: keyring)
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
# `records` are ChainEpoch rows, in any order.
|
|
29
|
+
def initialize(records, keyring:)
|
|
30
|
+
@records = records.sort_by(&:epoch).freeze
|
|
31
|
+
@keyring = keyring
|
|
32
|
+
@by_number = @records.to_h { |record| [record.epoch, record] }
|
|
33
|
+
@first_start = @records.map(&:first_sequence).min
|
|
34
|
+
@counts = Hash.new(0)
|
|
35
|
+
@before_first_epoch = 0
|
|
36
|
+
@summaries = []
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
# Checks the records themselves once the walk has counted their rows (EpochCheck);
|
|
40
|
+
# `head` is the chain record as read after the walk. Leaves the result in `summaries`.
|
|
41
|
+
def check!(record, head, report:)
|
|
42
|
+
@summaries = EpochCheck.new(keyring: keyring, report: report)
|
|
43
|
+
.call(self, genesis_digest: record&.genesis_digest, head: head)
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
# [kind, detail] for a row that contradicts the epoch records, or nil.
|
|
47
|
+
def judge(entry)
|
|
48
|
+
return epoch_bearing(entry) if entry.contract_version.to_i >= EPOCH_CONTRACT
|
|
49
|
+
|
|
50
|
+
@before_first_epoch += 1 if before_first?(entry.chain_sequence)
|
|
51
|
+
nil
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
private
|
|
55
|
+
|
|
56
|
+
attr_reader :keyring
|
|
57
|
+
|
|
58
|
+
def epoch_bearing(entry)
|
|
59
|
+
number = entry.chain_epoch
|
|
60
|
+
epoch = number && @by_number[number]
|
|
61
|
+
return [:epoch_missing, missing(number)] unless epoch
|
|
62
|
+
|
|
63
|
+
@counts[number] += 1
|
|
64
|
+
if epoch.key_id != entry.chain_key_id
|
|
65
|
+
[:epoch_mismatch,
|
|
66
|
+
"signed by key #{entry.chain_key_id}, but epoch #{number}'s key is #{epoch.key_id}"]
|
|
67
|
+
elsif entry.chain_sequence < epoch.first_sequence
|
|
68
|
+
[:epoch_mismatch,
|
|
69
|
+
"names epoch #{number}, which begins at sequence #{epoch.first_sequence}"]
|
|
70
|
+
end
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
# True for a row chained before any epoch began — and for every row when none has.
|
|
74
|
+
def before_first?(sequence)
|
|
75
|
+
@first_start.nil? || sequence < @first_start
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
def missing(number)
|
|
79
|
+
number.nil? ? "names no key epoch" : "names epoch #{number}, which has no record"
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
# The epoch records themselves (specs/003 FR-010): numbered 0..n, each beginning after
|
|
84
|
+
# the one before (or with it: two rotations with no write between them begin at the same
|
|
85
|
+
# sequence) and no later than one past the head, one key each, and each matching
|
|
86
|
+
# its digest — which links it to the epoch before, or, for epoch 0, to the genesis. A
|
|
87
|
+
# record whose key is not configured cannot be checked, and that is said, never passed
|
|
88
|
+
# over. Returns one Verification::EpochSummary per record; breaks go to `report`, as
|
|
89
|
+
# (kind, nil, detail, nil, key_id, epoch:) — the verifier's own `add`.
|
|
90
|
+
class EpochCheck
|
|
91
|
+
def initialize(keyring:, report:)
|
|
92
|
+
@keyring = keyring
|
|
93
|
+
@report = report
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
# `head` is the chain record as read after the walk — later than the epochs were
|
|
97
|
+
# read, so an epoch a rotation opened mid-run is never taken for one past the head.
|
|
98
|
+
def call(rules, genesis_digest:, head:)
|
|
99
|
+
previous = { digest: genesis_digest, first_sequence: 0, keys: [] }
|
|
100
|
+
expected = 0
|
|
101
|
+
rules.records.map do |epoch|
|
|
102
|
+
(expected...epoch.epoch).each do |gap|
|
|
103
|
+
report(:epoch_missing, gap, "epoch #{gap} has no record")
|
|
104
|
+
end
|
|
105
|
+
expected = epoch.epoch + 1
|
|
106
|
+
summary(epoch, rules.counts[epoch.epoch], check(epoch, previous, head)).tap do
|
|
107
|
+
previous = { digest: epoch.epoch_digest, first_sequence: epoch.first_sequence,
|
|
108
|
+
keys: previous[:keys] + [epoch.key_id] }
|
|
109
|
+
end
|
|
110
|
+
end
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
private
|
|
114
|
+
|
|
115
|
+
attr_reader :keyring
|
|
116
|
+
|
|
117
|
+
# :altered, :key_not_configured or :verified, reporting what makes it so.
|
|
118
|
+
def check(epoch, previous, head)
|
|
119
|
+
problem = out_of_place(epoch, previous, head) || digest_problem(epoch, previous)
|
|
120
|
+
return :altered.tap { report(:epoch_altered, epoch.epoch, problem) } if problem
|
|
121
|
+
return :verified if keyring.for(epoch.key_id)
|
|
122
|
+
|
|
123
|
+
report(:unknown_key, epoch.epoch, "epoch #{epoch.epoch} is signed by key #{epoch.key_id}",
|
|
124
|
+
epoch.key_id)
|
|
125
|
+
:key_not_configured
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
def report(kind, number, detail, key_id = nil)
|
|
129
|
+
@report.call(kind, nil, detail, nil, key_id, epoch: number)
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
def out_of_place(epoch, previous, head)
|
|
133
|
+
name = "epoch #{epoch.epoch}"
|
|
134
|
+
if epoch.epoch.positive? && epoch.first_sequence < previous[:first_sequence]
|
|
135
|
+
"#{name} begins at sequence #{epoch.first_sequence}, before the epoch before it"
|
|
136
|
+
elsif head && epoch.first_sequence > head.head_sequence + 1
|
|
137
|
+
"#{name} begins at sequence #{epoch.first_sequence}, past the head"
|
|
138
|
+
elsif previous[:keys].include?(epoch.key_id)
|
|
139
|
+
"#{name} reuses the key of an earlier epoch"
|
|
140
|
+
end
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
def digest_problem(epoch, previous)
|
|
144
|
+
secret = keyring.for(epoch.key_id)
|
|
145
|
+
return nil if secret.nil? || previous[:digest].nil?
|
|
146
|
+
|
|
147
|
+
expected = Canonical.epoch_digest(secret, epoch, previous_digest: previous[:digest])
|
|
148
|
+
"epoch #{epoch.epoch} does not match its digest" unless expected == epoch.epoch_digest
|
|
149
|
+
end
|
|
150
|
+
|
|
151
|
+
def summary(epoch, count, status)
|
|
152
|
+
Verification::EpochSummary.new(epoch: epoch.epoch, key_id: epoch.key_id,
|
|
153
|
+
first_sequence: epoch.first_sequence, entry_count: count,
|
|
154
|
+
status: status)
|
|
155
|
+
end
|
|
156
|
+
end
|
|
157
|
+
end
|
|
158
|
+
end
|
|
159
|
+
end
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Reeve
|
|
4
|
+
module Audit
|
|
5
|
+
class Chain
|
|
6
|
+
# The key epochs, as a write sees them (specs/003 FR-001–FR-005).
|
|
7
|
+
#
|
|
8
|
+
# One instance per append or rotation, bound to the connection it runs on and used
|
|
9
|
+
# only under the chain record's lock (Head), so finding a key's epoch and opening the
|
|
10
|
+
# next one are a single step as far as any other writer can tell. The epoch record
|
|
11
|
+
# commits or rolls back with the write that opened it.
|
|
12
|
+
#
|
|
13
|
+
# A key opens one epoch, ever. A process signing with a key whose epoch is not the
|
|
14
|
+
# latest — one still on the old key during a rolling deploy — writes under that
|
|
15
|
+
# earlier epoch: a straggler, legitimate, and verified by its own key.
|
|
16
|
+
class Epochs
|
|
17
|
+
def initialize(connection:)
|
|
18
|
+
@connection = connection
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
# The number of the epoch `signer`'s key opened, opening the next one at `sequence`
|
|
22
|
+
# if it has none.
|
|
23
|
+
def for_signing!(signer, sequence:)
|
|
24
|
+
number_for(signer.signing_id) || open!(signer, first_sequence: sequence)[:epoch]
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def number_for(key_id)
|
|
28
|
+
sql = "SELECT epoch FROM #{EPOCHS_TABLE} WHERE key_id = #{connection.quote(key_id)}"
|
|
29
|
+
value = connection.select_value(current(sql))
|
|
30
|
+
value.nil? ? nil : Integer(value)
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
# Every epoch, oldest first, as hashes with symbol keys.
|
|
34
|
+
def all
|
|
35
|
+
sql = "SELECT epoch, key_id, first_sequence FROM #{EPOCHS_TABLE} ORDER BY epoch"
|
|
36
|
+
connection.select_all(current(sql))
|
|
37
|
+
.map do |row|
|
|
38
|
+
{ epoch: Integer(row["epoch"]), key_id: row["key_id"],
|
|
39
|
+
first_sequence: Integer(row["first_sequence"]) }
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
# Writes epoch `max + 1` (or 0) for `signer`'s key and returns it. Linked to the
|
|
44
|
+
# latest epoch's digest, or to the genesis for epoch 0, so the chain must have
|
|
45
|
+
# started.
|
|
46
|
+
def open!(signer, first_sequence:)
|
|
47
|
+
last = latest
|
|
48
|
+
epoch = { epoch: last ? last[:epoch] + 1 : 0, key_id: signer.signing_id,
|
|
49
|
+
first_sequence: first_sequence, opened_at: Time.now.utc.floor(6) }
|
|
50
|
+
previous = last ? last[:epoch_digest] : genesis_digest
|
|
51
|
+
digest = Canonical.epoch_digest(signer.signing_secret, epoch, previous_digest: previous)
|
|
52
|
+
epoch.merge(epoch_digest: digest).tap { |values| insert(values) }
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
# The newest epoch's number and digest, or nil when no key has opened one.
|
|
56
|
+
def latest
|
|
57
|
+
sql = "SELECT epoch, epoch_digest FROM #{EPOCHS_TABLE} ORDER BY epoch DESC LIMIT 1"
|
|
58
|
+
row = connection.select_one(current(sql))
|
|
59
|
+
row && { epoch: Integer(row["epoch"]), epoch_digest: row["epoch_digest"] }
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
private
|
|
63
|
+
|
|
64
|
+
attr_reader :connection
|
|
65
|
+
|
|
66
|
+
def current(sql)
|
|
67
|
+
Head.current(connection, sql)
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def insert(values)
|
|
71
|
+
quoted = values.values.map { |value| connection.quote(value) }
|
|
72
|
+
connection.insert("INSERT INTO #{EPOCHS_TABLE} (#{values.keys.join(', ')}) " \
|
|
73
|
+
"VALUES (#{quoted.join(', ')})")
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
def genesis_digest
|
|
77
|
+
connection.select_value("SELECT genesis_digest FROM #{TABLE} " \
|
|
78
|
+
"WHERE name = #{connection.quote(ChainRecord::LEDGER)}")
|
|
79
|
+
end
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
end
|
|
83
|
+
end
|