reeve 0.7.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +117 -0
- data/README.md +236 -9
- data/lib/generators/reeve/install/templates/create_audit_entries.rb.tt +29 -4
- data/lib/generators/reeve/install/templates/initializer.rb.tt +10 -0
- data/lib/generators/reeve/upgrade/templates/add_ledger_retention_to_reeve_audit_entries.rb.tt +37 -0
- data/lib/generators/reeve/upgrade/upgrade_generator.rb +10 -1
- data/lib/generators/reeve/upgrade/upgrade_notes.rb +15 -4
- data/lib/reeve/audit/chain/anchoring.rb +91 -0
- data/lib/reeve/audit/chain/canonical.rb +17 -1
- data/lib/reeve/audit/chain/checkpoint.rb +228 -0
- data/lib/reeve/audit/chain/checkpoint_check.rb +156 -0
- data/lib/reeve/audit/chain/entrypoints.rb +29 -0
- data/lib/reeve/audit/chain/epochs.rb +10 -4
- data/lib/reeve/audit/chain/head.rb +26 -1
- data/lib/reeve/audit/chain/keyring.rb +6 -0
- data/lib/reeve/audit/chain/report.rb +188 -31
- data/lib/reeve/audit/chain/retention/archive.rb +98 -0
- data/lib/reeve/audit/chain/retention/archive_check.rb +179 -0
- data/lib/reeve/audit/chain/retention/check.rb +94 -0
- data/lib/reeve/audit/chain/retention/connection.rb +65 -0
- data/lib/reeve/audit/chain/retention/digest.rb +29 -0
- data/lib/reeve/audit/chain/retention/plan.rb +100 -0
- data/lib/reeve/audit/chain/retention/preflight.rb +48 -0
- data/lib/reeve/audit/chain/retention/purge.rb +155 -0
- data/lib/reeve/audit/chain/retention/purge_result.rb +134 -0
- data/lib/reeve/audit/chain/retention/records.rb +23 -0
- data/lib/reeve/audit/chain/verification.rb +31 -9
- data/lib/reeve/audit/chain/verifier.rb +64 -19
- data/lib/reeve/audit/chain/walk.rb +9 -3
- data/lib/reeve/audit/chain.rb +12 -3
- data/lib/reeve/audit/chain_record.rb +3 -2
- data/lib/reeve/audit/file_anchor.rb +49 -0
- data/lib/reeve/audit/retention_record.rb +24 -0
- data/lib/reeve/audit.rb +16 -0
- data/lib/reeve/configuration.rb +25 -0
- data/lib/reeve/errors.rb +15 -0
- data/lib/reeve/railtie.rb +2 -2
- data/lib/reeve/tasks/descriptions.rb +22 -0
- data/lib/reeve/tasks/purge_options.rb +54 -0
- data/lib/reeve/tasks.rb +82 -14
- data/lib/reeve/testing/checks/ledger_chain.rb +56 -2
- data/lib/reeve/testing/ledger.rb +19 -3
- data/lib/reeve/version.rb +1 -1
- metadata +21 -2
|
@@ -25,10 +25,14 @@ module Reeve
|
|
|
25
25
|
RECORD_MISSING = "chained rows exist but the chain record does not"
|
|
26
26
|
UNCHAINED_ROW = "a contract-3+ row outside the chain, written after it started (by clock)"
|
|
27
27
|
|
|
28
|
-
|
|
28
|
+
# `checkpoints` are Checkpoints read from outside the database, and `from` one of them to
|
|
29
|
+
# walk from instead of the genesis (specs/004; Anchoring).
|
|
30
|
+
def initialize(entry_class:, keyring:, batch_size: BATCH_SIZE, checkpoints: [], from: nil)
|
|
29
31
|
@entry_class = entry_class
|
|
30
32
|
@keyring = keyring
|
|
31
33
|
@batch_size = batch_size
|
|
34
|
+
@anchoring = Anchoring.new(entry_class: entry_class, keyring: keyring,
|
|
35
|
+
checkpoints: checkpoints, from: from)
|
|
32
36
|
end
|
|
33
37
|
|
|
34
38
|
def call
|
|
@@ -36,37 +40,70 @@ module Reeve
|
|
|
36
40
|
|
|
37
41
|
ChainRecord.require_same_database!(entry_class)
|
|
38
42
|
|
|
43
|
+
@breaks = []
|
|
39
44
|
record = read_head
|
|
40
|
-
return
|
|
45
|
+
return verify_chain(record) if record&.head_sequence&.positive? || chained_rows?
|
|
46
|
+
return unwalked(:no_chain) unless anchoring.any?
|
|
41
47
|
|
|
42
|
-
|
|
48
|
+
# A ledger with no chain that checkpoints saw: broken, not "no chain" (FR-013).
|
|
49
|
+
unwalked(checkpoints: anchoring.emptied(record, report: method(:add)))
|
|
43
50
|
end
|
|
44
51
|
|
|
45
52
|
private
|
|
46
53
|
|
|
47
|
-
attr_reader :entry_class, :keyring, :batch_size, :breaks
|
|
54
|
+
attr_reader :entry_class, :keyring, :batch_size, :breaks, :anchoring
|
|
48
55
|
|
|
49
56
|
# The epochs are read after the head: every row the walk visits committed, with any
|
|
50
57
|
# epoch it opened, before that read. They are checked against the head read *after*
|
|
51
58
|
# the walk, which is later than they were read, so an epoch opened by a rotation
|
|
52
59
|
# during the run is never mistaken for one placed beyond the head.
|
|
53
60
|
def verify_chain(record)
|
|
54
|
-
@breaks = []
|
|
55
61
|
epochs = EpochRules.read(keyring: keyring, entry_class: entry_class)
|
|
62
|
+
retention = check_retention(record)
|
|
63
|
+
failed = anchoring.failed_start(record, epochs.records, report: method(:add))
|
|
64
|
+
return unwalked(record: record, checkpoints: failed) if failed
|
|
65
|
+
|
|
56
66
|
walk = walk(record, epochs)
|
|
67
|
+
head_at_end = check_around(record, walk, epochs)
|
|
68
|
+
anchored = anchoring.check(record, head_at_end || record, epochs.records,
|
|
69
|
+
report: method(:add))
|
|
70
|
+
|
|
71
|
+
result(record, head_at_end, walk.last_sequence, epochs,
|
|
72
|
+
**anchored, retention: retention, awaiting_purge: awaiting_purge)
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
# The purges, if any (specs/005): the latest that holds is where the walk starts.
|
|
76
|
+
def check_retention(record)
|
|
77
|
+
records = Retention::Records.read(entry_class: entry_class)
|
|
78
|
+
return nil if records.empty?
|
|
79
|
+
|
|
80
|
+
checked = Retention::Check.new(keyring: keyring, report: method(:add))
|
|
81
|
+
.call(records, genesis_digest: record&.genesis_digest)
|
|
82
|
+
anchoring.floor = checked.boundary
|
|
83
|
+
checked
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
# Rows still at or below the boundary: a purge stopped midway, or a restore (FR-010).
|
|
87
|
+
def awaiting_purge
|
|
88
|
+
floor = anchoring.floor
|
|
89
|
+
floor ? entry_class.where("chain_sequence <= ?", floor.through_sequence).count : 0
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
# The chain record, rows past the head, unchained rows and the epochs, each checked
|
|
93
|
+
# against what the walk found; returns the head as read after the walk.
|
|
94
|
+
def check_around(record, walk, epochs)
|
|
57
95
|
check_record(record, walk)
|
|
58
96
|
head_at_end = record && read_head
|
|
59
97
|
check_beyond_head(head_at_end || record)
|
|
60
98
|
check_unchained(record)
|
|
61
99
|
epochs.check!(record, head_at_end || record, report: method(:add))
|
|
62
|
-
|
|
63
|
-
result(record, head_at_end, walk.last_sequence, epochs)
|
|
100
|
+
head_at_end
|
|
64
101
|
end
|
|
65
102
|
|
|
66
103
|
def walk(record, epochs)
|
|
67
104
|
Walk.new(entry_class: entry_class, keyring: keyring, batch_size: batch_size,
|
|
68
|
-
link: record
|
|
69
|
-
|
|
105
|
+
link: anchoring.link(record), report: method(:add), epochs: epochs,
|
|
106
|
+
start: anchoring.start).call(record&.head_sequence)
|
|
70
107
|
end
|
|
71
108
|
|
|
72
109
|
# A seam as well as a read: the specs wrap it to land a write mid-walk.
|
|
@@ -127,16 +164,19 @@ module Reeve
|
|
|
127
164
|
end
|
|
128
165
|
end
|
|
129
166
|
|
|
130
|
-
# `epoch` names the key epoch a break is about when it has
|
|
131
|
-
|
|
167
|
+
# `epoch` names the key epoch, and `checkpoint` the checkpoint, a break is about when it has
|
|
168
|
+
# no sequence (specs/003, specs/004).
|
|
169
|
+
def add(kind, sequence, detail = nil, invocation_id = nil, key_id = nil, epoch: nil,
|
|
170
|
+
checkpoint: nil, retention: nil)
|
|
132
171
|
found = { kind: kind, sequence: sequence, detail: detail, invocation_id: invocation_id }
|
|
133
|
-
breaks << Verification::Break.new(**found, key_id: key_id, epoch: epoch
|
|
172
|
+
breaks << Verification::Break.new(**found, key_id: key_id, epoch: epoch,
|
|
173
|
+
checkpoint: checkpoint, retention: retention)
|
|
134
174
|
end
|
|
135
175
|
|
|
136
|
-
def result(record, head_at_end, last_sequence, epochs)
|
|
176
|
+
def result(record, head_at_end, last_sequence, epochs, **found)
|
|
137
177
|
Verification.new(
|
|
138
178
|
status: Verification.status_for(breaks), breaks: breaks,
|
|
139
|
-
covered: covered(record, last_sequence),
|
|
179
|
+
covered: covered(record, last_sequence), walked_from: anchoring.from, **found,
|
|
140
180
|
epochs: epochs.summaries, before_first_epoch: epochs.before_first_epoch,
|
|
141
181
|
head_at_start: record&.head_sequence, head_at_end: head_at_end&.head_sequence,
|
|
142
182
|
uncovered_now: count_uncovered, uncovered_at_genesis: record&.pre_chain_count,
|
|
@@ -146,13 +186,18 @@ module Reeve
|
|
|
146
186
|
|
|
147
187
|
def covered(record, last_sequence)
|
|
148
188
|
last = record ? record.head_sequence : last_sequence
|
|
149
|
-
|
|
189
|
+
first = anchoring.start + 1
|
|
190
|
+
last >= first ? first..last : nil
|
|
150
191
|
end
|
|
151
192
|
|
|
152
|
-
# An answer given before any walk: no key to walk with,
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
193
|
+
# An answer given before any walk: no key to walk with, no chain to walk, or checkpoints
|
|
194
|
+
# whose own breaks say why there was nothing to walk (specs/004). Without a status, the
|
|
195
|
+
# breaks decide it.
|
|
196
|
+
def unwalked(status = nil, reason = nil, record: nil, checkpoints: [])
|
|
197
|
+
Verification.new(status: status || Verification.status_for(breaks), reason: reason,
|
|
198
|
+
breaks: breaks || [], head_at_start: record&.head_sequence,
|
|
199
|
+
checkpoints: checkpoints, uncovered_now: count_uncovered,
|
|
200
|
+
unchained_now: count_unchained)
|
|
156
201
|
end
|
|
157
202
|
|
|
158
203
|
def chained_rows?
|
|
@@ -21,14 +21,19 @@ module Reeve
|
|
|
21
21
|
# `link` is the digest the first row must link to (the genesis), or nil when there
|
|
22
22
|
# is no chain record to take it from. `report` receives (kind, sequence, detail).
|
|
23
23
|
# `epochs` is the EpochRules the rows are checked against (specs/003 FR-011).
|
|
24
|
-
|
|
24
|
+
#
|
|
25
|
+
# `start` is the sequence of a checkpoint the walk begins after (specs/004 FR-015), with
|
|
26
|
+
# `link` its digest: the walk is then exactly the one from the genesis, resumed there.
|
|
27
|
+
def initialize(entry_class:, keyring:, batch_size:, link:, report:, epochs:, start: 0)
|
|
25
28
|
@entry_class = entry_class
|
|
26
29
|
@keyring = keyring
|
|
27
30
|
@batch_size = batch_size
|
|
28
31
|
@report = report
|
|
29
32
|
@epochs = epochs
|
|
30
|
-
@
|
|
31
|
-
@
|
|
33
|
+
@start = start
|
|
34
|
+
@expected = start + 1
|
|
35
|
+
@last_sequence = start
|
|
36
|
+
@last_digest = start.positive? ? link : nil
|
|
32
37
|
@link = link
|
|
33
38
|
@verify_first_link = !link.nil?
|
|
34
39
|
end
|
|
@@ -124,6 +129,7 @@ module Reeve
|
|
|
124
129
|
# against SC-004's 300. SQLite derived the bound on its own, which hid it there.
|
|
125
130
|
def batch_scope(after, limit)
|
|
126
131
|
scope = entry_class.where.not(chain_sequence: nil)
|
|
132
|
+
scope = scope.where("chain_sequence > ?", @start) if @start.positive? && after.nil?
|
|
127
133
|
scope = scope.where("chain_sequence <= ?", limit) if limit
|
|
128
134
|
if after
|
|
129
135
|
scope = scope.where("chain_sequence >= ?", after[0])
|
data/lib/reeve/audit/chain.rb
CHANGED
|
@@ -36,6 +36,8 @@ module Reeve
|
|
|
36
36
|
# stays locked until that transaction ends, so every other guarded call waits behind it.
|
|
37
37
|
# IsolatedRecorder holds it for one insert.
|
|
38
38
|
class Chain
|
|
39
|
+
extend Entrypoints
|
|
40
|
+
|
|
39
41
|
TABLE = "reeve_audit_chains"
|
|
40
42
|
|
|
41
43
|
COLUMNS = %w[chain_sequence chain_key_id chain_digest chain_epoch].freeze
|
|
@@ -55,9 +57,16 @@ module Reeve
|
|
|
55
57
|
|
|
56
58
|
# Verifies the whole ledger. With `keys:`, verifies with those keys alone — how an
|
|
57
59
|
# auditor checks a ledger with their own copy of the key.
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
60
|
+
#
|
|
61
|
+
# `checkpoints:` (specs/004) are checkpoint lines or Checkpoints to check the ledger
|
|
62
|
+
# against; nil means the configured anchor's, and [] none. They are never read from the
|
|
63
|
+
# database. A line that does not parse raises CheckpointError before anything is walked.
|
|
64
|
+
#
|
|
65
|
+
# `from: :latest` walks from the checkpoint with the highest sequence instead of the
|
|
66
|
+
# genesis (FR-015), after checking that checkpoint on its own; a Checkpoint names one.
|
|
67
|
+
def self.verify(keys: nil, entry_class: Entry, checkpoints: nil, from: nil, **options)
|
|
68
|
+
Verifier.new(entry_class: entry_class, keyring: Keyring.verifying(keys), from: from,
|
|
69
|
+
checkpoints: Checkpoint.read_all(checkpoints), **options).call
|
|
61
70
|
end
|
|
62
71
|
|
|
63
72
|
# Makes the configured ledger key the key of the latest epoch (specs/003 FR-006) and
|
|
@@ -50,13 +50,14 @@ module Reeve
|
|
|
50
50
|
# refusing, with the fix, is the honest answer.
|
|
51
51
|
def self.require_same_database!(entry_class)
|
|
52
52
|
ledger_database = entry_class.connection_db_config.configuration_hash
|
|
53
|
-
[self, ChainEpoch].each do |model|
|
|
53
|
+
[self, ChainEpoch, RetentionRecord].each do |model|
|
|
54
54
|
next if model.connection_db_config.configuration_hash == ledger_database
|
|
55
55
|
|
|
56
56
|
short = model.name.split("::").last
|
|
57
57
|
raise ConfigurationError,
|
|
58
58
|
"#{model.name} is connected to a different database than #{entry_class.name}. " \
|
|
59
|
-
"Connect ChainRecord and
|
|
59
|
+
"Connect ChainRecord, ChainEpoch and RetentionRecord the same way as the ledger " \
|
|
60
|
+
"(for example " \
|
|
60
61
|
"with the same connects_to) so the chain is read where it is written; #{short} " \
|
|
61
62
|
"is the one that is not."
|
|
62
63
|
end
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Reeve
|
|
4
|
+
module Audit
|
|
5
|
+
# The anchor reeve ships (specs/004 FR-006a, research R6): a file it only ever appends to,
|
|
6
|
+
# one checkpoint per line.
|
|
7
|
+
#
|
|
8
|
+
# config.ledger_anchor = Reeve::Audit::FileAnchor.new("/mnt/ledger-anchor/production.checkpoints")
|
|
9
|
+
#
|
|
10
|
+
# A building block, not a guarantee. A checkpoint is worth exactly as much as the store's
|
|
11
|
+
# refusal to let the database's users change or delete it, and a file on the database host
|
|
12
|
+
# refuses nothing. Point it at storage those users cannot reach, such as a mount backed by
|
|
13
|
+
# an object-locked bucket or a path a separate shipper collects.
|
|
14
|
+
#
|
|
15
|
+
# Append mode puts every write at the end of the file even with several processes writing,
|
|
16
|
+
# and the lock keeps each line whole; the file is never opened any other way for writing.
|
|
17
|
+
class FileAnchor
|
|
18
|
+
attr_reader :path
|
|
19
|
+
|
|
20
|
+
def initialize(path)
|
|
21
|
+
@path = path.to_s
|
|
22
|
+
freeze
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def publish(line)
|
|
26
|
+
if line.include?("\n")
|
|
27
|
+
raise ArgumentError,
|
|
28
|
+
"a checkpoint is one line; this one has a line break in it"
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
File.open(path, File::WRONLY | File::APPEND | File::CREAT, 0o644) do |file|
|
|
32
|
+
file.flock(File::LOCK_EX)
|
|
33
|
+
file.write("#{line}\n")
|
|
34
|
+
file.fsync
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def checkpoints
|
|
39
|
+
return [] unless File.exist?(path)
|
|
40
|
+
|
|
41
|
+
File.readlines(path, chomp: true).reject { |line| line.strip.empty? }
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
def inspect
|
|
45
|
+
"#<#{self.class.name} #{path}>"
|
|
46
|
+
end
|
|
47
|
+
end
|
|
48
|
+
end
|
|
49
|
+
end
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Reeve
|
|
4
|
+
module Audit
|
|
5
|
+
# One purge (specs/005 data-model "reeve_audit_retentions"): which checkpoint the ledger
|
|
6
|
+
# was purged up to, what was removed, and the archive written, signed and chained to the
|
|
7
|
+
# purge before it.
|
|
8
|
+
#
|
|
9
|
+
# Written by `bin/rails reeve:purge_ledger` through the retention connection, never this
|
|
10
|
+
# model; the model only reads, for verification, and refuses to change one. As with
|
|
11
|
+
# ChainEpoch, that protects against the library, not against raw SQL; the verifier reports
|
|
12
|
+
# a record changed outside reeve.
|
|
13
|
+
class RetentionRecord < ActiveRecord::Base
|
|
14
|
+
self.table_name = "reeve_audit_retentions"
|
|
15
|
+
|
|
16
|
+
before_update { raise ActiveRecord::ReadOnlyRecord, "a retention record is written once" }
|
|
17
|
+
before_destroy { throw :abort }
|
|
18
|
+
|
|
19
|
+
def readonly?
|
|
20
|
+
persisted? || super
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
end
|
data/lib/reeve/audit.rb
CHANGED
|
@@ -115,15 +115,31 @@ require_relative "audit/chain/normalizer"
|
|
|
115
115
|
require_relative "audit/chain/canonical"
|
|
116
116
|
require_relative "audit/chain_record"
|
|
117
117
|
require_relative "audit/chain_epoch"
|
|
118
|
+
require_relative "audit/retention_record"
|
|
118
119
|
require_relative "audit/chain/verification"
|
|
119
120
|
require_relative "audit/chain/report"
|
|
120
121
|
require_relative "audit/chain/head"
|
|
121
122
|
require_relative "audit/chain/epochs"
|
|
122
123
|
require_relative "audit/chain/rotation"
|
|
124
|
+
require_relative "audit/chain/checkpoint"
|
|
123
125
|
require_relative "audit/chain/epoch_rules"
|
|
126
|
+
require_relative "audit/chain/checkpoint_check"
|
|
127
|
+
require_relative "audit/chain/anchoring"
|
|
128
|
+
require_relative "audit/chain/retention/records"
|
|
129
|
+
require_relative "audit/chain/retention/digest"
|
|
130
|
+
require_relative "audit/chain/retention/check"
|
|
131
|
+
require_relative "audit/chain/retention/connection"
|
|
132
|
+
require_relative "audit/chain/retention/purge_result"
|
|
133
|
+
require_relative "audit/chain/retention/plan"
|
|
134
|
+
require_relative "audit/chain/retention/archive"
|
|
135
|
+
require_relative "audit/chain/retention/archive_check"
|
|
136
|
+
require_relative "audit/chain/retention/preflight"
|
|
137
|
+
require_relative "audit/chain/retention/purge"
|
|
138
|
+
require_relative "audit/chain/entrypoints"
|
|
124
139
|
require_relative "audit/chain/schema"
|
|
125
140
|
require_relative "audit/chain/walk"
|
|
126
141
|
require_relative "audit/chain/verifier"
|
|
127
142
|
require_relative "audit/chain"
|
|
143
|
+
require_relative "audit/file_anchor"
|
|
128
144
|
|
|
129
145
|
Reeve::Audit.install!
|
data/lib/reeve/configuration.rb
CHANGED
|
@@ -25,6 +25,7 @@ module Reeve
|
|
|
25
25
|
principal_resolver unguarded_tools audit_failure_mode redact_arguments
|
|
26
26
|
max_recorded_ids policy_adapter default_action audit_recorder logger
|
|
27
27
|
compliance_principals compliance_tools inventory ledger_key retired_ledger_keys
|
|
28
|
+
ledger_anchor ledger_retention_connection
|
|
28
29
|
].freeze
|
|
29
30
|
|
|
30
31
|
# Shown by #to_h and #inspect as fingerprints. A configuration object gets printed —
|
|
@@ -48,6 +49,8 @@ module Reeve
|
|
|
48
49
|
@inventory = nil
|
|
49
50
|
@ledger_key = nil
|
|
50
51
|
@retired_ledger_keys = []
|
|
52
|
+
@ledger_anchor = nil
|
|
53
|
+
@ledger_retention_connection = nil
|
|
51
54
|
end
|
|
52
55
|
|
|
53
56
|
# The key that signs the audit ledger's chain (specs/002). Held in configuration, never
|
|
@@ -203,6 +206,28 @@ module Reeve
|
|
|
203
206
|
recorder.nil? ? nil : require_protocol!(:audit_recorder, recorder, [:record])
|
|
204
207
|
end
|
|
205
208
|
|
|
209
|
+
# Where ledger checkpoints are kept (specs/004): an object with #publish(line) and
|
|
210
|
+
# #checkpoints, such as Reeve::Audit::FileAnchor. It must be storage the database's users
|
|
211
|
+
# cannot change; nil (the default) means checkpoints are made only on request and stored by
|
|
212
|
+
# whoever asked.
|
|
213
|
+
def ledger_anchor=(anchor)
|
|
214
|
+
@ledger_anchor =
|
|
215
|
+
anchor.nil? ? nil : require_protocol!(:ledger_anchor, anchor, %i[publish checkpoints])
|
|
216
|
+
end
|
|
217
|
+
|
|
218
|
+
# The database connection `bin/rails reeve:purge_ledger` deletes through (specs/005): an
|
|
219
|
+
# abstract ActiveRecord class the host connects to the ledger's database under a role of
|
|
220
|
+
# its own, the only one with DELETE on the ledger. Never the application's connection.
|
|
221
|
+
def ledger_retention_connection=(retention)
|
|
222
|
+
@ledger_retention_connection =
|
|
223
|
+
if retention.nil?
|
|
224
|
+
nil
|
|
225
|
+
else
|
|
226
|
+
require_protocol!(:ledger_retention_connection, retention,
|
|
227
|
+
%i[connection_pool connection_db_config])
|
|
228
|
+
end
|
|
229
|
+
end
|
|
230
|
+
|
|
206
231
|
def logger=(logger)
|
|
207
232
|
@logger = logger.nil? ? nil : require_protocol!(:logger, logger, [:warn])
|
|
208
233
|
end
|
data/lib/reeve/errors.rb
CHANGED
|
@@ -85,6 +85,21 @@ module Reeve
|
|
|
85
85
|
end
|
|
86
86
|
end
|
|
87
87
|
|
|
88
|
+
# Raised when a ledger checkpoint cannot be read (specs/004 research R5): a line that is
|
|
89
|
+
# not a checkpoint, or one in a format this version does not know. Not a break — it says
|
|
90
|
+
# nothing about the ledger — but never skipped either: a store whose lines do not read
|
|
91
|
+
# cannot be counted as anchoring anything. `position` is the line's place in its list,
|
|
92
|
+
# from 1, so the operator can find it; the line itself is not repeated.
|
|
93
|
+
class CheckpointError < Error
|
|
94
|
+
attr_reader :position
|
|
95
|
+
|
|
96
|
+
def initialize(reason, position: nil)
|
|
97
|
+
@position = position
|
|
98
|
+
where = position ? "checkpoint #{position}" : "a checkpoint"
|
|
99
|
+
super("#{where} cannot be read: #{reason}")
|
|
100
|
+
end
|
|
101
|
+
end
|
|
102
|
+
|
|
88
103
|
# Raised when the ledger write fails and audit_failure_mode is :fail (FR-012).
|
|
89
104
|
#
|
|
90
105
|
# It carries a rule like any other denial, because the failure is not recorded anywhere
|
data/lib/reeve/railtie.rb
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
module Reeve
|
|
4
|
-
# Registers `bin/rails reeve:verify_ledger
|
|
5
|
-
# does nothing else.
|
|
4
|
+
# Registers the ledger tasks (`bin/rails reeve:verify_ledger`, `reeve:rotate_ledger_key`,
|
|
5
|
+
# `reeve:checkpoint_ledger`, `reeve:purge_ledger`, `reeve:verify_archive`), and does nothing else.
|
|
6
6
|
#
|
|
7
7
|
# Loaded by lib/reeve.rb only when Rails::Railtie is already defined, so `require "reeve"`
|
|
8
8
|
# in a plain Ruby process still loads nothing beyond reeve's own files and its short stdlib
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Reeve
|
|
4
|
+
# The ledger's Rake tasks, as `bin/rails -T` lists them (Reeve::Tasks.define).
|
|
5
|
+
module Tasks
|
|
6
|
+
# Each task's description, and the method of this module it runs.
|
|
7
|
+
DESCRIPTIONS = {
|
|
8
|
+
verify_ledger: "Verify the audit ledger's chain " \
|
|
9
|
+
"(exit 0 intact, 1 broken, 2 could not verify; LIMIT=, FORMAT=json)",
|
|
10
|
+
checkpoint_ledger: "Checkpoint the audit ledger's chain to config.ledger_anchor " \
|
|
11
|
+
"(exit 0 made, 1 refused or the anchor failed)",
|
|
12
|
+
purge_ledger: "Purge the audit ledger's oldest entries up to a checkpoint (THROUGH= or " \
|
|
13
|
+
"OLDER_THAN=Nd; ARCHIVE=path or NO_ARCHIVE=1; nothing removed without " \
|
|
14
|
+
"CONFIRM=1)",
|
|
15
|
+
verify_archive: "Verify an archive reeve:purge_ledger wrote, on its own " \
|
|
16
|
+
"(ARCHIVE=path; exit 0 intact, 1 broken, 2 could not verify)",
|
|
17
|
+
rotate_ledger_key: "Make config.ledger_key the ledger's current key epoch (exit 0 " \
|
|
18
|
+
"rotated or current, 1 refused; ACCEPT_UNVERIFIABLE=1 if the old key " \
|
|
19
|
+
"is lost)"
|
|
20
|
+
}.freeze
|
|
21
|
+
end
|
|
22
|
+
end
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Reeve
|
|
4
|
+
module Tasks
|
|
5
|
+
# The environment `bin/rails reeve:purge_ledger` reads (specs/005, contracts/retention.md
|
|
6
|
+
# "Rake"), as keyword arguments for Reeve::Audit::Chain.purge!. Anything it does not accept
|
|
7
|
+
# is an ArgumentError naming what it does.
|
|
8
|
+
module PurgeOptions
|
|
9
|
+
module_function
|
|
10
|
+
|
|
11
|
+
def from(env)
|
|
12
|
+
through = env.fetch("THROUGH", "")
|
|
13
|
+
older = env.fetch("OLDER_THAN", "")
|
|
14
|
+
{
|
|
15
|
+
through: through.empty? ? nil : whole_number("THROUGH", through),
|
|
16
|
+
older_than: older.empty? ? nil : days("OLDER_THAN", older),
|
|
17
|
+
archive: env["ARCHIVE"].to_s.empty? ? nil : env["ARCHIVE"],
|
|
18
|
+
no_archive: flag("NO_ARCHIVE", env), confirm: flag("CONFIRM", env)
|
|
19
|
+
}.tap { |options| require_one_archive_choice!(options) }
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def require_one_archive_choice!(options)
|
|
23
|
+
return unless options[:archive] && options[:no_archive]
|
|
24
|
+
|
|
25
|
+
raise ArgumentError, "give ARCHIVE=path or NO_ARCHIVE=1, not both"
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def whole_number(name, raw)
|
|
29
|
+
Integer(raw, 10).tap { |value| raise ArgumentError unless value.positive? }
|
|
30
|
+
rescue ArgumentError
|
|
31
|
+
raise ArgumentError,
|
|
32
|
+
"#{name} must be a checkpoint's sequence (a whole number), got #{raw.inspect}"
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def days(name, raw)
|
|
36
|
+
match = /\A(\d+)d\z/.match(raw)
|
|
37
|
+
unless match
|
|
38
|
+
raise ArgumentError,
|
|
39
|
+
"#{name} must be a number of days such as 365d, got #{raw.inspect}"
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
Integer(match[1], 10)
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def flag(name, env)
|
|
46
|
+
raw = env.fetch(name, "")
|
|
47
|
+
return false if raw.empty?
|
|
48
|
+
return true if raw == "1"
|
|
49
|
+
|
|
50
|
+
raise ArgumentError, "#{name} must be 1 or unset, got #{raw.inspect}"
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
end
|
data/lib/reeve/tasks.rb
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require "json"
|
|
4
|
+
require_relative "tasks/purge_options"
|
|
5
|
+
require_relative "tasks/descriptions"
|
|
4
6
|
|
|
5
7
|
module Reeve
|
|
6
8
|
# `bin/rails reeve:verify_ledger` (specs/002 FR-017, contracts/ledger-chain.md "Rake").
|
|
@@ -15,6 +17,7 @@ module Reeve
|
|
|
15
17
|
EXIT = { intact: 0, broken: 1, unverifiable: 2, no_chain: 2 }.freeze
|
|
16
18
|
FORMATS = %w[text json].freeze
|
|
17
19
|
ROTATE_EXIT = { rotated: 0, current: 0, refused: 1 }.freeze
|
|
20
|
+
PURGE_EXIT = { dry_run: 0, purged: 0, nothing_to_purge: 0, refused: 1 }.freeze
|
|
18
21
|
|
|
19
22
|
module_function
|
|
20
23
|
|
|
@@ -23,6 +26,11 @@ module Reeve
|
|
|
23
26
|
#
|
|
24
27
|
# LIMIT caps the breaks listed in the text report (default 100; the totals are never
|
|
25
28
|
# capped). FORMAT=json writes the whole result, every break, and ignores LIMIT.
|
|
29
|
+
#
|
|
30
|
+
# Checks the ledger against config.ledger_anchor's checkpoints, if one is configured
|
|
31
|
+
# (specs/004). CHECKPOINTS=path reads them from a file of checkpoint lines instead, and
|
|
32
|
+
# CHECKPOINTS= (empty) verifies without any. A checkpoint that cannot be read is exit 2.
|
|
33
|
+
# FROM=latest walks from the highest checkpoint instead of the genesis.
|
|
26
34
|
def verify_ledger(env: ENV, out: $stdout)
|
|
27
35
|
format = env.fetch("FORMAT", "text")
|
|
28
36
|
unless FORMATS.include?(format)
|
|
@@ -31,9 +39,12 @@ module Reeve
|
|
|
31
39
|
|
|
32
40
|
limit = limit_from(env)
|
|
33
41
|
require "reeve/audit"
|
|
34
|
-
result = Reeve::Audit::Chain.verify
|
|
42
|
+
result = Reeve::Audit::Chain.verify(checkpoints: checkpoints_from(env), from: from_env(env))
|
|
35
43
|
out.puts(format == "json" ? JSON.generate(result.as_json) : result.to_s(limit: limit))
|
|
36
44
|
EXIT.fetch(result.status)
|
|
45
|
+
rescue Reeve::CheckpointError => e
|
|
46
|
+
out.puts("reeve ledger: #{e.message}; nothing was verified")
|
|
47
|
+
EXIT.fetch(:unverifiable)
|
|
37
48
|
end
|
|
38
49
|
|
|
39
50
|
# Makes the configured ledger key the latest epoch's key (specs/003, contracts/
|
|
@@ -47,7 +58,71 @@ module Reeve
|
|
|
47
58
|
require "reeve/audit"
|
|
48
59
|
rotation = Reeve::Audit::Chain.rotate_key!(accept_unverifiable: accept)
|
|
49
60
|
out.puts(rotation.to_s)
|
|
50
|
-
ROTATE_EXIT.fetch(rotation.status)
|
|
61
|
+
status = ROTATE_EXIT.fetch(rotation.status)
|
|
62
|
+
return status unless rotation.rotated? && Reeve.config.ledger_anchor
|
|
63
|
+
|
|
64
|
+
checkpoint_after_rotation(out)
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
# Pins the epoch just opened, which no entry names yet (specs/004 FR-008). A failed anchor
|
|
68
|
+
# fails the task but does not undo the rotation: the epoch is open either way.
|
|
69
|
+
def checkpoint_after_rotation(out)
|
|
70
|
+
result = Reeve::Audit::Chain.checkpoint!
|
|
71
|
+
out.puts(result.to_s)
|
|
72
|
+
result.made? && !result.anchor_failed? ? 0 : 1
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
# Makes a checkpoint of the ledger's chain and hands it to config.ledger_anchor (specs/004,
|
|
76
|
+
# contracts/anchoring.md), writes what happened and the checkpoint's line to `out`, and
|
|
77
|
+
# returns the exit status: 0 made (and published, if there is an anchor), 1 refused or the
|
|
78
|
+
# anchor failed. The line is printed either way, so a failed store does not lose it.
|
|
79
|
+
def checkpoint_ledger(env: ENV, out: $stdout)
|
|
80
|
+
require "reeve/audit"
|
|
81
|
+
result = Reeve::Audit::Chain.checkpoint!
|
|
82
|
+
out.puts(result.to_s)
|
|
83
|
+
result.made? && !result.anchor_failed? ? 0 : 1
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
def from_env(env)
|
|
87
|
+
raw = env.fetch("FROM", "")
|
|
88
|
+
return nil if raw.empty?
|
|
89
|
+
return :latest if raw == "latest"
|
|
90
|
+
|
|
91
|
+
raise ArgumentError, "FROM must be latest or unset, got #{raw.inspect}"
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
# Purges the ledger's oldest entries up to a checkpoint (specs/005, contracts/retention.md),
|
|
95
|
+
# writes what it did to `out`, and returns the exit status: 0 for a dry run, a purge, or
|
|
96
|
+
# nothing to purge; 1 refused. Nothing is removed without CONFIRM=1.
|
|
97
|
+
def purge_ledger(env: ENV, out: $stdout)
|
|
98
|
+
options = PurgeOptions.from(env)
|
|
99
|
+
require "reeve/audit"
|
|
100
|
+
result = Reeve::Audit::Chain.purge!(**options)
|
|
101
|
+
out.puts(result.to_s)
|
|
102
|
+
PURGE_EXIT.fetch(result.status)
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
# Verifies an archive a purge wrote, on its own (specs/005 FR-012); exit codes as
|
|
106
|
+
# verify_ledger's. ARCHIVE=path names it.
|
|
107
|
+
def verify_archive(env: ENV, out: $stdout)
|
|
108
|
+
path = env.fetch("ARCHIVE", "")
|
|
109
|
+
raise ArgumentError, "ARCHIVE must name an archive file" if path.empty?
|
|
110
|
+
|
|
111
|
+
require "reeve/audit"
|
|
112
|
+
result = Reeve::Audit::Chain.verify_archive(path)
|
|
113
|
+
out.puts(result.to_s)
|
|
114
|
+
EXIT.fetch(result.status)
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
# nil (the configured anchor's) unless CHECKPOINTS is set.
|
|
118
|
+
def checkpoints_from(env)
|
|
119
|
+
return nil unless env.key?("CHECKPOINTS")
|
|
120
|
+
|
|
121
|
+
path = env["CHECKPOINTS"].to_s
|
|
122
|
+
return [] if path.empty?
|
|
123
|
+
raise Reeve::CheckpointError, "no checkpoint file at #{path}" unless File.file?(path)
|
|
124
|
+
|
|
125
|
+
Reeve::Audit::FileAnchor.new(path).checkpoints
|
|
51
126
|
end
|
|
52
127
|
|
|
53
128
|
def accept_unverifiable_from(env)
|
|
@@ -65,24 +140,17 @@ module Reeve
|
|
|
65
140
|
raise ArgumentError, "LIMIT must be a whole number of breaks to list, got #{raw.inspect}"
|
|
66
141
|
end
|
|
67
142
|
|
|
68
|
-
# The Rake
|
|
69
|
-
# the host's initializer — and with it the ledger key — is loaded first.
|
|
143
|
+
# The Rake tasks, defined on the current Rake application. Each depends on :environment,
|
|
144
|
+
# so the host's initializer — and with it the ledger key — is loaded first.
|
|
70
145
|
#
|
|
71
146
|
# The DSL is borrowed by a throwaway object, so Reeve::Tasks itself never gains `task`,
|
|
72
147
|
# `sh` and the rest of Rake's methods.
|
|
73
148
|
def define
|
|
74
149
|
Object.new.extend(Rake::DSL).instance_exec do
|
|
75
150
|
namespace :reeve do
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
exit Reeve::Tasks.verify_ledger
|
|
80
|
-
end
|
|
81
|
-
|
|
82
|
-
desc "Make config.ledger_key the ledger's current key epoch (exit 0 rotated or " \
|
|
83
|
-
"current, 1 refused; ACCEPT_UNVERIFIABLE=1 if the old key is lost)"
|
|
84
|
-
task rotate_ledger_key: :environment do
|
|
85
|
-
exit Reeve::Tasks.rotate_ledger_key
|
|
151
|
+
DESCRIPTIONS.each do |name, description|
|
|
152
|
+
desc description
|
|
153
|
+
task(name => :environment) { exit Reeve::Tasks.public_send(name) }
|
|
86
154
|
end
|
|
87
155
|
end
|
|
88
156
|
end
|