reeve 0.5.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +281 -3
- data/README.md +151 -6
- data/lib/generators/reeve/install/templates/create_audit_entries.rb.tt +63 -3
- data/lib/generators/reeve/install/templates/initializer.rb.tt +20 -0
- data/lib/generators/reeve/upgrade/templates/add_key_epochs_to_reeve_audit_entries.rb.tt +35 -0
- data/lib/generators/reeve/upgrade/templates/add_ledger_chain_to_reeve_audit_entries.rb.tt +51 -0
- data/lib/generators/reeve/upgrade/upgrade_generator.rb +25 -2
- data/lib/generators/reeve/upgrade/upgrade_notes.rb +51 -0
- data/lib/reeve/audit/chain/canonical.rb +144 -0
- data/lib/reeve/audit/chain/epoch_rules.rb +159 -0
- data/lib/reeve/audit/chain/epochs.rb +77 -0
- data/lib/reeve/audit/chain/head.rb +122 -0
- data/lib/reeve/audit/chain/keyring.rb +53 -0
- data/lib/reeve/audit/chain/normalizer.rb +88 -0
- data/lib/reeve/audit/chain/report.rb +177 -0
- data/lib/reeve/audit/chain/rotation.rb +124 -0
- data/lib/reeve/audit/chain/schema.rb +59 -0
- data/lib/reeve/audit/chain/verification.rb +131 -0
- data/lib/reeve/audit/chain/verifier.rb +168 -0
- data/lib/reeve/audit/chain/walk.rb +137 -0
- data/lib/reeve/audit/chain.rb +207 -0
- data/lib/reeve/audit/chain_epoch.rb +24 -0
- data/lib/reeve/audit/chain_record.rb +66 -0
- data/lib/reeve/audit/isolated_recorder.rb +5 -0
- data/lib/reeve/audit/recorder.rb +32 -19
- data/lib/reeve/audit.rb +37 -6
- data/lib/reeve/configuration.rb +66 -2
- data/lib/reeve/errors.rb +15 -0
- data/lib/reeve/ledger_key.rb +47 -0
- data/lib/reeve/railtie.rb +16 -0
- data/lib/reeve/tasks.rb +91 -0
- data/lib/reeve/testing/checks/contract_version.rb +2 -1
- data/lib/reeve/testing/checks/ledger_chain.rb +116 -0
- data/lib/reeve/testing/checks.rb +29 -4
- data/lib/reeve/testing/compliance_assertions.rb +3 -2
- data/lib/reeve/testing/compliance_suite.rb +2 -2
- data/lib/reeve/testing/ledger.rb +11 -1
- data/lib/reeve/testing/report.rb +8 -0
- data/lib/reeve/testing.rb +1 -1
- data/lib/reeve/version.rb +1 -1
- data/lib/reeve.rb +2 -0
- metadata +24 -2
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# Audit-entry contract 2 → 3. Generated by `bin/rails generate reeve:upgrade`.
|
|
4
|
+
#
|
|
5
|
+
# Chains every row reeve writes from here on to the one before it with a keyed digest, so a
|
|
6
|
+
# row changed, removed or reordered outside reeve is reported by
|
|
7
|
+
# `bin/rails reeve:verify_ledger`.
|
|
8
|
+
#
|
|
9
|
+
# Rows already in the table are not touched: there is no honest digest to give a row
|
|
10
|
+
# nobody was there to sign, and nothing is backfilled into an append-only table. They are
|
|
11
|
+
# reported as "not covered" from now on — exactly as trustworthy as they were before this
|
|
12
|
+
# migration, and no more. The three columns are nullable for that reason.
|
|
13
|
+
#
|
|
14
|
+
# The chain needs a key, held outside the database. Add this to
|
|
15
|
+
# config/initializers/reeve.rb, or every guarded call fails closed after you migrate:
|
|
16
|
+
#
|
|
17
|
+
# config.ledger_key = Rails.application.key_generator.generate_key("reeve audit ledger", 32)
|
|
18
|
+
#
|
|
19
|
+
# The chain record's head moves on every write, so the role your application connects as
|
|
20
|
+
# needs UPDATE on reeve_audit_chains (the ledger itself stays INSERT, SELECT):
|
|
21
|
+
#
|
|
22
|
+
# GRANT INSERT, SELECT, UPDATE ON reeve_audit_chains TO app_role;
|
|
23
|
+
#
|
|
24
|
+
# A role holding that UPDATE can rewind the head, and a rewound head hides the deletion of
|
|
25
|
+
# the newest rows. Tamper-evidence detects a change after the fact; the grants are what
|
|
26
|
+
# prevent one.
|
|
27
|
+
#
|
|
28
|
+
# On a very large ledger, the unique index below scans the table once. On PostgreSQL that
|
|
29
|
+
# scan blocks writes for its duration; to avoid it, add `disable_ddl_transaction!` to this
|
|
30
|
+
# class and `algorithm: :concurrently` to the add_index call.
|
|
31
|
+
class AddLedgerChainToReeveAuditEntries < ActiveRecord::Migration[7.0]
|
|
32
|
+
def change
|
|
33
|
+
add_column :reeve_audit_entries, :chain_sequence, :bigint
|
|
34
|
+
add_column :reeve_audit_entries, :chain_key_id, :string, limit: 16
|
|
35
|
+
add_column :reeve_audit_entries, :chain_digest, :string, limit: 64
|
|
36
|
+
add_index :reeve_audit_entries, :chain_sequence, unique: true
|
|
37
|
+
|
|
38
|
+
create_table :reeve_audit_chains do |t|
|
|
39
|
+
t.string :name, null: false
|
|
40
|
+
t.datetime :genesis_at, null: false, precision: 6
|
|
41
|
+
t.bigint :pre_chain_count, null: false
|
|
42
|
+
t.bigint :pre_chain_unchained_count, null: false
|
|
43
|
+
t.string :genesis_key_id, null: false, limit: 16
|
|
44
|
+
t.string :genesis_digest, null: false, limit: 64
|
|
45
|
+
t.bigint :head_sequence, null: false
|
|
46
|
+
t.string :head_digest, null: false, limit: 64
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
add_index :reeve_audit_chains, :name, unique: true
|
|
50
|
+
end
|
|
51
|
+
end
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
require "rails/generators"
|
|
4
4
|
require "rails/generators/active_record"
|
|
5
|
+
require_relative "upgrade_notes"
|
|
5
6
|
|
|
6
7
|
module Reeve
|
|
7
8
|
module Generators
|
|
@@ -33,15 +34,35 @@ module Reeve
|
|
|
33
34
|
# contracts/audit-entry.md, which exists so that stays hypothetical: an append-only
|
|
34
35
|
# ledger cannot be backfilled, so a column that is not additive has no honest value
|
|
35
36
|
# to give the rows already written.
|
|
37
|
+
# `creates` names any table a step builds. A step is applied only when every column it
|
|
38
|
+
# adds and every table it creates exist: a ledger with one half and not the other is
|
|
39
|
+
# half-migrated, and calling it current would leave the host with no way forward.
|
|
36
40
|
STEPS = [
|
|
37
41
|
{
|
|
38
42
|
contract: 2,
|
|
39
43
|
adds: %w[contract_version],
|
|
44
|
+
creates: [],
|
|
40
45
|
template: "add_contract_version_to_reeve_audit_entries.rb.tt",
|
|
41
46
|
destination: "add_contract_version_to_reeve_audit_entries.rb"
|
|
47
|
+
},
|
|
48
|
+
{
|
|
49
|
+
contract: 3,
|
|
50
|
+
adds: %w[chain_sequence chain_key_id chain_digest],
|
|
51
|
+
creates: %w[reeve_audit_chains],
|
|
52
|
+
template: "add_ledger_chain_to_reeve_audit_entries.rb.tt",
|
|
53
|
+
destination: "add_ledger_chain_to_reeve_audit_entries.rb"
|
|
54
|
+
},
|
|
55
|
+
{
|
|
56
|
+
contract: 4,
|
|
57
|
+
adds: %w[chain_epoch],
|
|
58
|
+
creates: %w[reeve_audit_chain_epochs],
|
|
59
|
+
template: "add_key_epochs_to_reeve_audit_entries.rb.tt",
|
|
60
|
+
destination: "add_key_epochs_to_reeve_audit_entries.rb"
|
|
42
61
|
}
|
|
43
62
|
].freeze
|
|
44
63
|
|
|
64
|
+
KEY_LINE = UpgradeNotes::KEY_LINE
|
|
65
|
+
|
|
45
66
|
source_root File.expand_path("templates", __dir__)
|
|
46
67
|
|
|
47
68
|
desc "Brings an existing reeve audit ledger up to the current audit-entry contract."
|
|
@@ -93,7 +114,7 @@ module Reeve
|
|
|
93
114
|
end
|
|
94
115
|
|
|
95
116
|
def pending_message
|
|
96
|
-
<<~NEXT
|
|
117
|
+
message = <<~NEXT
|
|
97
118
|
|
|
98
119
|
#{@emitted.size} migration#{'s' if @emitted.size != 1} written, taking the ledger
|
|
99
120
|
to audit-entry contract #{@emitted.last[:contract]}. Run:
|
|
@@ -103,13 +124,15 @@ module Reeve
|
|
|
103
124
|
Existing rows are stamped with the contract they were actually written under,
|
|
104
125
|
not with the current one.
|
|
105
126
|
NEXT
|
|
127
|
+
message + UpgradeNotes.for(@emitted.map { |step| step[:contract] })
|
|
106
128
|
end
|
|
107
129
|
|
|
108
130
|
# Asked of the table rather than of a version the host recorded: the table is the
|
|
109
131
|
# thing being changed, and it is the only party that cannot be out of date about
|
|
110
132
|
# itself.
|
|
111
133
|
def applied?(step)
|
|
112
|
-
(step[:adds] - existing_columns).empty?
|
|
134
|
+
(step[:adds] - existing_columns).empty? &&
|
|
135
|
+
step[:creates].all? { |table| connection.table_exists?(table) }
|
|
113
136
|
end
|
|
114
137
|
|
|
115
138
|
def existing_columns
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Reeve
|
|
4
|
+
module Generators
|
|
5
|
+
# What `reeve:upgrade` tells the host after writing a step whose contract changes more
|
|
6
|
+
# than the table. The host is reading this output at exactly the moment they need it.
|
|
7
|
+
module UpgradeNotes
|
|
8
|
+
KEY_LINE = "config.ledger_key = " \
|
|
9
|
+
'Rails.application.key_generator.generate_key("reeve audit ledger", 32)'
|
|
10
|
+
|
|
11
|
+
# Contract 3 changes what the application must configure, not only the table: without
|
|
12
|
+
# a key, every guarded call fails closed once the migration runs.
|
|
13
|
+
CHAIN = <<~CHAIN.freeze
|
|
14
|
+
|
|
15
|
+
The ledger is now chained (contract 3). Add this line to config/initializers/reeve.rb
|
|
16
|
+
before you deploy, or every guarded call fails closed:
|
|
17
|
+
|
|
18
|
+
#{KEY_LINE}
|
|
19
|
+
|
|
20
|
+
It derives the key from secret_key_base, so rotating secret_key_base rotates the
|
|
21
|
+
ledger key: keep the old derived key in config.retired_ledger_keys, or entries it
|
|
22
|
+
signed become unverifiable. A dedicated key avoids that coupling.
|
|
23
|
+
|
|
24
|
+
Very large ledger on PostgreSQL? The migration builds a unique index that blocks
|
|
25
|
+
writes while it scans; the migration's header shows how to build it with
|
|
26
|
+
algorithm: :concurrently instead.
|
|
27
|
+
CHAIN
|
|
28
|
+
|
|
29
|
+
# Contract 4 changes how a rotation is done, so the host learns the command here.
|
|
30
|
+
EPOCHS = <<~EPOCHS
|
|
31
|
+
|
|
32
|
+
The ledger now records key epochs (contract 4): the first chained write opens
|
|
33
|
+
epoch 0 with the configured ledger key. To rotate the key later, set the new key
|
|
34
|
+
as config.ledger_key, keep the old one in config.retired_ledger_keys, deploy, and run:
|
|
35
|
+
|
|
36
|
+
bin/rails reeve:rotate_ledger_key
|
|
37
|
+
|
|
38
|
+
The application role needs INSERT and SELECT on reeve_audit_chain_epochs.
|
|
39
|
+
EPOCHS
|
|
40
|
+
|
|
41
|
+
BY_CONTRACT = { 3 => CHAIN, 4 => EPOCHS }.freeze
|
|
42
|
+
|
|
43
|
+
module_function
|
|
44
|
+
|
|
45
|
+
# The notes for the contracts just written, oldest first.
|
|
46
|
+
def for(contracts)
|
|
47
|
+
contracts.filter_map { |contract| BY_CONTRACT[contract] }.join
|
|
48
|
+
end
|
|
49
|
+
end
|
|
50
|
+
end
|
|
51
|
+
end
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "bigdecimal"
|
|
4
|
+
require "json"
|
|
5
|
+
require "openssl"
|
|
6
|
+
|
|
7
|
+
module Reeve
|
|
8
|
+
module Audit
|
|
9
|
+
class Chain
|
|
10
|
+
# The exact text a digest is computed over (specs/002 FR-002, FR-003; research R1, R2
|
|
11
|
+
# step 2; data-model "Canonical entry").
|
|
12
|
+
#
|
|
13
|
+
# Computed once when a row is written and again, from what the database returns,
|
|
14
|
+
# when it is verified — so everything here exists to make those two readings produce
|
|
15
|
+
# identical text:
|
|
16
|
+
#
|
|
17
|
+
# * object keys sorted at every depth, so a database that reorders JSON keys
|
|
18
|
+
# (PostgreSQL jsonb, MySQL JSON) changes nothing;
|
|
19
|
+
# * one spelling per number, so `1.0e+20` written and `100000000000000000000` read
|
|
20
|
+
# back are the same number;
|
|
21
|
+
# * times as integer microseconds since the epoch, so neither the time zone nor the
|
|
22
|
+
# driver's Time class matters;
|
|
23
|
+
# * strings, booleans and nil typed, so "1" and 1 never digest alike.
|
|
24
|
+
#
|
|
25
|
+
# The field list is written out rather than read from the table — a check that
|
|
26
|
+
# derives its expectation from the thing it checks checks nothing — and is keyed by
|
|
27
|
+
# contract version, so a future column joins the digest for rows written under that
|
|
28
|
+
# contract only. The tags keep an entry digest and a genesis digest from ever being
|
|
29
|
+
# interchangeable, and let a future canonical form be told apart from this one.
|
|
30
|
+
module Canonical
|
|
31
|
+
ENTRY_TAG = "reeve-ledger-entry-v1"
|
|
32
|
+
GENESIS_TAG = "reeve-ledger-genesis-v1"
|
|
33
|
+
|
|
34
|
+
EPOCH_TAG = "reeve-ledger-epoch-v1"
|
|
35
|
+
|
|
36
|
+
CONTRACT_3_FIELDS = %w[
|
|
37
|
+
chain_sequence chain_key_id contract_version
|
|
38
|
+
invocation_id occurred_at agent_id agent_name principal_type principal_id
|
|
39
|
+
tool_name arguments outcome rule detail record_type record_ids record_count
|
|
40
|
+
truncated derived guard duration_ms metadata
|
|
41
|
+
].freeze
|
|
42
|
+
|
|
43
|
+
# Contract 4 (specs/003) adds the key epoch, next to the key it names.
|
|
44
|
+
ENTRY_FIELDS = {
|
|
45
|
+
3 => CONTRACT_3_FIELDS,
|
|
46
|
+
4 => CONTRACT_3_FIELDS.dup.insert(2, "chain_epoch").freeze
|
|
47
|
+
}.freeze
|
|
48
|
+
|
|
49
|
+
GENESIS_FIELDS = %w[genesis_at pre_chain_count pre_chain_unchained_count
|
|
50
|
+
genesis_key_id].freeze
|
|
51
|
+
|
|
52
|
+
EPOCH_FIELDS = %w[epoch key_id first_sequence opened_at].freeze
|
|
53
|
+
|
|
54
|
+
TIME_FIELDS = %w[occurred_at genesis_at opened_at].freeze
|
|
55
|
+
|
|
56
|
+
module_function
|
|
57
|
+
|
|
58
|
+
def entry_digest(secret, row, previous_digest:)
|
|
59
|
+
hmac(secret, ENTRY_TAG, entry_message_body(row, previous_digest))
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
def genesis_digest(secret, genesis)
|
|
63
|
+
pairs = GENESIS_FIELDS.map { |field| [field, value(genesis, field)] }
|
|
64
|
+
hmac(secret, GENESIS_TAG, dump(pairs))
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
# An epoch record (specs/003 FR-002), linked to the epoch before it — or, for epoch 0,
|
|
68
|
+
# to the genesis — so the epochs are a chain of their own.
|
|
69
|
+
def epoch_digest(secret, epoch, previous_digest:)
|
|
70
|
+
pairs = EPOCH_FIELDS.map { |field| [field, value(epoch, field)] }
|
|
71
|
+
hmac(secret, EPOCH_TAG, dump([["previous_digest", previous_digest], *pairs]))
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
def entry_message_body(row, previous_digest)
|
|
75
|
+
version = value(row, "contract_version")
|
|
76
|
+
fields = ENTRY_FIELDS.fetch(version) do
|
|
77
|
+
raise ArgumentError,
|
|
78
|
+
"there is no canonical field list for audit-entry contract " \
|
|
79
|
+
"#{version.inspect}; only contract " \
|
|
80
|
+
"#{ENTRY_FIELDS.keys.join(', ')} rows are chained"
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
pairs = fields.map { |field| [field, value(row, field)] }
|
|
84
|
+
dump([["previous_digest", previous_digest], *pairs])
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
def microseconds(time)
|
|
88
|
+
(time.to_r * 1_000_000).floor
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
def dump(value)
|
|
92
|
+
case value
|
|
93
|
+
when Array then "[#{value.map { |item| dump(item) }.join(',')}]"
|
|
94
|
+
when Hash then object(value)
|
|
95
|
+
else scalar(value)
|
|
96
|
+
end
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
def scalar(value)
|
|
100
|
+
case value
|
|
101
|
+
when nil, true, false then value.nil? ? "null" : value.to_s
|
|
102
|
+
when Integer, Float, BigDecimal then number(value)
|
|
103
|
+
when String, Symbol then JSON.generate(value.to_s)
|
|
104
|
+
else
|
|
105
|
+
raise ArgumentError,
|
|
106
|
+
"#{value.class} has no canonical form; normalize it before it is digested"
|
|
107
|
+
end
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
def object(hash)
|
|
111
|
+
pairs = hash.map { |key, item| [key.to_s, item] }.sort_by(&:first)
|
|
112
|
+
"{#{pairs.map { |key, item| "#{JSON.generate(key)}:#{dump(item)}" }.join(',')}}"
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
# The shortest exact decimal: no exponent, no trailing zeros, no "-0".
|
|
116
|
+
def number(value)
|
|
117
|
+
if value.is_a?(Float) && !value.finite?
|
|
118
|
+
raise ArgumentError,
|
|
119
|
+
"a non-finite number has no canonical form; normalize it before it is digested"
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
decimal = value.is_a?(Float) ? BigDecimal(value.to_s) : BigDecimal(value)
|
|
123
|
+
return "0" if decimal.zero?
|
|
124
|
+
return decimal.to_i.to_s if decimal.frac.zero?
|
|
125
|
+
|
|
126
|
+
decimal.to_s("F")
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
def value(row, field)
|
|
130
|
+
raw = if row.is_a?(Hash)
|
|
131
|
+
row.key?(field) ? row[field] : row[field.to_sym]
|
|
132
|
+
else
|
|
133
|
+
row[field]
|
|
134
|
+
end
|
|
135
|
+
TIME_FIELDS.include?(field) && !raw.nil? ? microseconds(raw) : raw
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
def hmac(secret, tag, body)
|
|
139
|
+
OpenSSL::HMAC.hexdigest("SHA256", secret, "#{tag}\n#{body}")
|
|
140
|
+
end
|
|
141
|
+
end
|
|
142
|
+
end
|
|
143
|
+
end
|
|
144
|
+
end
|
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Reeve
|
|
4
|
+
module Audit
|
|
5
|
+
class Chain
|
|
6
|
+
# The key epochs as verification sees them (specs/003 FR-011): read once, then asked
|
|
7
|
+
# about every row the walk visits. Counts, as it goes, the rows that named each epoch and
|
|
8
|
+
# the rows chained before the first, for the report.
|
|
9
|
+
#
|
|
10
|
+
# A row at contract 4 names its epoch, and must carry that epoch's key and not precede
|
|
11
|
+
# its start. A contract-3 row names none and is checked exactly as 0.6.0 checked it,
|
|
12
|
+
# wherever it sits: one written by a 0.6.0 server after the first epoch opened may carry
|
|
13
|
+
# any key that server had, and a verifier without that key cannot tell it from a
|
|
14
|
+
# rewritten one — calling it tampered would report a rolling deploy as an attack.
|
|
15
|
+
class EpochRules
|
|
16
|
+
attr_reader :records, :counts, :before_first_epoch, :summaries
|
|
17
|
+
|
|
18
|
+
# Every epoch record, in one query whatever the ledger's size (FR-014). A ledger whose
|
|
19
|
+
# gem was upgraded before its table was migrated has none — and no row that could
|
|
20
|
+
# name one, since writes refuse until it is migrated (FR-018).
|
|
21
|
+
def self.read(keyring:, entry_class:)
|
|
22
|
+
present = entry_class.connection_pool.with_connection do |connection|
|
|
23
|
+
connection.table_exists?(EPOCHS_TABLE)
|
|
24
|
+
end
|
|
25
|
+
new(present ? ChainEpoch.uncached { ChainEpoch.all.to_a } : [], keyring: keyring)
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
# `records` are ChainEpoch rows, in any order.
|
|
29
|
+
def initialize(records, keyring:)
|
|
30
|
+
@records = records.sort_by(&:epoch).freeze
|
|
31
|
+
@keyring = keyring
|
|
32
|
+
@by_number = @records.to_h { |record| [record.epoch, record] }
|
|
33
|
+
@first_start = @records.map(&:first_sequence).min
|
|
34
|
+
@counts = Hash.new(0)
|
|
35
|
+
@before_first_epoch = 0
|
|
36
|
+
@summaries = []
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
# Checks the records themselves once the walk has counted their rows (EpochCheck);
|
|
40
|
+
# `head` is the chain record as read after the walk. Leaves the result in `summaries`.
|
|
41
|
+
def check!(record, head, report:)
|
|
42
|
+
@summaries = EpochCheck.new(keyring: keyring, report: report)
|
|
43
|
+
.call(self, genesis_digest: record&.genesis_digest, head: head)
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
# [kind, detail] for a row that contradicts the epoch records, or nil.
|
|
47
|
+
def judge(entry)
|
|
48
|
+
return epoch_bearing(entry) if entry.contract_version.to_i >= EPOCH_CONTRACT
|
|
49
|
+
|
|
50
|
+
@before_first_epoch += 1 if before_first?(entry.chain_sequence)
|
|
51
|
+
nil
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
private
|
|
55
|
+
|
|
56
|
+
attr_reader :keyring
|
|
57
|
+
|
|
58
|
+
def epoch_bearing(entry)
|
|
59
|
+
number = entry.chain_epoch
|
|
60
|
+
epoch = number && @by_number[number]
|
|
61
|
+
return [:epoch_missing, missing(number)] unless epoch
|
|
62
|
+
|
|
63
|
+
@counts[number] += 1
|
|
64
|
+
if epoch.key_id != entry.chain_key_id
|
|
65
|
+
[:epoch_mismatch,
|
|
66
|
+
"signed by key #{entry.chain_key_id}, but epoch #{number}'s key is #{epoch.key_id}"]
|
|
67
|
+
elsif entry.chain_sequence < epoch.first_sequence
|
|
68
|
+
[:epoch_mismatch,
|
|
69
|
+
"names epoch #{number}, which begins at sequence #{epoch.first_sequence}"]
|
|
70
|
+
end
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
# True for a row chained before any epoch began — and for every row when none has.
|
|
74
|
+
def before_first?(sequence)
|
|
75
|
+
@first_start.nil? || sequence < @first_start
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
def missing(number)
|
|
79
|
+
number.nil? ? "names no key epoch" : "names epoch #{number}, which has no record"
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
# The epoch records themselves (specs/003 FR-010): numbered 0..n, each beginning after
|
|
84
|
+
# the one before (or with it: two rotations with no write between them begin at the same
|
|
85
|
+
# sequence) and no later than one past the head, one key each, and each matching
|
|
86
|
+
# its digest — which links it to the epoch before, or, for epoch 0, to the genesis. A
|
|
87
|
+
# record whose key is not configured cannot be checked, and that is said, never passed
|
|
88
|
+
# over. Returns one Verification::EpochSummary per record; breaks go to `report`, as
|
|
89
|
+
# (kind, nil, detail, nil, key_id, epoch:) — the verifier's own `add`.
|
|
90
|
+
class EpochCheck
|
|
91
|
+
def initialize(keyring:, report:)
|
|
92
|
+
@keyring = keyring
|
|
93
|
+
@report = report
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
# `head` is the chain record as read after the walk — later than the epochs were
|
|
97
|
+
# read, so an epoch a rotation opened mid-run is never taken for one past the head.
|
|
98
|
+
def call(rules, genesis_digest:, head:)
|
|
99
|
+
previous = { digest: genesis_digest, first_sequence: 0, keys: [] }
|
|
100
|
+
expected = 0
|
|
101
|
+
rules.records.map do |epoch|
|
|
102
|
+
(expected...epoch.epoch).each do |gap|
|
|
103
|
+
report(:epoch_missing, gap, "epoch #{gap} has no record")
|
|
104
|
+
end
|
|
105
|
+
expected = epoch.epoch + 1
|
|
106
|
+
summary(epoch, rules.counts[epoch.epoch], check(epoch, previous, head)).tap do
|
|
107
|
+
previous = { digest: epoch.epoch_digest, first_sequence: epoch.first_sequence,
|
|
108
|
+
keys: previous[:keys] + [epoch.key_id] }
|
|
109
|
+
end
|
|
110
|
+
end
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
private
|
|
114
|
+
|
|
115
|
+
attr_reader :keyring
|
|
116
|
+
|
|
117
|
+
# :altered, :key_not_configured or :verified, reporting what makes it so.
|
|
118
|
+
def check(epoch, previous, head)
|
|
119
|
+
problem = out_of_place(epoch, previous, head) || digest_problem(epoch, previous)
|
|
120
|
+
return :altered.tap { report(:epoch_altered, epoch.epoch, problem) } if problem
|
|
121
|
+
return :verified if keyring.for(epoch.key_id)
|
|
122
|
+
|
|
123
|
+
report(:unknown_key, epoch.epoch, "epoch #{epoch.epoch} is signed by key #{epoch.key_id}",
|
|
124
|
+
epoch.key_id)
|
|
125
|
+
:key_not_configured
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
def report(kind, number, detail, key_id = nil)
|
|
129
|
+
@report.call(kind, nil, detail, nil, key_id, epoch: number)
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
def out_of_place(epoch, previous, head)
|
|
133
|
+
name = "epoch #{epoch.epoch}"
|
|
134
|
+
if epoch.epoch.positive? && epoch.first_sequence < previous[:first_sequence]
|
|
135
|
+
"#{name} begins at sequence #{epoch.first_sequence}, before the epoch before it"
|
|
136
|
+
elsif head && epoch.first_sequence > head.head_sequence + 1
|
|
137
|
+
"#{name} begins at sequence #{epoch.first_sequence}, past the head"
|
|
138
|
+
elsif previous[:keys].include?(epoch.key_id)
|
|
139
|
+
"#{name} reuses the key of an earlier epoch"
|
|
140
|
+
end
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
def digest_problem(epoch, previous)
|
|
144
|
+
secret = keyring.for(epoch.key_id)
|
|
145
|
+
return nil if secret.nil? || previous[:digest].nil?
|
|
146
|
+
|
|
147
|
+
expected = Canonical.epoch_digest(secret, epoch, previous_digest: previous[:digest])
|
|
148
|
+
"epoch #{epoch.epoch} does not match its digest" unless expected == epoch.epoch_digest
|
|
149
|
+
end
|
|
150
|
+
|
|
151
|
+
def summary(epoch, count, status)
|
|
152
|
+
Verification::EpochSummary.new(epoch: epoch.epoch, key_id: epoch.key_id,
|
|
153
|
+
first_sequence: epoch.first_sequence, entry_count: count,
|
|
154
|
+
status: status)
|
|
155
|
+
end
|
|
156
|
+
end
|
|
157
|
+
end
|
|
158
|
+
end
|
|
159
|
+
end
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Reeve
|
|
4
|
+
module Audit
|
|
5
|
+
class Chain
|
|
6
|
+
# The key epochs, as a write sees them (specs/003 FR-001–FR-005).
|
|
7
|
+
#
|
|
8
|
+
# One instance per append or rotation, bound to the connection it runs on and used
|
|
9
|
+
# only under the chain record's lock (Head), so finding a key's epoch and opening the
|
|
10
|
+
# next one are a single step as far as any other writer can tell. The epoch record
|
|
11
|
+
# commits or rolls back with the write that opened it.
|
|
12
|
+
#
|
|
13
|
+
# A key opens one epoch, ever. A process signing with a key whose epoch is not the
|
|
14
|
+
# latest — one still on the old key during a rolling deploy — writes under that
|
|
15
|
+
# earlier epoch: a straggler, legitimate, and verified by its own key.
|
|
16
|
+
class Epochs
|
|
17
|
+
def initialize(connection:)
|
|
18
|
+
@connection = connection
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
# The number of the epoch `signer`'s key opened, opening the next one at `sequence`
|
|
22
|
+
# if it has none.
|
|
23
|
+
def for_signing!(signer, sequence:)
|
|
24
|
+
number_for(signer.signing_id) || open!(signer, first_sequence: sequence)[:epoch]
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def number_for(key_id)
|
|
28
|
+
sql = "SELECT epoch FROM #{EPOCHS_TABLE} WHERE key_id = #{connection.quote(key_id)}"
|
|
29
|
+
value = connection.select_value(current(sql))
|
|
30
|
+
value.nil? ? nil : Integer(value)
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
# Every epoch, oldest first, as hashes with symbol keys.
|
|
34
|
+
def all
|
|
35
|
+
sql = "SELECT epoch, key_id, first_sequence FROM #{EPOCHS_TABLE} ORDER BY epoch"
|
|
36
|
+
connection.select_all(current(sql))
|
|
37
|
+
.map do |row|
|
|
38
|
+
{ epoch: Integer(row["epoch"]), key_id: row["key_id"],
|
|
39
|
+
first_sequence: Integer(row["first_sequence"]) }
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
# Writes epoch `max + 1` (or 0) for `signer`'s key and returns it. Linked to the
|
|
44
|
+
# latest epoch's digest, or to the genesis for epoch 0, so the chain must have
|
|
45
|
+
# started.
|
|
46
|
+
def open!(signer, first_sequence:)
|
|
47
|
+
sql = "SELECT epoch, epoch_digest FROM #{EPOCHS_TABLE} ORDER BY epoch DESC LIMIT 1"
|
|
48
|
+
latest = connection.select_one(current(sql))
|
|
49
|
+
epoch = { epoch: latest ? Integer(latest["epoch"]) + 1 : 0, key_id: signer.signing_id,
|
|
50
|
+
first_sequence: first_sequence, opened_at: Time.now.utc.floor(6) }
|
|
51
|
+
previous = latest ? latest["epoch_digest"] : genesis_digest
|
|
52
|
+
digest = Canonical.epoch_digest(signer.signing_secret, epoch, previous_digest: previous)
|
|
53
|
+
epoch.merge(epoch_digest: digest).tap { |values| insert(values) }
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
private
|
|
57
|
+
|
|
58
|
+
attr_reader :connection
|
|
59
|
+
|
|
60
|
+
def current(sql)
|
|
61
|
+
Head.current(connection, sql)
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
def insert(values)
|
|
65
|
+
quoted = values.values.map { |value| connection.quote(value) }
|
|
66
|
+
connection.insert("INSERT INTO #{EPOCHS_TABLE} (#{values.keys.join(', ')}) " \
|
|
67
|
+
"VALUES (#{quoted.join(', ')})")
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def genesis_digest
|
|
71
|
+
connection.select_value("SELECT genesis_digest FROM #{TABLE} " \
|
|
72
|
+
"WHERE name = #{connection.quote(ChainRecord::LEDGER)}")
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
end
|
|
77
|
+
end
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Reeve
|
|
4
|
+
module Audit
|
|
5
|
+
class Chain
|
|
6
|
+
# The chain record's head, advanced under its lock (research R3).
|
|
7
|
+
#
|
|
8
|
+
# One instance per append, bound to the connection that append runs on; every
|
|
9
|
+
# statement here goes through that connection, so the lock, the row and the head move
|
|
10
|
+
# share one transaction and commit or roll back together.
|
|
11
|
+
#
|
|
12
|
+
# UPDATE head_sequence by one -- the lock, taken first
|
|
13
|
+
# SELECT the new head_sequence and the old head_digest
|
|
14
|
+
# ... the caller inserts the row ...
|
|
15
|
+
# UPDATE head_digest to the row's digest
|
|
16
|
+
#
|
|
17
|
+
# Taking the lock with a write is what makes this behave the same on all three
|
|
18
|
+
# engines: SQLite cannot upgrade a read lock while another writer waits, and ignores
|
|
19
|
+
# SELECT ... FOR UPDATE.
|
|
20
|
+
class Head
|
|
21
|
+
LOCKING_READ_ADAPTERS = /mysql|trilogy/i
|
|
22
|
+
|
|
23
|
+
# A read made under the head's lock must see the latest committed rows. PostgreSQL
|
|
24
|
+
# and SQLite read them anyway; InnoDB at REPEATABLE READ serves a plain SELECT from
|
|
25
|
+
# the snapshot of the caller's transaction, which a writer that committed while this
|
|
26
|
+
# one queued for the lock is not in (code review, specs/003). A locking read is
|
|
27
|
+
# current; LOCK IN SHARE MODE is the spelling every MySQL version accepts.
|
|
28
|
+
def self.current(connection, sql)
|
|
29
|
+
connection.adapter_name.match?(LOCKING_READ_ADAPTERS) ? "#{sql} LOCK IN SHARE MODE" : sql
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def initialize(connection:, entry_class:)
|
|
33
|
+
@connection = connection
|
|
34
|
+
@entry_class = entry_class
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
# Creates the chain record with its genesis — link 0, head 0 — unless it exists.
|
|
38
|
+
#
|
|
39
|
+
# Deliberately not part of the write that follows. Run first inside it, an UPDATE that
|
|
40
|
+
# matched no row took a gap lock on MySQL's unique index, and two writers starting the
|
|
41
|
+
# chain at once each tried to INSERT into the gap the other held: a deadlock, found
|
|
42
|
+
# by CI, invisible on PostgreSQL, which has no gap locks. On its own, two concurrent
|
|
43
|
+
# INSERTs of one unique name serialize: one wins and the other gets a duplicate key,
|
|
44
|
+
# which means the chain has started and is all it needs to know.
|
|
45
|
+
#
|
|
46
|
+
# So a started chain can exist with nothing in it yet (head 0) — if the first chained
|
|
47
|
+
# write then fails, say. The verifier reports that as no chain, not as a pass.
|
|
48
|
+
def start!(signer)
|
|
49
|
+
return if connection.select_value("SELECT 1 FROM #{TABLE} WHERE #{named}")
|
|
50
|
+
|
|
51
|
+
genesis = genesis_for(signer)
|
|
52
|
+
digest = Canonical.genesis_digest(signer.signing_secret, genesis)
|
|
53
|
+
values = genesis.merge(name: ChainRecord::LEDGER, genesis_digest: digest,
|
|
54
|
+
head_sequence: 0, head_digest: digest)
|
|
55
|
+
quoted = values.values.map { |value| connection.quote(value) }
|
|
56
|
+
|
|
57
|
+
entry_class.transaction(requires_new: true) do
|
|
58
|
+
connection.insert("INSERT INTO #{TABLE} (#{values.keys.join(', ')}) " \
|
|
59
|
+
"VALUES (#{quoted.join(', ')})")
|
|
60
|
+
end
|
|
61
|
+
rescue ActiveRecord::RecordNotUnique
|
|
62
|
+
nil
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
# Locks the head and returns the next sequence and the digest it links to. The chain
|
|
66
|
+
# must have been started.
|
|
67
|
+
def advance!
|
|
68
|
+
vanished = increment.zero?
|
|
69
|
+
raise ActiveRecord::StatementInvalid, "the ledger chain record vanished" if vanished
|
|
70
|
+
|
|
71
|
+
head = connection.select_one("SELECT head_sequence, head_digest FROM #{TABLE} " \
|
|
72
|
+
"WHERE #{named}")
|
|
73
|
+
[Integer(head["head_sequence"]), head["head_digest"]]
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
# Takes the head's lock without consuming a sequence (specs/003 FR-008) and returns the
|
|
77
|
+
# head: a rotation queues at the same point as a write, so the epoch it opens begins
|
|
78
|
+
# exactly one past the last row any writer committed. A write that changes nothing
|
|
79
|
+
# still takes the row lock on PostgreSQL and MySQL and the write lock on SQLite.
|
|
80
|
+
def lock!
|
|
81
|
+
updated = connection.update("UPDATE #{TABLE} SET head_sequence = head_sequence " \
|
|
82
|
+
"WHERE #{named}")
|
|
83
|
+
raise ActiveRecord::StatementInvalid, "the ledger chain record vanished" if updated.zero?
|
|
84
|
+
|
|
85
|
+
sql = "SELECT head_sequence FROM #{TABLE} WHERE #{named}"
|
|
86
|
+
Integer(connection.select_value(Head.current(connection, sql)))
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
def point_at!(digest)
|
|
90
|
+
connection.update("UPDATE #{TABLE} SET head_digest = #{connection.quote(digest)} " \
|
|
91
|
+
"WHERE #{named}")
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
# The connection this append runs on; Epochs uses it so the epoch an append opens
|
|
95
|
+
# commits or rolls back with the append.
|
|
96
|
+
attr_reader :connection
|
|
97
|
+
|
|
98
|
+
private
|
|
99
|
+
|
|
100
|
+
attr_reader :entry_class
|
|
101
|
+
|
|
102
|
+
def increment
|
|
103
|
+
connection.update("UPDATE #{TABLE} SET head_sequence = head_sequence + 1 WHERE #{named}")
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
def genesis_for(signer)
|
|
107
|
+
{
|
|
108
|
+
genesis_at: Time.now.utc.floor(6),
|
|
109
|
+
pre_chain_count: entry_class.where("contract_version < 3").count,
|
|
110
|
+
pre_chain_unchained_count:
|
|
111
|
+
entry_class.where(UNCHAINED).count,
|
|
112
|
+
genesis_key_id: signer.signing_id
|
|
113
|
+
}
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
def named
|
|
117
|
+
"name = #{connection.quote(ChainRecord::LEDGER)}"
|
|
118
|
+
end
|
|
119
|
+
end
|
|
120
|
+
end
|
|
121
|
+
end
|
|
122
|
+
end
|