forest_admin_agent 1.38.2 → 1.39.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/AUDIT_TRAIL.md +347 -0
- data/lib/forest_admin_agent/audit_trail/action_capture.rb +99 -0
- data/lib/forest_admin_agent/audit_trail/audit_record.rb +16 -0
- data/lib/forest_admin_agent/audit_trail/capture.rb +229 -0
- data/lib/forest_admin_agent/audit_trail/diff.rb +156 -0
- data/lib/forest_admin_agent/audit_trail/record_state.rb +43 -0
- data/lib/forest_admin_agent/audit_trail/recording.rb +57 -0
- data/lib/forest_admin_agent/audit_trail/snapshots.rb +85 -0
- data/lib/forest_admin_agent/audit_trail/sql/audit_connection_base.rb +53 -0
- data/lib/forest_admin_agent/audit_trail/sql/audit_log.rb +16 -0
- data/lib/forest_admin_agent/audit_trail/sql/field_filter.rb +58 -0
- data/lib/forest_admin_agent/audit_trail/sql/migrations.rb +53 -0
- data/lib/forest_admin_agent/audit_trail/sql/migrator.rb +117 -0
- data/lib/forest_admin_agent/audit_trail/sql/text_search.rb +78 -0
- data/lib/forest_admin_agent/audit_trail/store.rb +253 -0
- data/lib/forest_admin_agent/audit_trail.rb +68 -0
- data/lib/forest_admin_agent/builder/agent_factory.rb +20 -0
- data/lib/forest_admin_agent/http/correlation_id.rb +37 -0
- data/lib/forest_admin_agent/http/correlation_id_middleware.rb +29 -0
- data/lib/forest_admin_agent/http/router.rb +6 -0
- data/lib/forest_admin_agent/routes/action/actions.rb +76 -1
- data/lib/forest_admin_agent/routes/capabilities/collections.rb +11 -1
- data/lib/forest_admin_agent/routes/resources/audit_trail.rb +237 -0
- data/lib/forest_admin_agent/routes/resources/audit_trail_correlation.rb +101 -0
- data/lib/forest_admin_agent/routes/resources/audit_trail_route.rb +115 -0
- data/lib/forest_admin_agent/routes/resources/list.rb +1 -1
- data/lib/forest_admin_agent/routes/resources/related/list_related.rb +1 -1
- data/lib/forest_admin_agent/routes/resources/show.rb +1 -1
- data/lib/forest_admin_agent/serializer/forest_serializer_override.rb +14 -13
- data/lib/forest_admin_agent/utils/caller_parser.rb +4 -0
- data/lib/forest_admin_agent/utils/query_string_parser.rb +59 -10
- data/lib/forest_admin_agent/utils/schema/schema_emitter.rb +1 -1
- data/lib/forest_admin_agent/version.rb +1 -1
- data/lib/forest_admin_agent.rb +4 -0
- metadata +23 -2
|
@@ -0,0 +1,229 @@
|
|
|
1
|
+
module ForestAdminAgent
|
|
2
|
+
module AuditTrail
|
|
3
|
+
# Datasource-agnostic capture layer, installed by the agent factory as soon as an audit-trail database is
|
|
4
|
+
# configured. It instruments every collection through the Forest customizer hooks, so it behaves the same
|
|
5
|
+
# whatever the audited datasource is (ActiveRecord, Mongoid, ...).
|
|
6
|
+
#
|
|
7
|
+
# Every operation is recorded twice: a PENDING row before the write, confirmed DONE after it. One code
|
|
8
|
+
# path in both `critical` modes, so `status` always means the same thing — and what the protocol buys is
|
|
9
|
+
# that no write goes unaudited, not that every row holds exact after-values. A row left pending says the
|
|
10
|
+
# write may or may not have landed, which is evidence rather than a defect.
|
|
11
|
+
class Capture
|
|
12
|
+
include Recording
|
|
13
|
+
|
|
14
|
+
# Signature imposed by DatasourceCustomizer#use; the agent always instruments the whole datasource.
|
|
15
|
+
def run(datasource_customizer, _collection_customizer = nil, options = {})
|
|
16
|
+
@store = options[:store]
|
|
17
|
+
@redact = options[:redact] || {}
|
|
18
|
+
@snapshots = Snapshots.new
|
|
19
|
+
|
|
20
|
+
datasource_customizer.collections.each_value { |collection| instrument(collection) }
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
private
|
|
24
|
+
|
|
25
|
+
def instrument(collection_customizer)
|
|
26
|
+
schema = collection_customizer.collection.schema
|
|
27
|
+
# Writable columns only: Forest audits what it writes. Read-only fields cover computed/virtual
|
|
28
|
+
# fields and DB-managed columns, none of which Forest mutates.
|
|
29
|
+
columns = schema[:fields].select do |_name, field|
|
|
30
|
+
field.type == 'Column' && !field.is_read_only
|
|
31
|
+
end.keys
|
|
32
|
+
primary_keys = ForestAdminDatasourceToolkit::Utils::Schema.primary_keys(collection_customizer.collection)
|
|
33
|
+
# Reads must carry the primary keys (even read-only ones) so the record id can be built; the
|
|
34
|
+
# diff itself stays restricted to the writable columns.
|
|
35
|
+
projection = ForestAdminDatasourceToolkit::Components::Query::Projection.new(
|
|
36
|
+
(primary_keys + columns).uniq
|
|
37
|
+
)
|
|
38
|
+
target = { columns: columns, primary_keys: primary_keys, projection: projection,
|
|
39
|
+
name: collection_customizer.name }
|
|
40
|
+
|
|
41
|
+
add_create_hooks(collection_customizer, target)
|
|
42
|
+
add_update_hooks(collection_customizer, target)
|
|
43
|
+
add_delete_hooks(collection_customizer, target)
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
# The "after" hooks are prepended: `execute_after` stops at the first exception, so a customization
|
|
47
|
+
# raising in its own after hook would otherwise drop the record of a write that already happened.
|
|
48
|
+
# The "before" hooks stay appended, so they read the data, filter and patch every other customization
|
|
49
|
+
# has had its say on.
|
|
50
|
+
def add_create_hooks(collection_customizer, target)
|
|
51
|
+
collection_customizer.add_hook('Before', 'Create') do |context|
|
|
52
|
+
# No record id yet — that is what the column being nullable is for.
|
|
53
|
+
rows = [{ record_id: nil, new_values: pick(context.data, target[:columns]) }]
|
|
54
|
+
|
|
55
|
+
@snapshots.push(context.data, ids: pending_rows(context.caller, 'create', target[:name], rows))
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
collection_customizer.add_hook('After', 'Create', prepend: true) do |context|
|
|
59
|
+
pending = @snapshots.pop_for(context.data)
|
|
60
|
+
next unless pending
|
|
61
|
+
|
|
62
|
+
confirm(pending[:ids].first,
|
|
63
|
+
record_id: record_id(context.record, target[:primary_keys]),
|
|
64
|
+
new_values: redacted(target[:name], pick(context.record, target[:columns])))
|
|
65
|
+
end
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
def add_update_hooks(collection_customizer, target)
|
|
69
|
+
collection_customizer.add_hook('Before', 'Update') do |context|
|
|
70
|
+
records = @snapshots.take(context, target[:projection])
|
|
71
|
+
@snapshots.push(
|
|
72
|
+
context.filter,
|
|
73
|
+
records: records,
|
|
74
|
+
patch: context.patch,
|
|
75
|
+
ids: pending_rows(
|
|
76
|
+
context.caller, 'update', target[:name],
|
|
77
|
+
records.map do |record|
|
|
78
|
+
{ record_id: record_id(record, target[:primary_keys]),
|
|
79
|
+
previous_values: pick(record, context.patch.keys & target[:columns]),
|
|
80
|
+
new_values: pick(context.patch, target[:columns]) }
|
|
81
|
+
end
|
|
82
|
+
)
|
|
83
|
+
)
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
collection_customizer.add_hook('After', 'Update', prepend: true) do |context|
|
|
87
|
+
pending = @snapshots.pop_for(context.filter)
|
|
88
|
+
next unless pending
|
|
89
|
+
|
|
90
|
+
confirm_updates(context, pending, target)
|
|
91
|
+
end
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
def add_delete_hooks(collection_customizer, target)
|
|
95
|
+
collection_customizer.add_hook('Before', 'Delete') do |context|
|
|
96
|
+
records = @snapshots.take(context, target[:projection])
|
|
97
|
+
@snapshots.push(
|
|
98
|
+
context.filter,
|
|
99
|
+
records: records,
|
|
100
|
+
ids: pending_rows(
|
|
101
|
+
context.caller, 'delete', target[:name],
|
|
102
|
+
records.map do |record|
|
|
103
|
+
{ record_id: record_id(record, target[:primary_keys]),
|
|
104
|
+
previous_values: pick(record, target[:columns]) }
|
|
105
|
+
end
|
|
106
|
+
)
|
|
107
|
+
)
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
collection_customizer.add_hook('After', 'Delete', prepend: true) do |context|
|
|
111
|
+
pending = @snapshots.pop_for(context.filter)
|
|
112
|
+
next unless pending
|
|
113
|
+
|
|
114
|
+
pending[:ids].each { |id| confirm(id) }
|
|
115
|
+
end
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
# The diff is taken against the record as persisted, not the patch that was requested, so a value the
|
|
119
|
+
# datasource normalised or a decorator rewrote is recorded as what actually landed. The id is packed from
|
|
120
|
+
# those same values, so an update that changed a primary key files the row under the id the history will
|
|
121
|
+
# be queried by.
|
|
122
|
+
def confirm_updates(context, pending, target)
|
|
123
|
+
pks = target[:primary_keys]
|
|
124
|
+
persisted = reread(context, pending[:records], pending[:patch], target)
|
|
125
|
+
empty = []
|
|
126
|
+
|
|
127
|
+
pending[:records].each_with_index do |record, index|
|
|
128
|
+
# No row read back: the write may or may not have landed, and inventing after-values from the patch
|
|
129
|
+
# would confirm — or worse, discard — a row for something that may never have happened. Left pending,
|
|
130
|
+
# which is exactly what that state means.
|
|
131
|
+
after = persisted[record_id(record.merge(pending[:patch]), pks)]
|
|
132
|
+
next if after.nil?
|
|
133
|
+
|
|
134
|
+
delta = Diff.changed_values(record, after, target[:columns])
|
|
135
|
+
|
|
136
|
+
if delta[:new_values].empty?
|
|
137
|
+
empty << pending[:ids][index]
|
|
138
|
+
else
|
|
139
|
+
before_id = record_id(record, pks)
|
|
140
|
+
after_id = record_id(after, pks)
|
|
141
|
+
|
|
142
|
+
confirm(pending[:ids][index],
|
|
143
|
+
record_id: after_id,
|
|
144
|
+
# Only when the key actually moved, so a history query can walk back to the rows filed
|
|
145
|
+
# under the id this record used to have.
|
|
146
|
+
previous_record_id: after_id == before_id ? nil : before_id,
|
|
147
|
+
previous_values: redacted(target[:name], delta[:previous_values]),
|
|
148
|
+
new_values: redacted(target[:name], delta[:new_values]))
|
|
149
|
+
end
|
|
150
|
+
end
|
|
151
|
+
|
|
152
|
+
audit_safely { @store.discard(empty.compact) } if empty.any?
|
|
153
|
+
end
|
|
154
|
+
|
|
155
|
+
# Reads the updated records back in one query, keyed by their own id, so each snapshot can find what
|
|
156
|
+
# actually landed — including when the patch moved a primary key.
|
|
157
|
+
def reread(context, records, patch, target)
|
|
158
|
+
pks = target[:primary_keys]
|
|
159
|
+
return {} if records.empty?
|
|
160
|
+
|
|
161
|
+
audit_safely do
|
|
162
|
+
condition = ids_condition(pks, records.map { |record| record.merge(patch).slice(*pks) })
|
|
163
|
+
filter = ForestAdminDatasourceToolkit::Components::Query::Filter.new(condition_tree: condition)
|
|
164
|
+
|
|
165
|
+
context.collection.list(filter, target[:projection]).to_h { |row| [record_id(row, pks), row] }
|
|
166
|
+
end || {}
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
# Built from the primary keys the instrumentation already resolved, rather than through
|
|
170
|
+
# ConditionTreeFactory, which would need the underlying collection a hook context does not hand out.
|
|
171
|
+
def ids_condition(primary_keys, ids)
|
|
172
|
+
leaf = ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeLeaf
|
|
173
|
+
factory = ForestAdminDatasourceToolkit::Components::Query::ConditionTree::ConditionTreeFactory
|
|
174
|
+
|
|
175
|
+
operators = ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Operators
|
|
176
|
+
|
|
177
|
+
if primary_keys.size == 1
|
|
178
|
+
key = primary_keys.first
|
|
179
|
+
|
|
180
|
+
leaf.new(key, operators::IN, ids.map { |id| id[key] }.uniq)
|
|
181
|
+
else
|
|
182
|
+
factory.union(
|
|
183
|
+
ids.map { |id| factory.intersect(id.map { |key, value| leaf.new(key, operators::EQUAL, value) }) }
|
|
184
|
+
)
|
|
185
|
+
end
|
|
186
|
+
end
|
|
187
|
+
|
|
188
|
+
# The one place the audit trail may refuse an operation, and only under `critical: true`: if we cannot
|
|
189
|
+
# record that a write is about to happen, the write does not happen.
|
|
190
|
+
def pending_rows(caller, operation, collection, rows)
|
|
191
|
+
timestamp = now
|
|
192
|
+
correlation_key = correlation_key_for(caller)
|
|
193
|
+
identity = identity_of(caller)
|
|
194
|
+
|
|
195
|
+
AuditTrail.gate do
|
|
196
|
+
@store.append_all(
|
|
197
|
+
rows.map do |row|
|
|
198
|
+
AuditRecord.new(
|
|
199
|
+
timestamp: timestamp, operation: operation, collection: collection, status: PENDING,
|
|
200
|
+
correlation_key: correlation_key, record_id: row[:record_id],
|
|
201
|
+
previous_values: redacted(collection, row[:previous_values] || {}),
|
|
202
|
+
new_values: redacted(collection, row[:new_values] || {}),
|
|
203
|
+
**identity
|
|
204
|
+
)
|
|
205
|
+
end
|
|
206
|
+
)
|
|
207
|
+
end || []
|
|
208
|
+
end
|
|
209
|
+
|
|
210
|
+
def confirm(id, attributes = {})
|
|
211
|
+
return unless id
|
|
212
|
+
|
|
213
|
+
audit_safely { @store.confirm(id, attributes) }
|
|
214
|
+
end
|
|
215
|
+
|
|
216
|
+
def redacted(collection, values)
|
|
217
|
+
redact(values, @redact[collection] || [])
|
|
218
|
+
end
|
|
219
|
+
|
|
220
|
+
def record_id(record, primary_keys)
|
|
221
|
+
primary_keys.map { |pk| record[pk].to_s }.join('|')
|
|
222
|
+
end
|
|
223
|
+
|
|
224
|
+
def pick(record, columns)
|
|
225
|
+
columns.to_h { |column| [column, record[column]] }
|
|
226
|
+
end
|
|
227
|
+
end
|
|
228
|
+
end
|
|
229
|
+
end
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
module ForestAdminAgent
|
|
2
|
+
module AuditTrail
|
|
3
|
+
# Minimal structural diff. Nested hashes and arrays of hashes are recursed into, so only the
|
|
4
|
+
# keys/indexes whose leaf value actually changed are kept — a single sub-field change does not
|
|
5
|
+
# store the whole object/array. Scalars, primitive arrays, dates and other values are compared and
|
|
6
|
+
# kept as a whole.
|
|
7
|
+
#
|
|
8
|
+
# Ruby's `==` already performs deep, key-order-independent equality on Hash and (ordered) equality
|
|
9
|
+
# on Array, so it is used directly as the equality primitive.
|
|
10
|
+
module Diff
|
|
11
|
+
# A key that does not exist on one side of the diff. It is never stored: the key is simply left out
|
|
12
|
+
# of that side's hash, so `{"flag" => nil}` (a key holding nil) and `{}` (no key) stay tellable
|
|
13
|
+
# apart in the database — which a revert needs and a string sentinel would only fake.
|
|
14
|
+
ABSENT = Object.new.freeze
|
|
15
|
+
|
|
16
|
+
module_function
|
|
17
|
+
|
|
18
|
+
# @return [Hash{Symbol=>Object}, nil] { previous:, next: } of the changed leaves, or nil when equal.
|
|
19
|
+
def diff(before, after)
|
|
20
|
+
return nil if before == after
|
|
21
|
+
|
|
22
|
+
return diff_hashes(before, after) if before.is_a?(Hash) && after.is_a?(Hash)
|
|
23
|
+
|
|
24
|
+
return diff_object_arrays(before, after) if object_array?(before) && object_array?(after)
|
|
25
|
+
|
|
26
|
+
{ previous: before.nil? ? nil : before, next: after.nil? ? nil : after }
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
# Build the previous/new value hashes for the writable columns that actually changed.
|
|
30
|
+
#
|
|
31
|
+
# @param before [Hash] snapshot of the record before the change (string keys)
|
|
32
|
+
# @param patch [Hash] the values being written (string keys); only present keys are considered
|
|
33
|
+
# @param columns [Array<String>] writable column names to inspect
|
|
34
|
+
# @return [Hash{Symbol=>Hash}] { previous_values:, new_values: }
|
|
35
|
+
def changed_values(before, patch, columns)
|
|
36
|
+
previous_values = {}
|
|
37
|
+
new_values = {}
|
|
38
|
+
|
|
39
|
+
columns.each do |column|
|
|
40
|
+
delta = patch.key?(column) ? diff(before[column], patch[column]) : nil
|
|
41
|
+
next unless delta
|
|
42
|
+
|
|
43
|
+
previous_values[column] = delta[:previous]
|
|
44
|
+
new_values[column] = delta[:next]
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
{ previous_values: previous_values, new_values: new_values }
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
# Arrays whose every element is a hash (record-like collections, e.g. a workflow history).
|
|
51
|
+
def object_array?(value)
|
|
52
|
+
value.is_a?(Array) && !value.empty? && value.all?(Hash)
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
def diff_hashes(before, after)
|
|
56
|
+
previous = {}
|
|
57
|
+
next_values = {}
|
|
58
|
+
|
|
59
|
+
(before.keys | after.keys).each do |key|
|
|
60
|
+
sub = diff_at(before, after, key)
|
|
61
|
+
next unless sub
|
|
62
|
+
|
|
63
|
+
previous[key] = sub[:previous] unless sub[:previous].equal?(ABSENT)
|
|
64
|
+
next_values[key] = sub[:next] unless sub[:next].equal?(ABSENT)
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
{ previous: previous, next: next_values }
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
# A key held with a nil value is not the same thing as a missing key: recursing on the values alone
|
|
71
|
+
# reads both as nil and reports no change at all.
|
|
72
|
+
def diff_at(before, after, key)
|
|
73
|
+
return diff(before[key], after[key]) if before.key?(key) == after.key?(key)
|
|
74
|
+
|
|
75
|
+
{
|
|
76
|
+
previous: before.key?(key) ? before[key] : ABSENT,
|
|
77
|
+
next: after.key?(key) ? after[key] : ABSENT
|
|
78
|
+
}
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
def diff_object_arrays(before, after)
|
|
82
|
+
previous = {}
|
|
83
|
+
next_values = {}
|
|
84
|
+
|
|
85
|
+
[before.length, after.length].max.times do |index|
|
|
86
|
+
sub = diff(before[index], after[index])
|
|
87
|
+
next unless sub
|
|
88
|
+
|
|
89
|
+
# Same rule as for hash keys: an index one side does not reach is left out of that side, so a
|
|
90
|
+
# revert can tell an appended element (drop it) from one whose value became nil (keep it).
|
|
91
|
+
previous[index] = sub[:previous] if index < before.length
|
|
92
|
+
next_values[index] = sub[:next] if index < after.length
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
{ previous: previous, next: next_values }
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
# Undo one recorded change: given the value as it stands now and the two sides of the diff that
|
|
99
|
+
# produced it, return the value as it was before. Nested hashes are walked so untouched keys keep
|
|
100
|
+
# their current value; a key missing from `previous` was added by the change, so it goes away.
|
|
101
|
+
#
|
|
102
|
+
# @param current [Object] the value as it stands now (may be nil when the record is gone)
|
|
103
|
+
# @param previous [Object] the `previous_values` side of the recorded diff
|
|
104
|
+
# @param changed [Object] the `new_values` side of the recorded diff
|
|
105
|
+
def revert(current, previous, changed)
|
|
106
|
+
return revert_array(current, previous, changed) if current.is_a?(Array) && partial?(previous, changed)
|
|
107
|
+
return revert_hash(current, previous, changed) if current.is_a?(Hash) && partial?(previous, changed)
|
|
108
|
+
|
|
109
|
+
previous
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
# Both sides being hashes is what `diff` emits for a structural diff; anything else replaced the
|
|
113
|
+
# value as a whole.
|
|
114
|
+
def partial?(previous, changed)
|
|
115
|
+
previous.is_a?(Hash) && changed.is_a?(Hash)
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
def revert_hash(current, previous, changed)
|
|
119
|
+
(previous.keys | changed.keys).each_with_object(current.dup) do |key, result|
|
|
120
|
+
if previous.key?(key)
|
|
121
|
+
result[key] = revert(current[key], previous[key], changed[key])
|
|
122
|
+
else
|
|
123
|
+
# Only the change introduced this key, so before the change there was none.
|
|
124
|
+
result.delete(key)
|
|
125
|
+
end
|
|
126
|
+
end
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
# Arrays of objects are diffed index by index, and JSON turns those indexes into strings on the way
|
|
130
|
+
# back out. Highest index first, so dropping an element the change appended leaves the lower ones
|
|
131
|
+
# where the diff expects them.
|
|
132
|
+
def revert_array(current, previous, changed)
|
|
133
|
+
indexes = (previous.keys | changed.keys).map(&:to_i).sort.reverse
|
|
134
|
+
|
|
135
|
+
indexes.each_with_object(current.dup) do |index, result|
|
|
136
|
+
if index?(previous, index)
|
|
137
|
+
result[index] = revert(current[index], at(previous, index), at(changed, index))
|
|
138
|
+
else
|
|
139
|
+
result.delete_at(index)
|
|
140
|
+
end
|
|
141
|
+
end
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
def index?(hash, index)
|
|
145
|
+
hash.key?(index) || hash.key?(index.to_s)
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
def at(hash, index)
|
|
149
|
+
hash.key?(index) ? hash[index] : hash[index.to_s]
|
|
150
|
+
end
|
|
151
|
+
|
|
152
|
+
private_class_method :diff_hashes, :diff_at, :diff_object_arrays, :partial?, :revert_hash,
|
|
153
|
+
:revert_array, :index?, :at
|
|
154
|
+
end
|
|
155
|
+
end
|
|
156
|
+
end
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
module ForestAdminAgent
|
|
2
|
+
module AuditTrail
|
|
3
|
+
# Rebuilds a record as it stood at a given instant by walking its history backwards from the record as
|
|
4
|
+
# it stands now, undoing every entry recorded after that instant (newest first).
|
|
5
|
+
#
|
|
6
|
+
# Only audited columns are reconstructed — read-only, computed and DB-managed fields are never recorded,
|
|
7
|
+
# so they cannot be restored.
|
|
8
|
+
module RecordState
|
|
9
|
+
module_function
|
|
10
|
+
|
|
11
|
+
# @param current [Hash, nil] the record as it stands now, nil when it no longer exists
|
|
12
|
+
# @param entries [Array<AuditRecord>] entries recorded after the instant, newest first
|
|
13
|
+
# @return [Hash, nil] the record at that instant, nil when it did not exist then
|
|
14
|
+
def at(current, entries)
|
|
15
|
+
entries.reduce(current) { |state, entry| undo(state, entry) }
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def undo(state, entry)
|
|
19
|
+
case entry.operation
|
|
20
|
+
when 'create'
|
|
21
|
+
# Created after the instant, so it did not exist then. An older entry can still bring a previous
|
|
22
|
+
# life of the same id back — the walk carries on.
|
|
23
|
+
nil
|
|
24
|
+
when 'delete'
|
|
25
|
+
# The delete recorded the whole record, which is exactly the state it was deleted from.
|
|
26
|
+
entry.previous_values
|
|
27
|
+
when 'update'
|
|
28
|
+
revert_columns(state, entry)
|
|
29
|
+
else
|
|
30
|
+
# Action rows carry no field change: their two value columns hold what was submitted to the action
|
|
31
|
+
# and what it answered, so applying either as a column change would corrupt the rebuild.
|
|
32
|
+
state
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def revert_columns(state, entry)
|
|
37
|
+
entry.previous_values.each_with_object((state || {}).dup) do |(column, previous), result|
|
|
38
|
+
result[column] = Diff.revert(result[column], previous, entry.new_values[column])
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
end
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
require 'time'
|
|
2
|
+
|
|
3
|
+
module ForestAdminAgent
|
|
4
|
+
module AuditTrail
|
|
5
|
+
# Shared by the two capture layers: {Capture} for the changes Forest writes, {ActionCapture} for the
|
|
6
|
+
# smart actions it runs. Holds the write protocol's vocabulary and its failure policy.
|
|
7
|
+
module Recording
|
|
8
|
+
REDACTED = '[redacted]'.freeze
|
|
9
|
+
# A row is inserted before the write and confirmed after it. One left PENDING means the write may or
|
|
10
|
+
# may not have landed — that residue is evidence, and it is the point.
|
|
11
|
+
PENDING = 'pending'.freeze
|
|
12
|
+
DONE = 'done'.freeze
|
|
13
|
+
|
|
14
|
+
IDENTITY = { user_id: :id, user_first_name: :first_name,
|
|
15
|
+
user_last_name: :last_name, user_email: :email }.freeze
|
|
16
|
+
|
|
17
|
+
# Denormalised at write time, so the row says who acted then rather than whoever holds that id today.
|
|
18
|
+
# Read defensively: a caller built by another code path need not carry a full identity, and a
|
|
19
|
+
# NoMethodError here would refuse the write outright under `critical: true`.
|
|
20
|
+
def identity_of(caller)
|
|
21
|
+
IDENTITY.transform_values { |reader| caller.respond_to?(reader) ? caller.public_send(reader) : nil }
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
# Same id for every change made within one request — set on the caller by the agent (see
|
|
25
|
+
# CallerParser), mirroring the Node agent's caller.requestId.
|
|
26
|
+
#
|
|
27
|
+
# nil when the caller carries none, which is what a write outside any request looks like. Inventing one
|
|
28
|
+
# would group the row into a request of its own, indistinguishable from a genuine single-row request.
|
|
29
|
+
def correlation_key_for(caller)
|
|
30
|
+
caller.respond_to?(:request_id) ? caller.request_id : nil
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def redact(values, redacted_fields)
|
|
34
|
+
return values if redacted_fields.empty?
|
|
35
|
+
|
|
36
|
+
values.each_with_object({}) do |(field, value), result|
|
|
37
|
+
result[field] = redacted_fields.include?(field) ? REDACTED : value
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
def now
|
|
42
|
+
Time.now.utc.iso8601(3)
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
# Everything after the pending insert is best-effort: by then the write has happened, so raising would
|
|
46
|
+
# report a failure for an operation that succeeded (and invite a retry that duplicates it). Losing the
|
|
47
|
+
# row is the lesser evil, so it is logged and dropped. Only the pending insert itself can refuse an
|
|
48
|
+
# operation, and only under `critical: true` — see {AuditTrail.critical?}.
|
|
49
|
+
def audit_safely
|
|
50
|
+
yield
|
|
51
|
+
rescue StandardError => e
|
|
52
|
+
AuditTrail.log_failure(e)
|
|
53
|
+
nil
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
end
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
module ForestAdminAgent
|
|
2
|
+
module AuditTrail
|
|
3
|
+
# What a "before" hook leaves for the matching "after" hook: the records as they stood, the patch, and the
|
|
4
|
+
# ids of the pending rows to confirm.
|
|
5
|
+
#
|
|
6
|
+
# Entries are keyed by the object the hook decorator hands to both contexts — the filter, or the data on a
|
|
7
|
+
# create — because taking the newest entry is wrong as soon as writes nest: an inner write that fails skips
|
|
8
|
+
# its after hook and stays on the stack, and the outer hook would then confirm the failed operation's rows
|
|
9
|
+
# as done and leave its own stranded. Both directions of that are lies.
|
|
10
|
+
#
|
|
11
|
+
# An operation raising between the two hooks strands its entry, hence the cap; its rows stay `pending` in
|
|
12
|
+
# the table, which is the truthful state for a write that may not have landed.
|
|
13
|
+
class Snapshots
|
|
14
|
+
include Recording
|
|
15
|
+
|
|
16
|
+
# ponytail: 16 deep is far past any legitimate nesting; raise it if one ever gets that far.
|
|
17
|
+
MAX_PENDING = 16
|
|
18
|
+
|
|
19
|
+
def push(key, snapshot)
|
|
20
|
+
stack = pending
|
|
21
|
+
stack.shift while stack.size >= MAX_PENDING
|
|
22
|
+
stack.push(snapshot.merge(key: key))
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
# The entry this operation left, or nothing rather than someone else's.
|
|
26
|
+
#
|
|
27
|
+
# A customization that replaced the filter or the data leaves no identity to match on: our before hook saw
|
|
28
|
+
# the replacement and the after context carries the original. With a single operation in flight that is
|
|
29
|
+
# unambiguous, so it still pairs; with several it does not guess, and the rows stay pending.
|
|
30
|
+
def pop_for(key)
|
|
31
|
+
stack = pending
|
|
32
|
+
index = stack.rindex { |entry| entry[:key].equal?(key) }
|
|
33
|
+
index = 0 if index.nil? && stack.size == 1
|
|
34
|
+
|
|
35
|
+
index.nil? ? nil : stack.delete_at(index)
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
# The records an operation is about to touch, capped. Reading "delete all" unbounded would materialise
|
|
39
|
+
# every matched row — and the pending/confirm protocol writes each of them twice. Truncation is logged
|
|
40
|
+
# rather than silent: an incomplete audit somebody knows about beats an OOM.
|
|
41
|
+
#
|
|
42
|
+
# Read outside the write's transaction — hooks bracket the write as separate calls and the data layer
|
|
43
|
+
# exposes no lock, on purpose, since it spans ActiveRecord, Mongoid, HTTP APIs. So two updates racing on
|
|
44
|
+
# one record both snapshot the same state, and the one that lands second records a `previous_values` that
|
|
45
|
+
# was already overwritten.
|
|
46
|
+
#
|
|
47
|
+
# An empty list on failure rather than no snapshot at all: the after hook pops unconditionally, so
|
|
48
|
+
# skipping the push would pair it with an unrelated entry. Reading it goes through the gate, not
|
|
49
|
+
# `audit_safely`: knowing what an operation is about to touch is part of being able to record it, so
|
|
50
|
+
# under `critical: true` a snapshot that cannot be read refuses the operation.
|
|
51
|
+
def take(context, projection)
|
|
52
|
+
cap = AuditTrail::MAX_RECORDS_PER_OPERATION
|
|
53
|
+
page = ForestAdminDatasourceToolkit::Components::Query::Page.new(offset: 0, limit: cap + 1)
|
|
54
|
+
records = AuditTrail.gate { context.collection.list(context.filter.override(page: page), projection) } || []
|
|
55
|
+
return records if records.size <= cap
|
|
56
|
+
|
|
57
|
+
refuse_or_truncate(context, records, cap)
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
private
|
|
61
|
+
|
|
62
|
+
# Truncating means the operation writes more records than it audits. Tolerable when the audit trail is
|
|
63
|
+
# advisory; under `critical: true` it breaks the one invariant the mode exists for, so the operation is
|
|
64
|
+
# refused instead — before the write, so there is nothing to repair.
|
|
65
|
+
def refuse_or_truncate(context, records, cap)
|
|
66
|
+
AuditTrail.refuse_over_cap! if AuditTrail.critical?
|
|
67
|
+
|
|
68
|
+
kept = records.first(cap)
|
|
69
|
+
AuditTrail.log_truncation(kept.size, audit_safely { count_matching(context) })
|
|
70
|
+
|
|
71
|
+
kept
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
def count_matching(context)
|
|
75
|
+
aggregation = ForestAdminDatasourceToolkit::Components::Query::Aggregation.new(operation: 'Count')
|
|
76
|
+
|
|
77
|
+
context.collection.aggregate(context.filter, aggregation).first&.fetch('value', nil)
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
def pending
|
|
81
|
+
Thread.current[:forest_audit_trail_snapshots] ||= []
|
|
82
|
+
end
|
|
83
|
+
end
|
|
84
|
+
end
|
|
85
|
+
end
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
begin
|
|
2
|
+
require 'active_record'
|
|
3
|
+
rescue LoadError
|
|
4
|
+
raise LoadError, 'config.audit_trail needs the activerecord gem: add `gem "activerecord"` to your Gemfile.'
|
|
5
|
+
end
|
|
6
|
+
|
|
7
|
+
module ForestAdminAgent
|
|
8
|
+
module AuditTrail
|
|
9
|
+
module Sql
|
|
10
|
+
# Dedicated abstract base so the audit storage keeps its own connection pool, isolated from the
|
|
11
|
+
# host application's ActiveRecord::Base connection. Also the level the `attribute` overrides in
|
|
12
|
+
# AuditLog need: declaring them straight on an ActiveRecord::Base child resolves the type
|
|
13
|
+
# eagerly and blows up before any connection is established.
|
|
14
|
+
class AuditConnectionBase < ActiveRecord::Base
|
|
15
|
+
self.abstract_class = true
|
|
16
|
+
|
|
17
|
+
class << self
|
|
18
|
+
# `establish_connection` is class-level, so two stores pointed at different databases would silently
|
|
19
|
+
# clobber each other's pool and both end up writing to whichever connected last. One audit database
|
|
20
|
+
# per agent is the supported shape; a second, different one is a configuration mistake worth hearing
|
|
21
|
+
# about at boot.
|
|
22
|
+
#
|
|
23
|
+
# Under one mutex for all stores, not one each: the check, the connect and the assignment have to be
|
|
24
|
+
# one step, or two stores connecting at once both pass the check and the loser writes to the winner's
|
|
25
|
+
# database.
|
|
26
|
+
def connect_to(database)
|
|
27
|
+
connection_mutex.synchronize do
|
|
28
|
+
if @database && @database != database
|
|
29
|
+
raise ForestAdminDatasourceToolkit::Exceptions::ForestException,
|
|
30
|
+
'The audit trail is already connected to another database. One agent, one audit database.'
|
|
31
|
+
end
|
|
32
|
+
next if @database
|
|
33
|
+
|
|
34
|
+
establish_connection(database)
|
|
35
|
+
@database = database
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def disconnect!
|
|
40
|
+
connection_mutex.synchronize do
|
|
41
|
+
@database = nil
|
|
42
|
+
remove_connection
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
def connection_mutex
|
|
47
|
+
@connection_mutex ||= Mutex.new
|
|
48
|
+
end
|
|
49
|
+
end
|
|
50
|
+
end
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
end
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
module ForestAdminAgent
|
|
2
|
+
module AuditTrail
|
|
3
|
+
module Sql
|
|
4
|
+
# Abstract template for the audit model. Each Store builds its own concrete subclass bound to its
|
|
5
|
+
# own (schema-qualified) table, so stores with different `table_name`/`schema` can't clobber a
|
|
6
|
+
# shared one. The JSON attribute overrides force Hash <-> JSON casting on every adapter (Postgres
|
|
7
|
+
# json, or text on SQLite) and are inherited by every subclass.
|
|
8
|
+
class AuditLog < AuditConnectionBase
|
|
9
|
+
self.abstract_class = true
|
|
10
|
+
|
|
11
|
+
attribute :previous_values, :json
|
|
12
|
+
attribute :new_values, :json
|
|
13
|
+
end
|
|
14
|
+
end
|
|
15
|
+
end
|
|
16
|
+
end
|