forest_admin_agent 1.38.2 → 1.39.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/AUDIT_TRAIL.md +347 -0
- data/lib/forest_admin_agent/audit_trail/action_capture.rb +99 -0
- data/lib/forest_admin_agent/audit_trail/audit_record.rb +16 -0
- data/lib/forest_admin_agent/audit_trail/capture.rb +229 -0
- data/lib/forest_admin_agent/audit_trail/diff.rb +156 -0
- data/lib/forest_admin_agent/audit_trail/record_state.rb +43 -0
- data/lib/forest_admin_agent/audit_trail/recording.rb +57 -0
- data/lib/forest_admin_agent/audit_trail/snapshots.rb +85 -0
- data/lib/forest_admin_agent/audit_trail/sql/audit_connection_base.rb +53 -0
- data/lib/forest_admin_agent/audit_trail/sql/audit_log.rb +16 -0
- data/lib/forest_admin_agent/audit_trail/sql/field_filter.rb +58 -0
- data/lib/forest_admin_agent/audit_trail/sql/migrations.rb +53 -0
- data/lib/forest_admin_agent/audit_trail/sql/migrator.rb +117 -0
- data/lib/forest_admin_agent/audit_trail/sql/text_search.rb +78 -0
- data/lib/forest_admin_agent/audit_trail/store.rb +253 -0
- data/lib/forest_admin_agent/audit_trail.rb +68 -0
- data/lib/forest_admin_agent/builder/agent_factory.rb +20 -0
- data/lib/forest_admin_agent/http/correlation_id.rb +37 -0
- data/lib/forest_admin_agent/http/correlation_id_middleware.rb +29 -0
- data/lib/forest_admin_agent/http/router.rb +6 -0
- data/lib/forest_admin_agent/routes/action/actions.rb +76 -1
- data/lib/forest_admin_agent/routes/capabilities/collections.rb +11 -1
- data/lib/forest_admin_agent/routes/resources/audit_trail.rb +237 -0
- data/lib/forest_admin_agent/routes/resources/audit_trail_correlation.rb +101 -0
- data/lib/forest_admin_agent/routes/resources/audit_trail_route.rb +115 -0
- data/lib/forest_admin_agent/routes/resources/list.rb +1 -1
- data/lib/forest_admin_agent/routes/resources/related/list_related.rb +1 -1
- data/lib/forest_admin_agent/routes/resources/show.rb +1 -1
- data/lib/forest_admin_agent/serializer/forest_serializer_override.rb +14 -13
- data/lib/forest_admin_agent/utils/caller_parser.rb +4 -0
- data/lib/forest_admin_agent/utils/query_string_parser.rb +59 -10
- data/lib/forest_admin_agent/utils/schema/schema_emitter.rb +1 -1
- data/lib/forest_admin_agent/version.rb +1 -1
- data/lib/forest_admin_agent.rb +4 -0
- metadata +23 -2
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
module ForestAdminAgent
|
|
2
|
+
module AuditTrail
|
|
3
|
+
module Sql
|
|
4
|
+
# SQL keeping only the audit entries whose diff touched one of the given fields. Both JSON columns are
|
|
5
|
+
# searched: a field the change added exists in `new_values` only, one it removed in `previous_values`
|
|
6
|
+
# only.
|
|
7
|
+
#
|
|
8
|
+
# The test is per adapter, and a field name is always a whole key — never a path — so a name holding a
|
|
9
|
+
# dot (`address.city`) has to be quoted or the database reads it as a traversal.
|
|
10
|
+
class FieldFilter
|
|
11
|
+
COLUMNS = %w[previous_values new_values].freeze
|
|
12
|
+
|
|
13
|
+
def initialize(connection)
|
|
14
|
+
@connection = connection
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def condition(fields)
|
|
18
|
+
adapter = @connection.adapter_name.downcase
|
|
19
|
+
|
|
20
|
+
case adapter
|
|
21
|
+
when /postgres/ then COLUMNS.map { |column| postgres_has_key(column, fields) }.join(' OR ')
|
|
22
|
+
when /sqlite/ then json_paths(fields) { |column, path| "json_type(#{column}, #{path}) IS NOT NULL" }
|
|
23
|
+
when /mysql|maria/ then mysql_has_keys(fields)
|
|
24
|
+
else
|
|
25
|
+
raise ForestAdminDatasourceToolkit::Exceptions::ForestException,
|
|
26
|
+
"Filtering the audit trail by field is not supported on #{adapter}"
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
private
|
|
31
|
+
|
|
32
|
+
# `jsonb_object_keys` rather than the `?|` operator: `?` is a bind placeholder for ActiveRecord and
|
|
33
|
+
# the function form needs no escaping. The column is `json`, hence the cast.
|
|
34
|
+
def postgres_has_key(column, fields)
|
|
35
|
+
list = fields.map { |field| @connection.quote(field) }.join(', ')
|
|
36
|
+
|
|
37
|
+
"EXISTS (SELECT 1 FROM jsonb_object_keys(#{column}::jsonb) AS key WHERE key IN (#{list}))"
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
# `json_type` and not `json_extract`: a key holding a JSON null extracts as SQL NULL, which would
|
|
41
|
+
# read as "no such key".
|
|
42
|
+
def json_paths(fields)
|
|
43
|
+
COLUMNS.flat_map { |column| fields.map { |field| yield(column, json_path(field)) } }.join(' OR ')
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
def mysql_has_keys(fields)
|
|
47
|
+
paths = fields.map { |field| json_path(field) }.join(', ')
|
|
48
|
+
|
|
49
|
+
COLUMNS.map { |column| "JSON_CONTAINS_PATH(#{column}, 'one', #{paths})" }.join(' OR ')
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
def json_path(field)
|
|
53
|
+
@connection.quote(%($."#{field.to_s.gsub(/["\\]/) { |char| "\\#{char}" }}"))
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
end
|
|
58
|
+
end
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
module ForestAdminAgent
|
|
2
|
+
module AuditTrail
|
|
3
|
+
module Sql
|
|
4
|
+
# The audit table's schema, as an ordered, append-only list. Nothing has shipped yet, so this is still a
|
|
5
|
+
# single entry; once a release is out, never edit one — add another, since a database out there has
|
|
6
|
+
# already recorded the earlier ones as applied. Every statement tolerates being replayed
|
|
7
|
+
# (`if_not_exists`), which is what makes a lost race harmless.
|
|
8
|
+
module Migrations
|
|
9
|
+
ALL = [
|
|
10
|
+
{
|
|
11
|
+
name: '001-create-audit-logs',
|
|
12
|
+
up: lambda do |connection, table|
|
|
13
|
+
# if_not_exists: a non-PG race (no advisory lock) can let two instances both reach here.
|
|
14
|
+
connection.create_table(table, if_not_exists: true) do |t|
|
|
15
|
+
t.datetime :timestamp, null: false
|
|
16
|
+
t.string :operation, null: false
|
|
17
|
+
t.string :collection, null: false
|
|
18
|
+
# Nullable, because a create's pending row has no id yet; text, because a packed composite id
|
|
19
|
+
# outgrows a varchar.
|
|
20
|
+
t.text :record_id
|
|
21
|
+
# No default on purpose: every write sets it, so a row arriving without one is a bug worth
|
|
22
|
+
# hearing about rather than a row that quietly claims to be done.
|
|
23
|
+
t.string :status, null: false
|
|
24
|
+
t.integer :user_id
|
|
25
|
+
# Denormalised from the caller at write time: who acted then, not whoever holds that id today.
|
|
26
|
+
t.text :user_first_name
|
|
27
|
+
t.text :user_last_name
|
|
28
|
+
t.text :user_email
|
|
29
|
+
# Set only on an update that moved a writable primary key: the id the row was filed under
|
|
30
|
+
# before. What lets a history query follow a record across a rename.
|
|
31
|
+
t.text :previous_record_id
|
|
32
|
+
# Smart-action rows only.
|
|
33
|
+
t.text :action_name
|
|
34
|
+
t.string :correlation_key
|
|
35
|
+
t.json :previous_values
|
|
36
|
+
t.json :new_values
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
base = table.split('.').last
|
|
40
|
+
# MySQL cannot index unbounded TEXT, so that one index needs a length prefix.
|
|
41
|
+
record_id_index = { name: "#{base}_record_id", if_not_exists: true }
|
|
42
|
+
record_id_index[:length] = 255 if connection.adapter_name.downcase.match?(/mysql|maria/)
|
|
43
|
+
|
|
44
|
+
connection.add_index(table, :record_id, **record_id_index)
|
|
45
|
+
connection.add_index(table, :correlation_key, name: "#{base}_correlation_key", if_not_exists: true)
|
|
46
|
+
connection.add_index(table, :user_id, name: "#{base}_user_id", if_not_exists: true)
|
|
47
|
+
end
|
|
48
|
+
}
|
|
49
|
+
].freeze
|
|
50
|
+
end
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
end
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
module ForestAdminAgent
|
|
2
|
+
module AuditTrail
|
|
3
|
+
module Sql
|
|
4
|
+
# Applies {Migrations::ALL} to the audit table, tracking what has run in a companion table named after
|
|
5
|
+
# it — `audit_logs_migration` beside `audit_logs` (both namespaced in the `forest` schema on Postgres).
|
|
6
|
+
# One tracker per audited table, so two stores configured with different `table_name`s each get their
|
|
7
|
+
# own schema history instead of reading each other's as done.
|
|
8
|
+
#
|
|
9
|
+
# On Postgres the migrations run inside a transaction-scoped advisory lock, so several agent
|
|
10
|
+
# instances booting at once apply them one after another instead of racing on the same DDL. The
|
|
11
|
+
# schema is created (and committed) first, made idempotent (CREATE SCHEMA IF NOT EXISTS +
|
|
12
|
+
# tolerating a concurrent create), because the lock cannot cover a not-yet-existing schema.
|
|
13
|
+
class Migrator
|
|
14
|
+
# Arbitrary but stable key pair identifying the audit-trail migration critical section.
|
|
15
|
+
ADVISORY_LOCK = [0x464f, 0x5254].freeze # "FO", "RT"
|
|
16
|
+
# duplicate_schema, and the unique violation on pg_namespace the same race can raise instead.
|
|
17
|
+
DUPLICATE_SCHEMA_STATES = %w[42P06 23505].freeze
|
|
18
|
+
|
|
19
|
+
def initialize(connection, schema:, table_name:)
|
|
20
|
+
@connection = connection
|
|
21
|
+
@schema = schema # nil on adapters without schema support
|
|
22
|
+
@table_name = table_name
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def run
|
|
26
|
+
ensure_schema
|
|
27
|
+
|
|
28
|
+
if postgres?
|
|
29
|
+
@connection.transaction do
|
|
30
|
+
@connection.execute("SELECT pg_advisory_xact_lock(#{ADVISORY_LOCK[0]}, #{ADVISORY_LOCK[1]})")
|
|
31
|
+
apply_pending
|
|
32
|
+
end
|
|
33
|
+
else
|
|
34
|
+
apply_pending
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
private
|
|
39
|
+
|
|
40
|
+
def postgres?
|
|
41
|
+
@connection.adapter_name.downcase.include?('postgres')
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
def schema?
|
|
45
|
+
postgres? && @schema.present?
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
def qualified(name)
|
|
49
|
+
schema? ? "#{@schema}.#{name}" : name
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
# Create the schema first and commit it: the migrations open DDL on the same connection, and a
|
|
53
|
+
# CREATE SCHEMA still pending in the lock transaction would not be visible to them.
|
|
54
|
+
def ensure_schema
|
|
55
|
+
return unless schema?
|
|
56
|
+
|
|
57
|
+
@connection.execute("CREATE SCHEMA IF NOT EXISTS #{@connection.quote_schema_name(@schema)}")
|
|
58
|
+
rescue ActiveRecord::RecordNotUnique
|
|
59
|
+
# 23505 on pg_namespace, already mapped to its own class by ActiveRecord: another instance
|
|
60
|
+
# created the schema between our IF NOT EXISTS check and the create itself.
|
|
61
|
+
nil
|
|
62
|
+
rescue ActiveRecord::StatementInvalid => e
|
|
63
|
+
raise unless duplicate_schema?(e)
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
# By SQLSTATE where the adapter exposes one, so an unrelated failure (no permission to create a
|
|
67
|
+
# schema, say) is not read as a lost race just because its message says "exists".
|
|
68
|
+
def duplicate_schema?(error)
|
|
69
|
+
state = sql_state(error)
|
|
70
|
+
return DUPLICATE_SCHEMA_STATES.include?(state) if state
|
|
71
|
+
|
|
72
|
+
/already exists|duplicate/i.match?(error.message)
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
def sql_state(error)
|
|
76
|
+
cause = error.cause
|
|
77
|
+
return nil unless cause.respond_to?(:result) && defined?(PG::Result)
|
|
78
|
+
|
|
79
|
+
cause.result.error_field(PG::Result::PG_DIAG_SQLSTATE)
|
|
80
|
+
rescue StandardError
|
|
81
|
+
nil
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
def apply_pending
|
|
85
|
+
done = applied_migrations
|
|
86
|
+
table = qualified(@table_name)
|
|
87
|
+
|
|
88
|
+
Migrations::ALL.each do |migration|
|
|
89
|
+
next if done.include?(migration[:name])
|
|
90
|
+
|
|
91
|
+
migration[:up].call(@connection, table)
|
|
92
|
+
@connection.execute(
|
|
93
|
+
"INSERT INTO #{@connection.quote_table_name(migrations_table)} (name) " \
|
|
94
|
+
"VALUES (#{@connection.quote(migration[:name])})"
|
|
95
|
+
)
|
|
96
|
+
end
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
def applied_migrations
|
|
100
|
+
ensure_migrations_table
|
|
101
|
+
|
|
102
|
+
@connection.select_values("SELECT name FROM #{@connection.quote_table_name(migrations_table)}")
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
def ensure_migrations_table
|
|
106
|
+
@connection.create_table(migrations_table, id: false, if_not_exists: true) do |t|
|
|
107
|
+
t.string :name, null: false
|
|
108
|
+
end
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
def migrations_table
|
|
112
|
+
qualified("#{@table_name}_migration")
|
|
113
|
+
end
|
|
114
|
+
end
|
|
115
|
+
end
|
|
116
|
+
end
|
|
117
|
+
end
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
require 'json'
|
|
2
|
+
|
|
3
|
+
module ForestAdminAgent
|
|
4
|
+
module AuditTrail
|
|
5
|
+
module Sql
|
|
6
|
+
# SQL keeping only the audit entries a free-text term matches, case-insensitively and as a substring:
|
|
7
|
+
# the action's name, who acted, and the keys and values recorded on both sides of the change — at any
|
|
8
|
+
# depth, since only the changed leaves of a JSON column are stored and the term has to reach them.
|
|
9
|
+
#
|
|
10
|
+
# Deliberately not searched: operation, correlation_key, record_id, collection, status and timestamp.
|
|
11
|
+
# Machine identifiers nobody searches for, and matching them turns one term into a pile of confusing
|
|
12
|
+
# hits.
|
|
13
|
+
#
|
|
14
|
+
# The values are matched against the JSON document as text, which is what lets one condition reach any
|
|
15
|
+
# depth and compose with pagination and the count. It cannot use an index, which is affordable here
|
|
16
|
+
# because a history query is already narrowed to one record.
|
|
17
|
+
class TextSearch
|
|
18
|
+
TEXT_COLUMNS = %w[action_name user_first_name user_last_name user_email].freeze
|
|
19
|
+
JSON_COLUMNS = %w[previous_values new_values].freeze
|
|
20
|
+
# `!` rather than a backslash: MySQL treats a backslash as an escape inside string literals too, so
|
|
21
|
+
# `ESCAPE '\'` needs doubling there and nowhere else.
|
|
22
|
+
ESCAPE = '!'.freeze
|
|
23
|
+
# A masked value is stored as this. It is removed before matching, so a search for "redacted" cannot
|
|
24
|
+
# hit it — and since the real value was never recorded, searching that finds nothing either. A search
|
|
25
|
+
# must never confirm a value the trail refused to keep.
|
|
26
|
+
REDACTED = Recording::REDACTED
|
|
27
|
+
|
|
28
|
+
def initialize(connection)
|
|
29
|
+
@connection = connection
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def condition(term)
|
|
33
|
+
text = term.to_s.downcase
|
|
34
|
+
# The value objects are matched as serialized JSON, where a quote, a backslash or a newline is
|
|
35
|
+
# escaped — so `15" monitor` sits in the document as `15\" monitor` and the raw term would never
|
|
36
|
+
# find it. Escaping the term the same way makes it match, and stops a bare quote from matching the
|
|
37
|
+
# document's own structure.
|
|
38
|
+
clauses = TEXT_COLUMNS.map { |column| like(column, text) }
|
|
39
|
+
clauses += JSON_COLUMNS.map { |column| like(searchable_json(column), json_escaped(text)) }
|
|
40
|
+
|
|
41
|
+
clauses.join(' OR ')
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
private
|
|
45
|
+
|
|
46
|
+
def like(expression, text)
|
|
47
|
+
"LOWER(#{expression}) LIKE #{@connection.quote("%#{escape(text)}%")} ESCAPE '#{ESCAPE}'"
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
# What JSON generation would have done to the term: `to_json` on the string, minus its own quotes.
|
|
51
|
+
def json_escaped(text)
|
|
52
|
+
text.to_json[1..-2]
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
def searchable_json(column)
|
|
56
|
+
"REPLACE(#{as_text(column)}, #{@connection.quote(REDACTED)}, '')"
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
def as_text(column)
|
|
60
|
+
adapter = @connection.adapter_name.downcase
|
|
61
|
+
|
|
62
|
+
case adapter
|
|
63
|
+
when /postgres/ then "#{column}::text"
|
|
64
|
+
when /sqlite/ then column
|
|
65
|
+
when /mysql|maria/ then "CAST(#{column} AS CHAR)"
|
|
66
|
+
else
|
|
67
|
+
raise ForestAdminDatasourceToolkit::Exceptions::ForestException,
|
|
68
|
+
"Searching the audit trail is not supported on #{adapter}"
|
|
69
|
+
end
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
def escape(term)
|
|
73
|
+
term.gsub(/[!%_]/) { |char| "#{ESCAPE}#{char}" }
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
end
|
|
77
|
+
end
|
|
78
|
+
end
|
|
@@ -0,0 +1,253 @@
|
|
|
1
|
+
require 'time'
|
|
2
|
+
|
|
3
|
+
module ForestAdminAgent
|
|
4
|
+
module AuditTrail
|
|
5
|
+
# SQL-backed storage that both writes every audited change and reads the per-record history back.
|
|
6
|
+
#
|
|
7
|
+
# {#connect!} opens the connection and migrates; the agent factory calls it at boot rather than leaving it
|
|
8
|
+
# to the first write, so a database the agent cannot reach is a startup failure instead of an agent that
|
|
9
|
+
# looks healthy while recording nothing — and, under `critical: true`, instead of one that refuses every
|
|
10
|
+
# write the moment somebody first tries to save something.
|
|
11
|
+
class Store
|
|
12
|
+
DEFAULT_SCHEMA = 'forest'.freeze
|
|
13
|
+
DEFAULT_TABLE = 'audit_logs'.freeze
|
|
14
|
+
COLUMNS = %i[timestamp operation collection record_id previous_record_id status user_id user_first_name
|
|
15
|
+
user_last_name user_email action_name correlation_key previous_values new_values].freeze
|
|
16
|
+
AUTHOR_COLUMNS = %i[user_id user_first_name user_last_name user_email].freeze
|
|
17
|
+
|
|
18
|
+
def initialize(database:, schema: DEFAULT_SCHEMA, table_name: DEFAULT_TABLE)
|
|
19
|
+
@database = database
|
|
20
|
+
@schema = schema
|
|
21
|
+
@table_name = table_name
|
|
22
|
+
@mutex = Mutex.new
|
|
23
|
+
@ready = false
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def connect!
|
|
27
|
+
ensure_ready
|
|
28
|
+
|
|
29
|
+
self
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def append(record)
|
|
33
|
+
append_all([record]).first
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
# Inserts rows and returns their ids, in the order given. Batched, because a "delete all" snapshot can
|
|
37
|
+
# be thousands of records and the pending/confirm protocol writes each of them twice.
|
|
38
|
+
#
|
|
39
|
+
# The ids are matched to their rows by `record_id` rather than by the order RETURNING happens to come
|
|
40
|
+
# back in, which Postgres does not promise: pairing them positionally would confirm each pending row with
|
|
41
|
+
# another record's diff. One row per record per operation, so that key is unique within a batch — bar a
|
|
42
|
+
# pending create, which has no id yet and is always a batch of one.
|
|
43
|
+
def append_all(records)
|
|
44
|
+
return [] if records.empty?
|
|
45
|
+
|
|
46
|
+
rows = records.map { |record| to_row(record) }
|
|
47
|
+
return rows.map { |row| model.create!(row).id } unless batch_returning?(rows)
|
|
48
|
+
|
|
49
|
+
returned = model.insert_all(rows, returning: %i[id record_id]).rows.to_h { |id, key| [key, id] }
|
|
50
|
+
|
|
51
|
+
rows.map { |row| returned[row[:record_id]] }
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
# One insert per row when the ids cannot be matched back: no RETURNING on this adapter (MySQL), or a
|
|
55
|
+
# batch whose record ids are not distinct enough to pair on.
|
|
56
|
+
def batch_returning?(rows)
|
|
57
|
+
return false unless model.connection.supports_insert_returning?
|
|
58
|
+
|
|
59
|
+
keys = rows.map { |row| row[:record_id] }
|
|
60
|
+
|
|
61
|
+
keys.none?(&:nil?) && keys.uniq.size == keys.size
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
def confirm(id, attributes)
|
|
65
|
+
row = model.find_by(id: id)
|
|
66
|
+
|
|
67
|
+
row&.update!(**attributes, status: Recording::DONE)
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
# A write that turned out to change nothing leaves no trace: the pending row goes rather than sitting
|
|
71
|
+
# there implying the write is unaccounted for.
|
|
72
|
+
def discard(ids)
|
|
73
|
+
model.where(id: ids).delete_all unless ids.empty?
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
def list_by_record(collection:, record_id:, skip: 0, limit: nil, user_ids: nil, start_timestamp: nil,
|
|
77
|
+
end_timestamp: nil, fields: nil, search: nil, order: 'asc')
|
|
78
|
+
# `id` (insertion order) breaks ties on equal timestamps in both directions, keeping pages
|
|
79
|
+
# deterministic and stable.
|
|
80
|
+
relation = scope(collection, record_id, user_ids: user_ids, start_timestamp: start_timestamp,
|
|
81
|
+
end_timestamp: end_timestamp, fields: fields, search: search)
|
|
82
|
+
.order(timestamp: order.to_s == 'desc' ? :desc : :asc, id: :asc)
|
|
83
|
+
.offset(skip || 0)
|
|
84
|
+
relation = relation.limit(limit) unless limit.nil?
|
|
85
|
+
|
|
86
|
+
relation.map { |row| from_row(row) }
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
def count_by_record(collection:, record_id:, user_ids: nil, start_timestamp: nil,
|
|
90
|
+
end_timestamp: nil, fields: nil, search: nil)
|
|
91
|
+
scope(collection, record_id, user_ids: user_ids, start_timestamp: start_timestamp,
|
|
92
|
+
end_timestamp: end_timestamp, fields: fields, search: search).count
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
# The distinct authors of the entries the current filters match, whatever page is being asked for. The
|
|
96
|
+
# identity comes from the rows themselves, so a user who has since been renamed or removed still reads
|
|
97
|
+
# as they were when they acted.
|
|
98
|
+
def authors_by_record(collection:, record_id:, user_ids: nil, start_timestamp: nil,
|
|
99
|
+
end_timestamp: nil, fields: nil, search: nil)
|
|
100
|
+
scope(collection, record_id, user_ids: user_ids, start_timestamp: start_timestamp,
|
|
101
|
+
end_timestamp: end_timestamp, fields: fields, search: search)
|
|
102
|
+
.where.not(user_id: nil)
|
|
103
|
+
.distinct
|
|
104
|
+
.pluck(*AUTHOR_COLUMNS)
|
|
105
|
+
.map { |values| AUTHOR_COLUMNS.zip(values).to_h }
|
|
106
|
+
.uniq { |author| author[:user_id] }
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
# The ids this record was renamed from, each with the moment it stopped being that id. Walking those back
|
|
110
|
+
# is what lets a history query reach rows written before a rename — they stay under the id they were
|
|
111
|
+
# written with, since that is the id they were true of — and the moment bounds how far: the id it left may
|
|
112
|
+
# have been taken by another record afterwards, whose rows are none of this record's business.
|
|
113
|
+
def renamed_from(collection:, record_id:)
|
|
114
|
+
model.where(collection: collection, record_id: record_id)
|
|
115
|
+
.where.not(previous_record_id: nil)
|
|
116
|
+
.pluck(:previous_record_id, :timestamp, :id)
|
|
117
|
+
.group_by(&:first)
|
|
118
|
+
.map do |id, rows|
|
|
119
|
+
# The row id comes along as the tie-breaker: the trail orders itself by (timestamp, id), so a
|
|
120
|
+
# bound that knew only the timestamp would mean something slightly different from "before".
|
|
121
|
+
_, at, row = rows.max_by { |(_, timestamp, row_id)| [timestamp, row_id] }
|
|
122
|
+
|
|
123
|
+
{ id: id, until: as_iso(at), until_row: row }
|
|
124
|
+
end
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
# Entries recorded strictly after `timestamp`, newest first: what a state reconstruction has to undo.
|
|
128
|
+
# Strictly after, so an entry stamped exactly at the requested instant counts as part of that state
|
|
129
|
+
# instead of being reverted out of it.
|
|
130
|
+
# Confirmed rows only: a pending one records an attempt whose outcome is unknown, and undoing a change
|
|
131
|
+
# that may never have happened would invent a state the record was never in. The history reads keep
|
|
132
|
+
# pending rows — they are evidence, and `status` tells the reader what they are — but a reconstruction
|
|
133
|
+
# cannot act on them.
|
|
134
|
+
def list_since(collection:, record_id:, timestamp:)
|
|
135
|
+
model.where(collection: collection, status: Recording::DONE)
|
|
136
|
+
.where(*segments_condition(record_id))
|
|
137
|
+
.where('timestamp > ?', as_time(timestamp))
|
|
138
|
+
.order(timestamp: :desc, id: :desc)
|
|
139
|
+
.map { |row| from_row(row) }
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
def list_by_correlation(collection:, record_id:, correlation_key:)
|
|
143
|
+
list_by_correlations(collection: collection, record_id: record_id, correlation_keys: [correlation_key])
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
def list_by_correlations(collection:, record_id:, correlation_keys:)
|
|
147
|
+
return [] if correlation_keys.empty?
|
|
148
|
+
|
|
149
|
+
model.where(collection: collection, record_id: record_id, correlation_key: correlation_keys)
|
|
150
|
+
.order(:timestamp, :id)
|
|
151
|
+
.map { |row| from_row(row) }
|
|
152
|
+
end
|
|
153
|
+
|
|
154
|
+
private
|
|
155
|
+
|
|
156
|
+
# Every filter is an AND, so the count matches exactly what a page of this history holds.
|
|
157
|
+
def scope(collection, record_id, user_ids: nil, start_timestamp: nil, end_timestamp: nil,
|
|
158
|
+
fields: nil, search: nil)
|
|
159
|
+
relation = model.where(collection: collection).where(*segments_condition(record_id))
|
|
160
|
+
relation = relation.where(user_id: user_ids) if user_ids
|
|
161
|
+
relation = relation.where(Sql::FieldFilter.new(model.connection).condition(fields)) if fields&.any?
|
|
162
|
+
relation = relation.where(Sql::TextSearch.new(model.connection).condition(search)) if search
|
|
163
|
+
# Compare as Time so ActiveRecord casts the bound to the datetime column's storage format
|
|
164
|
+
# (raw ISO strings with a `Z` would compare lexically against the cast rows and never match).
|
|
165
|
+
relation = relation.where('timestamp >= ?', as_time(start_timestamp)) if start_timestamp
|
|
166
|
+
relation = relation.where('timestamp <= ?', as_time(end_timestamp)) if end_timestamp
|
|
167
|
+
relation
|
|
168
|
+
end
|
|
169
|
+
|
|
170
|
+
def as_time(value)
|
|
171
|
+
value.is_a?(::Time) ? value : ::Time.iso8601(value.to_s)
|
|
172
|
+
end
|
|
173
|
+
|
|
174
|
+
def as_iso(value)
|
|
175
|
+
value.respond_to?(:iso8601) ? value.iso8601(3) : value.to_s
|
|
176
|
+
end
|
|
177
|
+
|
|
178
|
+
# One record's history is its current id plus every id it was renamed from, each earlier one only up to
|
|
179
|
+
# the rename: `record_id = '7' OR (record_id = '1' AND timestamp <= …)`. A plain `IN` would hand over the
|
|
180
|
+
# rows of whichever record holds that id now.
|
|
181
|
+
#
|
|
182
|
+
# Takes an id, several, or segments — `{ id:, until: }` — so a caller that has no rename to care about
|
|
183
|
+
# simply passes the id.
|
|
184
|
+
def segments_condition(record_id)
|
|
185
|
+
binds = []
|
|
186
|
+
sql = Array(record_id).map { |value| value.is_a?(Hash) ? value : { id: value, until: nil } }.map do |segment|
|
|
187
|
+
binds << segment[:id]
|
|
188
|
+
next 'record_id = ?' unless segment[:until]
|
|
189
|
+
|
|
190
|
+
binds << as_time(segment[:until])
|
|
191
|
+
# Same millisecond as the rename, and the id says which side of it a row falls on: another record
|
|
192
|
+
# taking the abandoned key that fast would otherwise land in this record's history.
|
|
193
|
+
next '(record_id = ? AND timestamp <= ?)' unless segment[:until_row]
|
|
194
|
+
|
|
195
|
+
binds << as_time(segment[:until]) << segment[:until_row]
|
|
196
|
+
'(record_id = ? AND (timestamp < ? OR (timestamp = ? AND id <= ?)))'
|
|
197
|
+
end
|
|
198
|
+
|
|
199
|
+
[sql.join(' OR '), *binds]
|
|
200
|
+
end
|
|
201
|
+
|
|
202
|
+
def model
|
|
203
|
+
ensure_ready
|
|
204
|
+
@model
|
|
205
|
+
end
|
|
206
|
+
|
|
207
|
+
def ensure_ready
|
|
208
|
+
return if @ready
|
|
209
|
+
|
|
210
|
+
@mutex.synchronize do
|
|
211
|
+
return if @ready
|
|
212
|
+
|
|
213
|
+
Sql::AuditConnectionBase.connect_to(@database)
|
|
214
|
+
connection = Sql::AuditConnectionBase.connection
|
|
215
|
+
Sql::Migrator.new(connection, schema: schema_for(connection), table_name: @table_name).run
|
|
216
|
+
@model = build_model(qualified(connection))
|
|
217
|
+
@ready = true
|
|
218
|
+
end
|
|
219
|
+
end
|
|
220
|
+
|
|
221
|
+
# A per-instance concrete subclass bound to this store's own table, so distinct stores can't
|
|
222
|
+
# clobber each other's table name. reset_column_information drops stale metadata for the table
|
|
223
|
+
# the migration just created/evolved.
|
|
224
|
+
def build_model(table)
|
|
225
|
+
Class.new(Sql::AuditLog) { self.table_name = table }.tap(&:reset_column_information)
|
|
226
|
+
end
|
|
227
|
+
|
|
228
|
+
def schema_for(connection)
|
|
229
|
+
connection.adapter_name.downcase.include?('postgres') ? @schema : nil
|
|
230
|
+
end
|
|
231
|
+
|
|
232
|
+
def qualified(connection)
|
|
233
|
+
schema = schema_for(connection)
|
|
234
|
+
schema ? "#{schema}.#{@table_name}" : @table_name
|
|
235
|
+
end
|
|
236
|
+
|
|
237
|
+
# An AuditRecord is a Struct and a row answers to `[]` too, so the mapping is the column list itself.
|
|
238
|
+
def to_row(record)
|
|
239
|
+
COLUMNS.to_h { |column| [column, record[column]] }
|
|
240
|
+
end
|
|
241
|
+
|
|
242
|
+
def from_row(row)
|
|
243
|
+
values = COLUMNS.to_h { |column| [column, row[column]] }
|
|
244
|
+
values[:id] = row.id
|
|
245
|
+
values[:timestamp] = row.timestamp.respond_to?(:iso8601) ? row.timestamp.iso8601(3) : row.timestamp.to_s
|
|
246
|
+
values[:previous_values] ||= {}
|
|
247
|
+
values[:new_values] ||= {}
|
|
248
|
+
|
|
249
|
+
AuditRecord.new(**values)
|
|
250
|
+
end
|
|
251
|
+
end
|
|
252
|
+
end
|
|
253
|
+
end
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
module ForestAdminAgent
|
|
2
|
+
# The audit trail is inert unless `config.audit_trail[:database]` was set: the agent factory builds the
|
|
3
|
+
# store during setup — connecting and migrating there rather than on first write — and everything (capture
|
|
4
|
+
# layers and routes) resolves it from here.
|
|
5
|
+
module AuditTrail
|
|
6
|
+
# One operation must not materialise an unbounded number of records: a "delete all" would otherwise read
|
|
7
|
+
# every matched row and, with the pending/confirm protocol, write each of them twice. Truncation is logged,
|
|
8
|
+
# never silent.
|
|
9
|
+
#
|
|
10
|
+
# Matches the Node agent's `MAX_SNAPSHOT_RECORDS`: the same feature behind the same config key, so a bulk
|
|
11
|
+
# operation must not be audited on one agent and truncated on the other. Change it in both or neither.
|
|
12
|
+
MAX_RECORDS_PER_OPERATION = 1000
|
|
13
|
+
|
|
14
|
+
# Auditing a subset while the write touches every match is the one thing `critical` exists to prevent, so
|
|
15
|
+
# over the cap the operation is refused instead — before anything is written, in both the write and the
|
|
16
|
+
# action path, which is why the message lives here rather than in either of them.
|
|
17
|
+
def self.refuse_over_cap!
|
|
18
|
+
raise ForestAdminDatasourceToolkit::Exceptions::ForestException,
|
|
19
|
+
'The audit trail is configured as critical and cannot record an operation touching more than ' \
|
|
20
|
+
"#{MAX_RECORDS_PER_OPERATION} records at once. Narrow the selection."
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def self.log_truncation(kept, total)
|
|
24
|
+
skipped = total ? total - kept : 'further'
|
|
25
|
+
|
|
26
|
+
Facades::Container.logger.log(
|
|
27
|
+
'Warn',
|
|
28
|
+
"[ForestAdmin] Audit trail: #{kept} records audited, #{skipped} skipped " \
|
|
29
|
+
"(cap #{MAX_RECORDS_PER_OPERATION} per operation)"
|
|
30
|
+
)
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def self.options
|
|
34
|
+
config = Facades::Container.config_from_cache
|
|
35
|
+
|
|
36
|
+
(config && config[:audit_trail]) || {}
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def self.store
|
|
40
|
+
options[:store]
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
# `critical: true` makes the pending insert a precondition of the write: if the audit trail cannot record
|
|
44
|
+
# that an operation is about to happen, the operation is refused. Nothing was written, so there is nothing
|
|
45
|
+
# to repair and no compensating write ever happens. Default false keeps today's behaviour, where a broken
|
|
46
|
+
# audit database costs rows rather than writes.
|
|
47
|
+
def self.critical?
|
|
48
|
+
options[:critical] == true
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
def self.log_failure(error)
|
|
52
|
+
Facades::Container.logger.log('Error', "[ForestAdmin] Audit trail unavailable, skipping: #{error.message}")
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
# Runs the pending insert under the configured policy: refusing the operation when critical, logging and
|
|
56
|
+
# carrying on otherwise.
|
|
57
|
+
def self.gate
|
|
58
|
+
return yield if critical?
|
|
59
|
+
|
|
60
|
+
begin
|
|
61
|
+
yield
|
|
62
|
+
rescue StandardError => e
|
|
63
|
+
log_failure(e)
|
|
64
|
+
nil
|
|
65
|
+
end
|
|
66
|
+
end
|
|
67
|
+
end
|
|
68
|
+
end
|
|
@@ -58,6 +58,7 @@ module ForestAdminAgent
|
|
|
58
58
|
end
|
|
59
59
|
|
|
60
60
|
def build
|
|
61
|
+
install_audit_trail
|
|
61
62
|
@container.register(:datasource, @customizer.datasource(@logger))
|
|
62
63
|
|
|
63
64
|
# Reset route cache to ensure routes are computed with all customizations
|
|
@@ -289,12 +290,31 @@ module ForestAdminAgent
|
|
|
289
290
|
@options[:customize_error_message] =
|
|
290
291
|
clean_option_value(@options[:customize_error_message], 'config.customize_error_message =')
|
|
291
292
|
@options[:logger] = clean_option_value(@options[:logger], 'config.logger =')
|
|
293
|
+
build_audit_trail_store
|
|
292
294
|
|
|
293
295
|
@container.register(:config, @options.to_h)
|
|
294
296
|
|
|
295
297
|
configure_rpc_polling_pool if @options[:rpc_max_polling_threads]
|
|
296
298
|
end
|
|
297
299
|
|
|
300
|
+
# The audit trail switches on as soon as a database is configured. The store connects and migrates here,
|
|
301
|
+
# at boot, rather than on the first write: an audit database the agent cannot reach should stop it
|
|
302
|
+
# starting, not leave it looking healthy while recording nothing — and under `critical: true` it would
|
|
303
|
+
# otherwise refuse every write from the moment somebody first tried to save something.
|
|
304
|
+
def build_audit_trail_store
|
|
305
|
+
options = @options[:audit_trail]
|
|
306
|
+
return unless options && options[:database]
|
|
307
|
+
|
|
308
|
+
options[:store] = AuditTrail::Store.new(**options.slice(:database, :schema, :table_name).compact).connect!
|
|
309
|
+
end
|
|
310
|
+
|
|
311
|
+
def install_audit_trail
|
|
312
|
+
options = @options[:audit_trail]
|
|
313
|
+
return if options.nil? || options[:store].nil?
|
|
314
|
+
|
|
315
|
+
@customizer.use(AuditTrail::Capture, { store: options[:store], redact: options[:redact] })
|
|
316
|
+
end
|
|
317
|
+
|
|
298
318
|
def configure_rpc_polling_pool
|
|
299
319
|
max_threads = @options[:rpc_max_polling_threads].to_i
|
|
300
320
|
return unless max_threads.positive?
|