forest_admin_agent 1.38.2 → 1.39.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. checksums.yaml +4 -4
  2. data/AUDIT_TRAIL.md +347 -0
  3. data/lib/forest_admin_agent/audit_trail/action_capture.rb +99 -0
  4. data/lib/forest_admin_agent/audit_trail/audit_record.rb +16 -0
  5. data/lib/forest_admin_agent/audit_trail/capture.rb +229 -0
  6. data/lib/forest_admin_agent/audit_trail/diff.rb +156 -0
  7. data/lib/forest_admin_agent/audit_trail/record_state.rb +43 -0
  8. data/lib/forest_admin_agent/audit_trail/recording.rb +57 -0
  9. data/lib/forest_admin_agent/audit_trail/snapshots.rb +85 -0
  10. data/lib/forest_admin_agent/audit_trail/sql/audit_connection_base.rb +53 -0
  11. data/lib/forest_admin_agent/audit_trail/sql/audit_log.rb +16 -0
  12. data/lib/forest_admin_agent/audit_trail/sql/field_filter.rb +58 -0
  13. data/lib/forest_admin_agent/audit_trail/sql/migrations.rb +53 -0
  14. data/lib/forest_admin_agent/audit_trail/sql/migrator.rb +117 -0
  15. data/lib/forest_admin_agent/audit_trail/sql/text_search.rb +78 -0
  16. data/lib/forest_admin_agent/audit_trail/store.rb +253 -0
  17. data/lib/forest_admin_agent/audit_trail.rb +68 -0
  18. data/lib/forest_admin_agent/builder/agent_factory.rb +20 -0
  19. data/lib/forest_admin_agent/http/correlation_id.rb +37 -0
  20. data/lib/forest_admin_agent/http/correlation_id_middleware.rb +29 -0
  21. data/lib/forest_admin_agent/http/router.rb +6 -0
  22. data/lib/forest_admin_agent/routes/action/actions.rb +76 -1
  23. data/lib/forest_admin_agent/routes/capabilities/collections.rb +11 -1
  24. data/lib/forest_admin_agent/routes/resources/audit_trail.rb +237 -0
  25. data/lib/forest_admin_agent/routes/resources/audit_trail_correlation.rb +101 -0
  26. data/lib/forest_admin_agent/routes/resources/audit_trail_route.rb +115 -0
  27. data/lib/forest_admin_agent/routes/resources/list.rb +1 -1
  28. data/lib/forest_admin_agent/routes/resources/related/list_related.rb +1 -1
  29. data/lib/forest_admin_agent/routes/resources/show.rb +1 -1
  30. data/lib/forest_admin_agent/serializer/forest_serializer_override.rb +14 -13
  31. data/lib/forest_admin_agent/utils/caller_parser.rb +4 -0
  32. data/lib/forest_admin_agent/utils/query_string_parser.rb +59 -10
  33. data/lib/forest_admin_agent/utils/schema/schema_emitter.rb +1 -1
  34. data/lib/forest_admin_agent/version.rb +1 -1
  35. data/lib/forest_admin_agent.rb +4 -0
  36. metadata +23 -2
@@ -0,0 +1,58 @@
1
+ module ForestAdminAgent
2
+ module AuditTrail
3
+ module Sql
4
+ # SQL keeping only the audit entries whose diff touched one of the given fields. Both JSON columns are
5
+ # searched: a field the change added exists in `new_values` only, one it removed in `previous_values`
6
+ # only.
7
+ #
8
+ # The test is per adapter, and a field name is always a whole key — never a path — so a name holding a
9
+ # dot (`address.city`) has to be quoted or the database reads it as a traversal.
10
+ class FieldFilter
11
+ COLUMNS = %w[previous_values new_values].freeze
12
+
13
+ def initialize(connection)
14
+ @connection = connection
15
+ end
16
+
17
+ def condition(fields)
18
+ adapter = @connection.adapter_name.downcase
19
+
20
+ case adapter
21
+ when /postgres/ then COLUMNS.map { |column| postgres_has_key(column, fields) }.join(' OR ')
22
+ when /sqlite/ then json_paths(fields) { |column, path| "json_type(#{column}, #{path}) IS NOT NULL" }
23
+ when /mysql|maria/ then mysql_has_keys(fields)
24
+ else
25
+ raise ForestAdminDatasourceToolkit::Exceptions::ForestException,
26
+ "Filtering the audit trail by field is not supported on #{adapter}"
27
+ end
28
+ end
29
+
30
+ private
31
+
32
+ # `jsonb_object_keys` rather than the `?|` operator: `?` is a bind placeholder for ActiveRecord and
33
+ # the function form needs no escaping. The column is `json`, hence the cast.
34
+ def postgres_has_key(column, fields)
35
+ list = fields.map { |field| @connection.quote(field) }.join(', ')
36
+
37
+ "EXISTS (SELECT 1 FROM jsonb_object_keys(#{column}::jsonb) AS key WHERE key IN (#{list}))"
38
+ end
39
+
40
+ # `json_type` and not `json_extract`: a key holding a JSON null extracts as SQL NULL, which would
41
+ # read as "no such key".
42
+ def json_paths(fields)
43
+ COLUMNS.flat_map { |column| fields.map { |field| yield(column, json_path(field)) } }.join(' OR ')
44
+ end
45
+
46
+ def mysql_has_keys(fields)
47
+ paths = fields.map { |field| json_path(field) }.join(', ')
48
+
49
+ COLUMNS.map { |column| "JSON_CONTAINS_PATH(#{column}, 'one', #{paths})" }.join(' OR ')
50
+ end
51
+
52
+ def json_path(field)
53
+ @connection.quote(%($."#{field.to_s.gsub(/["\\]/) { |char| "\\#{char}" }}"))
54
+ end
55
+ end
56
+ end
57
+ end
58
+ end
@@ -0,0 +1,53 @@
1
+ module ForestAdminAgent
2
+ module AuditTrail
3
+ module Sql
4
+ # The audit table's schema, as an ordered, append-only list. Nothing has shipped yet, so this is still a
5
+ # single entry; once a release is out, never edit one — add another, since a database out there has
6
+ # already recorded the earlier ones as applied. Every statement tolerates being replayed
7
+ # (`if_not_exists`), which is what makes a lost race harmless.
8
+ module Migrations
9
+ ALL = [
10
+ {
11
+ name: '001-create-audit-logs',
12
+ up: lambda do |connection, table|
13
+ # if_not_exists: a non-PG race (no advisory lock) can let two instances both reach here.
14
+ connection.create_table(table, if_not_exists: true) do |t|
15
+ t.datetime :timestamp, null: false
16
+ t.string :operation, null: false
17
+ t.string :collection, null: false
18
+ # Nullable, because a create's pending row has no id yet; text, because a packed composite id
19
+ # outgrows a varchar.
20
+ t.text :record_id
21
+ # No default on purpose: every write sets it, so a row arriving without one is a bug worth
22
+ # hearing about rather than a row that quietly claims to be done.
23
+ t.string :status, null: false
24
+ t.integer :user_id
25
+ # Denormalised from the caller at write time: who acted then, not whoever holds that id today.
26
+ t.text :user_first_name
27
+ t.text :user_last_name
28
+ t.text :user_email
29
+ # Set only on an update that moved a writable primary key: the id the row was filed under
30
+ # before. What lets a history query follow a record across a rename.
31
+ t.text :previous_record_id
32
+ # Smart-action rows only.
33
+ t.text :action_name
34
+ t.string :correlation_key
35
+ t.json :previous_values
36
+ t.json :new_values
37
+ end
38
+
39
+ base = table.split('.').last
40
+ # MySQL cannot index unbounded TEXT, so that one index needs a length prefix.
41
+ record_id_index = { name: "#{base}_record_id", if_not_exists: true }
42
+ record_id_index[:length] = 255 if connection.adapter_name.downcase.match?(/mysql|maria/)
43
+
44
+ connection.add_index(table, :record_id, **record_id_index)
45
+ connection.add_index(table, :correlation_key, name: "#{base}_correlation_key", if_not_exists: true)
46
+ connection.add_index(table, :user_id, name: "#{base}_user_id", if_not_exists: true)
47
+ end
48
+ }
49
+ ].freeze
50
+ end
51
+ end
52
+ end
53
+ end
@@ -0,0 +1,117 @@
1
+ module ForestAdminAgent
2
+ module AuditTrail
3
+ module Sql
4
+ # Applies {Migrations::ALL} to the audit table, tracking what has run in a companion table named after
5
+ # it — `audit_logs_migration` beside `audit_logs` (both namespaced in the `forest` schema on Postgres).
6
+ # One tracker per audited table, so two stores configured with different `table_name`s each get their
7
+ # own schema history instead of reading each other's as done.
8
+ #
9
+ # On Postgres the migrations run inside a transaction-scoped advisory lock, so several agent
10
+ # instances booting at once apply them one after another instead of racing on the same DDL. The
11
+ # schema is created (and committed) first, made idempotent (CREATE SCHEMA IF NOT EXISTS +
12
+ # tolerating a concurrent create), because the lock cannot cover a not-yet-existing schema.
13
+ class Migrator
14
+ # Arbitrary but stable key pair identifying the audit-trail migration critical section.
15
+ ADVISORY_LOCK = [0x464f, 0x5254].freeze # "FO", "RT"
16
+ # duplicate_schema, and the unique violation on pg_namespace the same race can raise instead.
17
+ DUPLICATE_SCHEMA_STATES = %w[42P06 23505].freeze
18
+
19
+ def initialize(connection, schema:, table_name:)
20
+ @connection = connection
21
+ @schema = schema # nil on adapters without schema support
22
+ @table_name = table_name
23
+ end
24
+
25
+ def run
26
+ ensure_schema
27
+
28
+ if postgres?
29
+ @connection.transaction do
30
+ @connection.execute("SELECT pg_advisory_xact_lock(#{ADVISORY_LOCK[0]}, #{ADVISORY_LOCK[1]})")
31
+ apply_pending
32
+ end
33
+ else
34
+ apply_pending
35
+ end
36
+ end
37
+
38
+ private
39
+
40
+ def postgres?
41
+ @connection.adapter_name.downcase.include?('postgres')
42
+ end
43
+
44
+ def schema?
45
+ postgres? && @schema.present?
46
+ end
47
+
48
+ def qualified(name)
49
+ schema? ? "#{@schema}.#{name}" : name
50
+ end
51
+
52
+ # Create the schema first and commit it: the migrations open DDL on the same connection, and a
53
+ # CREATE SCHEMA still pending in the lock transaction would not be visible to them.
54
+ def ensure_schema
55
+ return unless schema?
56
+
57
+ @connection.execute("CREATE SCHEMA IF NOT EXISTS #{@connection.quote_schema_name(@schema)}")
58
+ rescue ActiveRecord::RecordNotUnique
59
+ # 23505 on pg_namespace, already mapped to its own class by ActiveRecord: another instance
60
+ # created the schema between our IF NOT EXISTS check and the create itself.
61
+ nil
62
+ rescue ActiveRecord::StatementInvalid => e
63
+ raise unless duplicate_schema?(e)
64
+ end
65
+
66
+ # By SQLSTATE where the adapter exposes one, so an unrelated failure (no permission to create a
67
+ # schema, say) is not read as a lost race just because its message says "exists".
68
+ def duplicate_schema?(error)
69
+ state = sql_state(error)
70
+ return DUPLICATE_SCHEMA_STATES.include?(state) if state
71
+
72
+ /already exists|duplicate/i.match?(error.message)
73
+ end
74
+
75
+ def sql_state(error)
76
+ cause = error.cause
77
+ return nil unless cause.respond_to?(:result) && defined?(PG::Result)
78
+
79
+ cause.result.error_field(PG::Result::PG_DIAG_SQLSTATE)
80
+ rescue StandardError
81
+ nil
82
+ end
83
+
84
+ def apply_pending
85
+ done = applied_migrations
86
+ table = qualified(@table_name)
87
+
88
+ Migrations::ALL.each do |migration|
89
+ next if done.include?(migration[:name])
90
+
91
+ migration[:up].call(@connection, table)
92
+ @connection.execute(
93
+ "INSERT INTO #{@connection.quote_table_name(migrations_table)} (name) " \
94
+ "VALUES (#{@connection.quote(migration[:name])})"
95
+ )
96
+ end
97
+ end
98
+
99
+ def applied_migrations
100
+ ensure_migrations_table
101
+
102
+ @connection.select_values("SELECT name FROM #{@connection.quote_table_name(migrations_table)}")
103
+ end
104
+
105
+ def ensure_migrations_table
106
+ @connection.create_table(migrations_table, id: false, if_not_exists: true) do |t|
107
+ t.string :name, null: false
108
+ end
109
+ end
110
+
111
+ def migrations_table
112
+ qualified("#{@table_name}_migration")
113
+ end
114
+ end
115
+ end
116
+ end
117
+ end
@@ -0,0 +1,78 @@
1
+ require 'json'
2
+
3
+ module ForestAdminAgent
4
+ module AuditTrail
5
+ module Sql
6
+ # SQL keeping only the audit entries a free-text term matches, case-insensitively and as a substring:
7
+ # the action's name, who acted, and the keys and values recorded on both sides of the change — at any
8
+ # depth, since only the changed leaves of a JSON column are stored and the term has to reach them.
9
+ #
10
+ # Deliberately not searched: operation, correlation_key, record_id, collection, status and timestamp.
11
+ # Machine identifiers nobody searches for, and matching them turns one term into a pile of confusing
12
+ # hits.
13
+ #
14
+ # The values are matched against the JSON document as text, which is what lets one condition reach any
15
+ # depth and compose with pagination and the count. It cannot use an index, which is affordable here
16
+ # because a history query is already narrowed to one record.
17
+ class TextSearch
18
+ TEXT_COLUMNS = %w[action_name user_first_name user_last_name user_email].freeze
19
+ JSON_COLUMNS = %w[previous_values new_values].freeze
20
+ # `!` rather than a backslash: MySQL treats a backslash as an escape inside string literals too, so
21
+ # `ESCAPE '\'` needs doubling there and nowhere else.
22
+ ESCAPE = '!'.freeze
23
+ # A masked value is stored as this. It is removed before matching, so a search for "redacted" cannot
24
+ # hit it — and since the real value was never recorded, searching that finds nothing either. A search
25
+ # must never confirm a value the trail refused to keep.
26
+ REDACTED = Recording::REDACTED
27
+
28
+ def initialize(connection)
29
+ @connection = connection
30
+ end
31
+
32
+ def condition(term)
33
+ text = term.to_s.downcase
34
+ # The value objects are matched as serialized JSON, where a quote, a backslash or a newline is
35
+ # escaped — so `15" monitor` sits in the document as `15\" monitor` and the raw term would never
36
+ # find it. Escaping the term the same way makes it match, and stops a bare quote from matching the
37
+ # document's own structure.
38
+ clauses = TEXT_COLUMNS.map { |column| like(column, text) }
39
+ clauses += JSON_COLUMNS.map { |column| like(searchable_json(column), json_escaped(text)) }
40
+
41
+ clauses.join(' OR ')
42
+ end
43
+
44
+ private
45
+
46
+ def like(expression, text)
47
+ "LOWER(#{expression}) LIKE #{@connection.quote("%#{escape(text)}%")} ESCAPE '#{ESCAPE}'"
48
+ end
49
+
50
+ # What JSON generation would have done to the term: `to_json` on the string, minus its own quotes.
51
+ def json_escaped(text)
52
+ text.to_json[1..-2]
53
+ end
54
+
55
+ def searchable_json(column)
56
+ "REPLACE(#{as_text(column)}, #{@connection.quote(REDACTED)}, '')"
57
+ end
58
+
59
+ def as_text(column)
60
+ adapter = @connection.adapter_name.downcase
61
+
62
+ case adapter
63
+ when /postgres/ then "#{column}::text"
64
+ when /sqlite/ then column
65
+ when /mysql|maria/ then "CAST(#{column} AS CHAR)"
66
+ else
67
+ raise ForestAdminDatasourceToolkit::Exceptions::ForestException,
68
+ "Searching the audit trail is not supported on #{adapter}"
69
+ end
70
+ end
71
+
72
+ def escape(term)
73
+ term.gsub(/[!%_]/) { |char| "#{ESCAPE}#{char}" }
74
+ end
75
+ end
76
+ end
77
+ end
78
+ end
@@ -0,0 +1,253 @@
1
+ require 'time'
2
+
3
+ module ForestAdminAgent
4
+ module AuditTrail
5
+ # SQL-backed storage that both writes every audited change and reads the per-record history back.
6
+ #
7
+ # {#connect!} opens the connection and migrates; the agent factory calls it at boot rather than leaving it
8
+ # to the first write, so a database the agent cannot reach is a startup failure instead of an agent that
9
+ # looks healthy while recording nothing — and, under `critical: true`, instead of one that refuses every
10
+ # write the moment somebody first tries to save something.
11
+ class Store
12
+ DEFAULT_SCHEMA = 'forest'.freeze
13
+ DEFAULT_TABLE = 'audit_logs'.freeze
14
+ COLUMNS = %i[timestamp operation collection record_id previous_record_id status user_id user_first_name
15
+ user_last_name user_email action_name correlation_key previous_values new_values].freeze
16
+ AUTHOR_COLUMNS = %i[user_id user_first_name user_last_name user_email].freeze
17
+
18
+ def initialize(database:, schema: DEFAULT_SCHEMA, table_name: DEFAULT_TABLE)
19
+ @database = database
20
+ @schema = schema
21
+ @table_name = table_name
22
+ @mutex = Mutex.new
23
+ @ready = false
24
+ end
25
+
26
+ def connect!
27
+ ensure_ready
28
+
29
+ self
30
+ end
31
+
32
+ def append(record)
33
+ append_all([record]).first
34
+ end
35
+
36
+ # Inserts rows and returns their ids, in the order given. Batched, because a "delete all" snapshot can
37
+ # be thousands of records and the pending/confirm protocol writes each of them twice.
38
+ #
39
+ # The ids are matched to their rows by `record_id` rather than by the order RETURNING happens to come
40
+ # back in, which Postgres does not promise: pairing them positionally would confirm each pending row with
41
+ # another record's diff. One row per record per operation, so that key is unique within a batch — bar a
42
+ # pending create, which has no id yet and is always a batch of one.
43
+ def append_all(records)
44
+ return [] if records.empty?
45
+
46
+ rows = records.map { |record| to_row(record) }
47
+ return rows.map { |row| model.create!(row).id } unless batch_returning?(rows)
48
+
49
+ returned = model.insert_all(rows, returning: %i[id record_id]).rows.to_h { |id, key| [key, id] }
50
+
51
+ rows.map { |row| returned[row[:record_id]] }
52
+ end
53
+
54
+ # One insert per row when the ids cannot be matched back: no RETURNING on this adapter (MySQL), or a
55
+ # batch whose record ids are not distinct enough to pair on.
56
+ def batch_returning?(rows)
57
+ return false unless model.connection.supports_insert_returning?
58
+
59
+ keys = rows.map { |row| row[:record_id] }
60
+
61
+ keys.none?(&:nil?) && keys.uniq.size == keys.size
62
+ end
63
+
64
+ def confirm(id, attributes)
65
+ row = model.find_by(id: id)
66
+
67
+ row&.update!(**attributes, status: Recording::DONE)
68
+ end
69
+
70
+ # A write that turned out to change nothing leaves no trace: the pending row goes rather than sitting
71
+ # there implying the write is unaccounted for.
72
+ def discard(ids)
73
+ model.where(id: ids).delete_all unless ids.empty?
74
+ end
75
+
76
+ def list_by_record(collection:, record_id:, skip: 0, limit: nil, user_ids: nil, start_timestamp: nil,
77
+ end_timestamp: nil, fields: nil, search: nil, order: 'asc')
78
+ # `id` (insertion order) breaks ties on equal timestamps in both directions, keeping pages
79
+ # deterministic and stable.
80
+ relation = scope(collection, record_id, user_ids: user_ids, start_timestamp: start_timestamp,
81
+ end_timestamp: end_timestamp, fields: fields, search: search)
82
+ .order(timestamp: order.to_s == 'desc' ? :desc : :asc, id: :asc)
83
+ .offset(skip || 0)
84
+ relation = relation.limit(limit) unless limit.nil?
85
+
86
+ relation.map { |row| from_row(row) }
87
+ end
88
+
89
+ def count_by_record(collection:, record_id:, user_ids: nil, start_timestamp: nil,
90
+ end_timestamp: nil, fields: nil, search: nil)
91
+ scope(collection, record_id, user_ids: user_ids, start_timestamp: start_timestamp,
92
+ end_timestamp: end_timestamp, fields: fields, search: search).count
93
+ end
94
+
95
+ # The distinct authors of the entries the current filters match, whatever page is being asked for. The
96
+ # identity comes from the rows themselves, so a user who has since been renamed or removed still reads
97
+ # as they were when they acted.
98
+ def authors_by_record(collection:, record_id:, user_ids: nil, start_timestamp: nil,
99
+ end_timestamp: nil, fields: nil, search: nil)
100
+ scope(collection, record_id, user_ids: user_ids, start_timestamp: start_timestamp,
101
+ end_timestamp: end_timestamp, fields: fields, search: search)
102
+ .where.not(user_id: nil)
103
+ .distinct
104
+ .pluck(*AUTHOR_COLUMNS)
105
+ .map { |values| AUTHOR_COLUMNS.zip(values).to_h }
106
+ .uniq { |author| author[:user_id] }
107
+ end
108
+
109
+ # The ids this record was renamed from, each with the moment it stopped being that id. Walking those back
110
+ # is what lets a history query reach rows written before a rename — they stay under the id they were
111
+ # written with, since that is the id they were true of — and the moment bounds how far: the id it left may
112
+ # have been taken by another record afterwards, whose rows are none of this record's business.
113
+ def renamed_from(collection:, record_id:)
114
+ model.where(collection: collection, record_id: record_id)
115
+ .where.not(previous_record_id: nil)
116
+ .pluck(:previous_record_id, :timestamp, :id)
117
+ .group_by(&:first)
118
+ .map do |id, rows|
119
+ # The row id comes along as the tie-breaker: the trail orders itself by (timestamp, id), so a
120
+ # bound that knew only the timestamp would mean something slightly different from "before".
121
+ _, at, row = rows.max_by { |(_, timestamp, row_id)| [timestamp, row_id] }
122
+
123
+ { id: id, until: as_iso(at), until_row: row }
124
+ end
125
+ end
126
+
127
+ # Entries recorded strictly after `timestamp`, newest first: what a state reconstruction has to undo.
128
+ # Strictly after, so an entry stamped exactly at the requested instant counts as part of that state
129
+ # instead of being reverted out of it.
130
+ # Confirmed rows only: a pending one records an attempt whose outcome is unknown, and undoing a change
131
+ # that may never have happened would invent a state the record was never in. The history reads keep
132
+ # pending rows — they are evidence, and `status` tells the reader what they are — but a reconstruction
133
+ # cannot act on them.
134
+ def list_since(collection:, record_id:, timestamp:)
135
+ model.where(collection: collection, status: Recording::DONE)
136
+ .where(*segments_condition(record_id))
137
+ .where('timestamp > ?', as_time(timestamp))
138
+ .order(timestamp: :desc, id: :desc)
139
+ .map { |row| from_row(row) }
140
+ end
141
+
142
+ def list_by_correlation(collection:, record_id:, correlation_key:)
143
+ list_by_correlations(collection: collection, record_id: record_id, correlation_keys: [correlation_key])
144
+ end
145
+
146
+ def list_by_correlations(collection:, record_id:, correlation_keys:)
147
+ return [] if correlation_keys.empty?
148
+
149
+ model.where(collection: collection, record_id: record_id, correlation_key: correlation_keys)
150
+ .order(:timestamp, :id)
151
+ .map { |row| from_row(row) }
152
+ end
153
+
154
+ private
155
+
156
+ # Every filter is an AND, so the count matches exactly what a page of this history holds.
157
+ def scope(collection, record_id, user_ids: nil, start_timestamp: nil, end_timestamp: nil,
158
+ fields: nil, search: nil)
159
+ relation = model.where(collection: collection).where(*segments_condition(record_id))
160
+ relation = relation.where(user_id: user_ids) if user_ids
161
+ relation = relation.where(Sql::FieldFilter.new(model.connection).condition(fields)) if fields&.any?
162
+ relation = relation.where(Sql::TextSearch.new(model.connection).condition(search)) if search
163
+ # Compare as Time so ActiveRecord casts the bound to the datetime column's storage format
164
+ # (raw ISO strings with a `Z` would compare lexically against the cast rows and never match).
165
+ relation = relation.where('timestamp >= ?', as_time(start_timestamp)) if start_timestamp
166
+ relation = relation.where('timestamp <= ?', as_time(end_timestamp)) if end_timestamp
167
+ relation
168
+ end
169
+
170
+ def as_time(value)
171
+ value.is_a?(::Time) ? value : ::Time.iso8601(value.to_s)
172
+ end
173
+
174
+ def as_iso(value)
175
+ value.respond_to?(:iso8601) ? value.iso8601(3) : value.to_s
176
+ end
177
+
178
+ # One record's history is its current id plus every id it was renamed from, each earlier one only up to
179
+ # the rename: `record_id = '7' OR (record_id = '1' AND timestamp <= …)`. A plain `IN` would hand over the
180
+ # rows of whichever record holds that id now.
181
+ #
182
+ # Takes an id, several, or segments — `{ id:, until: }` — so a caller that has no rename to care about
183
+ # simply passes the id.
184
+ def segments_condition(record_id)
185
+ binds = []
186
+ sql = Array(record_id).map { |value| value.is_a?(Hash) ? value : { id: value, until: nil } }.map do |segment|
187
+ binds << segment[:id]
188
+ next 'record_id = ?' unless segment[:until]
189
+
190
+ binds << as_time(segment[:until])
191
+ # Same millisecond as the rename, and the id says which side of it a row falls on: another record
192
+ # taking the abandoned key that fast would otherwise land in this record's history.
193
+ next '(record_id = ? AND timestamp <= ?)' unless segment[:until_row]
194
+
195
+ binds << as_time(segment[:until]) << segment[:until_row]
196
+ '(record_id = ? AND (timestamp < ? OR (timestamp = ? AND id <= ?)))'
197
+ end
198
+
199
+ [sql.join(' OR '), *binds]
200
+ end
201
+
202
+ def model
203
+ ensure_ready
204
+ @model
205
+ end
206
+
207
+ def ensure_ready
208
+ return if @ready
209
+
210
+ @mutex.synchronize do
211
+ return if @ready
212
+
213
+ Sql::AuditConnectionBase.connect_to(@database)
214
+ connection = Sql::AuditConnectionBase.connection
215
+ Sql::Migrator.new(connection, schema: schema_for(connection), table_name: @table_name).run
216
+ @model = build_model(qualified(connection))
217
+ @ready = true
218
+ end
219
+ end
220
+
221
+ # A per-instance concrete subclass bound to this store's own table, so distinct stores can't
222
+ # clobber each other's table name. reset_column_information drops stale metadata for the table
223
+ # the migration just created/evolved.
224
+ def build_model(table)
225
+ Class.new(Sql::AuditLog) { self.table_name = table }.tap(&:reset_column_information)
226
+ end
227
+
228
+ def schema_for(connection)
229
+ connection.adapter_name.downcase.include?('postgres') ? @schema : nil
230
+ end
231
+
232
+ def qualified(connection)
233
+ schema = schema_for(connection)
234
+ schema ? "#{schema}.#{@table_name}" : @table_name
235
+ end
236
+
237
+ # An AuditRecord is a Struct and a row answers to `[]` too, so the mapping is the column list itself.
238
+ def to_row(record)
239
+ COLUMNS.to_h { |column| [column, record[column]] }
240
+ end
241
+
242
+ def from_row(row)
243
+ values = COLUMNS.to_h { |column| [column, row[column]] }
244
+ values[:id] = row.id
245
+ values[:timestamp] = row.timestamp.respond_to?(:iso8601) ? row.timestamp.iso8601(3) : row.timestamp.to_s
246
+ values[:previous_values] ||= {}
247
+ values[:new_values] ||= {}
248
+
249
+ AuditRecord.new(**values)
250
+ end
251
+ end
252
+ end
253
+ end
@@ -0,0 +1,68 @@
1
+ module ForestAdminAgent
2
+ # The audit trail is inert unless `config.audit_trail[:database]` was set: the agent factory builds the
3
+ # store during setup — connecting and migrating there rather than on first write — and everything (capture
4
+ # layers and routes) resolves it from here.
5
+ module AuditTrail
6
+ # One operation must not materialise an unbounded number of records: a "delete all" would otherwise read
7
+ # every matched row and, with the pending/confirm protocol, write each of them twice. Truncation is logged,
8
+ # never silent.
9
+ #
10
+ # Matches the Node agent's `MAX_SNAPSHOT_RECORDS`: the same feature behind the same config key, so a bulk
11
+ # operation must not be audited on one agent and truncated on the other. Change it in both or neither.
12
+ MAX_RECORDS_PER_OPERATION = 1000
13
+
14
+ # Auditing a subset while the write touches every match is the one thing `critical` exists to prevent, so
15
+ # over the cap the operation is refused instead — before anything is written, in both the write and the
16
+ # action path, which is why the message lives here rather than in either of them.
17
+ def self.refuse_over_cap!
18
+ raise ForestAdminDatasourceToolkit::Exceptions::ForestException,
19
+ 'The audit trail is configured as critical and cannot record an operation touching more than ' \
20
+ "#{MAX_RECORDS_PER_OPERATION} records at once. Narrow the selection."
21
+ end
22
+
23
+ def self.log_truncation(kept, total)
24
+ skipped = total ? total - kept : 'further'
25
+
26
+ Facades::Container.logger.log(
27
+ 'Warn',
28
+ "[ForestAdmin] Audit trail: #{kept} records audited, #{skipped} skipped " \
29
+ "(cap #{MAX_RECORDS_PER_OPERATION} per operation)"
30
+ )
31
+ end
32
+
33
+ def self.options
34
+ config = Facades::Container.config_from_cache
35
+
36
+ (config && config[:audit_trail]) || {}
37
+ end
38
+
39
+ def self.store
40
+ options[:store]
41
+ end
42
+
43
+ # `critical: true` makes the pending insert a precondition of the write: if the audit trail cannot record
44
+ # that an operation is about to happen, the operation is refused. Nothing was written, so there is nothing
45
+ # to repair and no compensating write ever happens. Default false keeps today's behaviour, where a broken
46
+ # audit database costs rows rather than writes.
47
+ def self.critical?
48
+ options[:critical] == true
49
+ end
50
+
51
+ def self.log_failure(error)
52
+ Facades::Container.logger.log('Error', "[ForestAdmin] Audit trail unavailable, skipping: #{error.message}")
53
+ end
54
+
55
+ # Runs the pending insert under the configured policy: refusing the operation when critical, logging and
56
+ # carrying on otherwise.
57
+ def self.gate
58
+ return yield if critical?
59
+
60
+ begin
61
+ yield
62
+ rescue StandardError => e
63
+ log_failure(e)
64
+ nil
65
+ end
66
+ end
67
+ end
68
+ end
@@ -58,6 +58,7 @@ module ForestAdminAgent
58
58
  end
59
59
 
60
60
  def build
61
+ install_audit_trail
61
62
  @container.register(:datasource, @customizer.datasource(@logger))
62
63
 
63
64
  # Reset route cache to ensure routes are computed with all customizations
@@ -289,12 +290,31 @@ module ForestAdminAgent
289
290
  @options[:customize_error_message] =
290
291
  clean_option_value(@options[:customize_error_message], 'config.customize_error_message =')
291
292
  @options[:logger] = clean_option_value(@options[:logger], 'config.logger =')
293
+ build_audit_trail_store
292
294
 
293
295
  @container.register(:config, @options.to_h)
294
296
 
295
297
  configure_rpc_polling_pool if @options[:rpc_max_polling_threads]
296
298
  end
297
299
 
300
+ # The audit trail switches on as soon as a database is configured. The store connects and migrates here,
301
+ # at boot, rather than on the first write: an audit database the agent cannot reach should stop it
302
+ # starting, not leave it looking healthy while recording nothing — and under `critical: true` it would
303
+ # otherwise refuse every write from the moment somebody first tried to save something.
304
+ def build_audit_trail_store
305
+ options = @options[:audit_trail]
306
+ return unless options && options[:database]
307
+
308
+ options[:store] = AuditTrail::Store.new(**options.slice(:database, :schema, :table_name).compact).connect!
309
+ end
310
+
311
+ def install_audit_trail
312
+ options = @options[:audit_trail]
313
+ return if options.nil? || options[:store].nil?
314
+
315
+ @customizer.use(AuditTrail::Capture, { store: options[:store], redact: options[:redact] })
316
+ end
317
+
298
318
  def configure_rpc_polling_pool
299
319
  max_threads = @options[:rpc_max_polling_threads].to_i
300
320
  return unless max_threads.positive?