provenance 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +19 -0
- data/LICENSE +21 -0
- data/README.md +358 -0
- data/lib/generators/provenance/install/install_generator.rb +34 -0
- data/lib/generators/provenance/install/templates/create_provenance_outbox.rb.tt +21 -0
- data/lib/generators/provenance/install/templates/initializer.rb.tt +12 -0
- data/lib/provenance/action.rb +160 -0
- data/lib/provenance/actor.rb +86 -0
- data/lib/provenance/configuration.rb +152 -0
- data/lib/provenance/context.rb +30 -0
- data/lib/provenance/controller.rb +93 -0
- data/lib/provenance/emitter.rb +66 -0
- data/lib/provenance/entity_change.rb +73 -0
- data/lib/provenance/errors.rb +12 -0
- data/lib/provenance/event.rb +42 -0
- data/lib/provenance/integrity.rb +75 -0
- data/lib/provenance/job.rb +62 -0
- data/lib/provenance/json_safe.rb +41 -0
- data/lib/provenance/middleware.rb +44 -0
- data/lib/provenance/model.rb +123 -0
- data/lib/provenance/outbox/record.rb +23 -0
- data/lib/provenance/outbox/relay.rb +81 -0
- data/lib/provenance/outbox.rb +67 -0
- data/lib/provenance/railtie.rb +35 -0
- data/lib/provenance/rake.rb +27 -0
- data/lib/provenance/recorder.rb +206 -0
- data/lib/provenance/redactor.rb +63 -0
- data/lib/provenance/registry.rb +48 -0
- data/lib/provenance/relay_job.rb +15 -0
- data/lib/provenance/rspec.rb +139 -0
- data/lib/provenance/serializers/cloud_events.rb +26 -0
- data/lib/provenance/serializers/native.rb +16 -0
- data/lib/provenance/serializers/ocsf.rb +127 -0
- data/lib/provenance/serializers.rb +31 -0
- data/lib/provenance/sinks/http.rb +81 -0
- data/lib/provenance/sinks/io.rb +24 -0
- data/lib/provenance/sinks/kafka.rb +45 -0
- data/lib/provenance/sinks/logger.rb +22 -0
- data/lib/provenance/sinks/memory.rb +29 -0
- data/lib/provenance/sinks/proc.rb +20 -0
- data/lib/provenance/sinks.rb +35 -0
- data/lib/provenance/testing.rb +36 -0
- data/lib/provenance/uuid.rb +20 -0
- data/lib/provenance/version.rb +6 -0
- data/lib/provenance.rb +166 -0
- data/lib/tasks/provenance.rake +22 -0
- data/schema/event-2.json +126 -0
- metadata +150 -0
checksums.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
---
|
|
2
|
+
SHA256:
|
|
3
|
+
metadata.gz: f14ec0ba7dff7626fb82cb169c9f76423f72e0dc059f571f84bbadf07282455c
|
|
4
|
+
data.tar.gz: 8643b14cb28e42d5c2c62e0365ddfa39e3b0030237bc363b1bcbcc77be4ceb38
|
|
5
|
+
SHA512:
|
|
6
|
+
metadata.gz: c299ddc878e226b43c8c14ed12e4f258d4882ce4bc0b65ccac5e235263fa068d9480d7185f9a03c31433f0fd3115a7c4d0cdab61cd5ece58c43e0ca9c866e318
|
|
7
|
+
data.tar.gz: 9b7bb4fa07e2d5c07ebedbbcd608894270c332a6b7837bc4fd0a5c8371500d0ee902ca4a408159ee95cae7edea0ce4020fcfa122c1c584c90569fd2925d3d564
|
data/CHANGELOG.md
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
## 2.0.0
|
|
4
|
+
|
|
5
|
+
- One action-level event per HTTP request, ActiveJob job, rake task or explicit
|
|
6
|
+
`Provenance.action` block, with actor, request context, outcome and entity changes.
|
|
7
|
+
- Opt-in model tracking with `has_provenance` (`only`, `except`, `redact`,
|
|
8
|
+
`associations`, `ignore_if`), link/unlink changes and bulk `update_all`,
|
|
9
|
+
`delete_all`, `insert_all`/`upsert_all` changes.
|
|
10
|
+
- Emission after all touched transactions commit; rolled-back changes are dropped.
|
|
11
|
+
- Native event format with a shipped JSON Schema (`schema/event-2.json`), plus OCSF
|
|
12
|
+
1.x and CloudEvents 1.0 serializers.
|
|
13
|
+
- Sinks: logger, IO, HTTP (retries with backoff and jitter), proc, memory and
|
|
14
|
+
optional Kafka.
|
|
15
|
+
- Outbox table with `Provenance::RelayJob`, `FOR UPDATE SKIP LOCKED` on PostgreSQL,
|
|
16
|
+
retries and retention pruning; `rails g provenance:install`.
|
|
17
|
+
- HMAC-SHA256 integrity chain with `Provenance::Integrity.verify` and
|
|
18
|
+
`rake provenance:verify`.
|
|
19
|
+
- RSpec support: `Provenance::Testing.capture` and the `emit_provenance_event` matcher.
|
data/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Ivan Nikolaev
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
data/README.md
ADDED
|
@@ -0,0 +1,358 @@
|
|
|
1
|
+
# Provenance
|
|
2
|
+
|
|
3
|
+
Provenance answers **who did what, through which entry point, with what outcome, and
|
|
4
|
+
which records changed**. It emits one *action-level* event per HTTP request,
|
|
5
|
+
background job, rake task or explicit block, and delivers it reliably to security and
|
|
6
|
+
observability tooling: SIEM, log pipelines, webhooks, Kafka.
|
|
7
|
+
|
|
8
|
+
Provenance is **not** a versioning or undo library. [paper_trail], [audited] and
|
|
9
|
+
[logidze] keep per-record history inside your application database so you can show or
|
|
10
|
+
restore previous versions. Provenance emits tamper-evident, standards-shaped events
|
|
11
|
+
*out of* the application, one per user intent, with the record changes attached.
|
|
12
|
+
|
|
13
|
+
Requirements: Ruby >= 3.2, Rails >= 7.2. MIT license.
|
|
14
|
+
|
|
15
|
+
## Quick start
|
|
16
|
+
|
|
17
|
+
```ruby
|
|
18
|
+
# Gemfile
|
|
19
|
+
gem "provenance", "~> 2.0"
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
```sh
|
|
23
|
+
bin/rails g provenance:install # outbox migration + config/initializers/provenance.rb
|
|
24
|
+
bin/rails db:migrate
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
```ruby
|
|
28
|
+
# config/initializers/provenance.rb
|
|
29
|
+
Provenance.configure do |c|
|
|
30
|
+
c.enabled = !Rails.env.test?
|
|
31
|
+
c.actor { |ctx| ctx.controller.try(:current_user) }
|
|
32
|
+
c.sink :logger
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
# app/models/invoice.rb
|
|
36
|
+
class Invoice < ApplicationRecord
|
|
37
|
+
has_provenance only: %i[status amount], redact: %i[notes], associations: %i[tags]
|
|
38
|
+
end
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
A `PATCH /invoices/7` that changes the status now emits:
|
|
42
|
+
|
|
43
|
+
```json
|
|
44
|
+
{
|
|
45
|
+
"schema": "provenance/event@2",
|
|
46
|
+
"id": "0192f5c6-7b1e-7cc2-9a51-0d4f6a8e2b10",
|
|
47
|
+
"occurred_at": "2026-10-05T12:00:00.123Z",
|
|
48
|
+
"app": "billing",
|
|
49
|
+
"action": { "kind": "http", "name": "invoices#update", "caused_by": null },
|
|
50
|
+
"actor": { "id": "42", "type": "User", "display": "a@b.c", "roles": ["admin"], "impersonator": null },
|
|
51
|
+
"request": { "id": "3f1c…", "method": "PATCH", "path": "/invoices/7", "status": 200, "ip": "10.0.0.1", "user_agent": "…" },
|
|
52
|
+
"outcome": { "result": "success", "error": null },
|
|
53
|
+
"changes": [
|
|
54
|
+
{ "entity": "Invoice", "entity_id": "7", "operation": "update", "diff": { "status": ["draft", "sent"] } }
|
|
55
|
+
],
|
|
56
|
+
"metadata": {}
|
|
57
|
+
}
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
The JSON Schema ships in [`schema/event-2.json`](schema/event-2.json).
|
|
61
|
+
|
|
62
|
+
## Concepts
|
|
63
|
+
|
|
64
|
+
- **Action** — a unit of user intent with a kind (`http`, `job`, `task`, `custom`),
|
|
65
|
+
a name, an actor, request context, an outcome and the entity changes made while it
|
|
66
|
+
was open. At most one event is emitted per action.
|
|
67
|
+
- **Actor** — `id`, `type`, `display`, `roles` and an optional `impersonator` of the
|
|
68
|
+
same shape for "login as".
|
|
69
|
+
- **Entity change** — `entity`, `entity_id`, `operation` (`create`, `update`,
|
|
70
|
+
`destroy`, `link`, `unlink`, `bulk_update`, `bulk_delete`, `bulk_insert`) and `diff`
|
|
71
|
+
(`{attr => [before, after]}` for updates, `{attr => after}` for creates,
|
|
72
|
+
`{attr => before}` for destroys).
|
|
73
|
+
|
|
74
|
+
### What gets captured
|
|
75
|
+
|
|
76
|
+
| Entry point | How | Action name |
|
|
77
|
+
|---|---|---|
|
|
78
|
+
| HTTP | Rack middleware inserted after `ActionDispatch::RequestId` | `"#{controller_path}##{action_name}"` |
|
|
79
|
+
| ActiveJob | `around_perform` | job class name |
|
|
80
|
+
| Rake | `Provenance::Rake.install!` (opt-in, e.g. in `Rakefile`) | task name |
|
|
81
|
+
| Explicit | `Provenance.action("users.import") { ... }` | given name |
|
|
82
|
+
| Model change outside any action | implicit `custom` action | `"Invoice.update"` |
|
|
83
|
+
|
|
84
|
+
An action is emitted when it closes **and** every transaction it touched has
|
|
85
|
+
committed. It is emitted if it has changes, has a non-success outcome, is a non-read
|
|
86
|
+
HTTP request (anything but GET/HEAD), was opened explicitly, or `audit_reads` is on.
|
|
87
|
+
|
|
88
|
+
Outcomes: `success`; `failure` for an unhandled exception (`error.class` and
|
|
89
|
+
`error.message`, truncated to 500 characters) or a 5xx response; `denied` for an
|
|
90
|
+
exception listed in `denied_exceptions` or a 401/403 response. Exceptions are always
|
|
91
|
+
re-raised.
|
|
92
|
+
|
|
93
|
+
Jobs enqueued inside an action carry the action id and actor in the job payload under
|
|
94
|
+
the `provenance` key, so the job event has `action.caused_by` set to the parent id.
|
|
95
|
+
|
|
96
|
+
## API
|
|
97
|
+
|
|
98
|
+
```ruby
|
|
99
|
+
Provenance.action("users.import", actor: admin, metadata: { file: "x.csv" }) do
|
|
100
|
+
# nested Provenance.action blocks attach to the outermost action
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
Provenance.current&.annotate(reason: "GDPR request #42")
|
|
104
|
+
Provenance.current&.actor = some_user
|
|
105
|
+
|
|
106
|
+
Provenance.without { Invoice.update_all(viewed: true) } # suppress tracking
|
|
107
|
+
|
|
108
|
+
class Admin::ReportsController < ApplicationController
|
|
109
|
+
provenance_action_name ->(controller) { "admin.reports.#{controller.action_name}" }
|
|
110
|
+
skip_provenance only: :preview
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
class CleanupJob < ApplicationJob
|
|
114
|
+
skip_provenance
|
|
115
|
+
end
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
### Models
|
|
119
|
+
|
|
120
|
+
```ruby
|
|
121
|
+
class Invoice < ApplicationRecord
|
|
122
|
+
has_many :tags
|
|
123
|
+
has_provenance only: %i[status amount], # or except: [...]
|
|
124
|
+
redact: %i[notes], # recorded as "[REDACTED]"
|
|
125
|
+
associations: %i[tags], # link/unlink changes
|
|
126
|
+
ignore_if: ->(invoice) { invoice.draft? }
|
|
127
|
+
end
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
`has_provenance` must come after the association declarations it references. Changes
|
|
131
|
+
are held per transaction and dropped if it rolls back. Bulk operations on tracked
|
|
132
|
+
models — `update_all`, `delete_all`, `insert_all`, `upsert_all` (and the methods built
|
|
133
|
+
on them) — produce one `bulk_*` change with `count`, a `where` fingerprint (SHA-256 of
|
|
134
|
+
the WHERE clause with bind values replaced by placeholders) and up to `bulk_ids_limit`
|
|
135
|
+
affected `ids` (`truncated: true` when cut). Ids come from a single `pluck` before the
|
|
136
|
+
write, or from the records when the relation is already loaded. `created_at` and
|
|
137
|
+
`updated_at` are ignored by default (`c.ignored_attributes`), as are primary keys.
|
|
138
|
+
|
|
139
|
+
### Actor resolution
|
|
140
|
+
|
|
141
|
+
`c.actor { |ctx| ... }` receives a context with `#controller`, `#job`, `#request` and
|
|
142
|
+
`#env` and returns a user-like object, a Hash or a `Provenance::Actor`. Objects are
|
|
143
|
+
read through `provenance_actor` (returning a Hash) if defined, otherwise `id`,
|
|
144
|
+
`provenance_display`/`email`/`name` and `provenance_roles`. Resolver errors are
|
|
145
|
+
reported to `on_error`; the event is emitted without an actor.
|
|
146
|
+
|
|
147
|
+
## Configuration
|
|
148
|
+
|
|
149
|
+
```ruby
|
|
150
|
+
Provenance.configure do |c|
|
|
151
|
+
c.app_name = "billing" # default: Rails app module name, underscored
|
|
152
|
+
c.enabled = !Rails.env.test? # default: true
|
|
153
|
+
c.actor { |ctx| ctx.controller.try(:current_user) }
|
|
154
|
+
c.redact_attributes += %i[iban] # merged with Rails filter_parameters
|
|
155
|
+
c.ignored_attributes = %w[created_at updated_at]
|
|
156
|
+
c.format = :native # :native | :ocsf | :cloudevents
|
|
157
|
+
c.sink :logger # repeatable
|
|
158
|
+
c.sink :http, url: ENV["AUDIT_URL"], headers: { "Authorization" => "Bearer …" }
|
|
159
|
+
c.outbox = true
|
|
160
|
+
c.outbox_retention = 7.days
|
|
161
|
+
c.outbox_batch_size = 100
|
|
162
|
+
c.outbox_enqueue_relay = true # enqueue RelayJob after each write
|
|
163
|
+
c.integrity.key = ENV["PROVENANCE_HMAC_KEY"] # nil disables
|
|
164
|
+
c.audit_reads = false # GET/HEAD actions without changes are skipped
|
|
165
|
+
c.denied_exceptions = ["CanCan::AccessDenied", "Pundit::NotAuthorizedError"]
|
|
166
|
+
c.bulk_ids_limit = 500
|
|
167
|
+
c.on_error { |exception, event| Rails.error.report(exception) } # default: Rails.logger.error
|
|
168
|
+
end
|
|
169
|
+
```
|
|
170
|
+
|
|
171
|
+
Redaction uses `ActiveSupport::ParameterFilter`, so `filter_parameters` entries such
|
|
172
|
+
as `:passw` match partially. Redacted values become `"[REDACTED]"`; a `nil` side of an
|
|
173
|
+
update stays `nil`. Encrypted attributes are always redacted. Event metadata is
|
|
174
|
+
filtered the same way. Diff values are JSON-safe: times become ISO 8601, `BigDecimal`
|
|
175
|
+
a string, binary strings `"[BINARY n bytes]"`.
|
|
176
|
+
|
|
177
|
+
## Delivery
|
|
178
|
+
|
|
179
|
+
### Sinks
|
|
180
|
+
|
|
181
|
+
A sink is any object with `#deliver(batch)`, where `batch` is an Array of serialized
|
|
182
|
+
events (Hashes). A failing sink never breaks the request: errors go to `on_error`
|
|
183
|
+
and the other sinks still receive the batch.
|
|
184
|
+
|
|
185
|
+
| Sink | Options |
|
|
186
|
+
|---|---|
|
|
187
|
+
| `:logger` | `logger:` (default `Rails.logger`), `level:` (default `:info`); one JSON line per event |
|
|
188
|
+
| `:io` | any IO, e.g. `c.sink :io, $stdout`; one JSON line per event |
|
|
189
|
+
| `:http` | `url:`, `headers:`, `open_timeout:`, `read_timeout:`, `retries:`, `backoff:`, `max_backoff:`; POSTs a JSON array, 2xx is success, exponential backoff with jitter |
|
|
190
|
+
| `:proc` | a callable or block: `c.sink(:proc) { |batch| ... }` |
|
|
191
|
+
| `:memory` | keeps events in memory, for tests |
|
|
192
|
+
| `:kafka` | `require "provenance/sinks/kafka"` (needs `rdkafka`); `topic:`, `config:` or `producer:`; one message per event keyed by event id |
|
|
193
|
+
| custom | `c.sink MySink.new` |
|
|
194
|
+
|
|
195
|
+
### Outbox
|
|
196
|
+
|
|
197
|
+
Inline delivery runs in the request thread after commit. For reliable delivery set
|
|
198
|
+
`c.outbox = true`: events are written to the `provenance_outbox` table and
|
|
199
|
+
`Provenance::RelayJob` delivers them.
|
|
200
|
+
|
|
201
|
+
The relay drains pending rows in id order (`FOR UPDATE SKIP LOCKED` on PostgreSQL,
|
|
202
|
+
plain ordering elsewhere), delivers each batch to every sink, marks rows delivered and
|
|
203
|
+
retries failed batches with exponential backoff (capped at one hour), stopping at the
|
|
204
|
+
first row that is not due so delivery order is kept. Delivered rows older than
|
|
205
|
+
`outbox_retention` are pruned; the newest row per app is kept as the anchor of the
|
|
206
|
+
integrity chain. Delivery is at-least-once: a batch that fails on one sink is retried
|
|
207
|
+
on all of them.
|
|
208
|
+
|
|
209
|
+
`RelayJob` is enqueued after each write by default. You can also schedule it (for
|
|
210
|
+
example with Solid Queue recurring tasks) and turn `outbox_enqueue_relay` off, or run
|
|
211
|
+
`bin/rails provenance:relay`.
|
|
212
|
+
|
|
213
|
+
## Formats
|
|
214
|
+
|
|
215
|
+
The format applies at delivery time; the outbox always stores native events.
|
|
216
|
+
|
|
217
|
+
### Native
|
|
218
|
+
|
|
219
|
+
`provenance/event@2`, described above and in `schema/event-2.json`.
|
|
220
|
+
|
|
221
|
+
### CloudEvents
|
|
222
|
+
|
|
223
|
+
CloudEvents 1.0 structured JSON:
|
|
224
|
+
|
|
225
|
+
| CloudEvents | Value |
|
|
226
|
+
|---|---|
|
|
227
|
+
| `specversion` | `"1.0"` |
|
|
228
|
+
| `id` | event `id` |
|
|
229
|
+
| `source` | `app` |
|
|
230
|
+
| `type` | `"provenance.action.<kind>"` |
|
|
231
|
+
| `subject` | action name |
|
|
232
|
+
| `time` | `occurred_at` |
|
|
233
|
+
| `datacontenttype` | `"application/json"` |
|
|
234
|
+
| `dataschema` | `"urn:provenance:event:2"` |
|
|
235
|
+
| `data` | the native event |
|
|
236
|
+
|
|
237
|
+
### OCSF
|
|
238
|
+
|
|
239
|
+
OCSF 1.3: every action becomes one **API Activity** event (`class_uid` 6003,
|
|
240
|
+
`category_uid` 6) and every entity change one **Entity Management** event
|
|
241
|
+
(`class_uid` 3004, `category_uid` 3), so a single native event may produce several
|
|
242
|
+
OCSF events. All of them share `metadata.correlation_uid` = native event id.
|
|
243
|
+
|
|
244
|
+
Common fields:
|
|
245
|
+
|
|
246
|
+
| OCSF | Source |
|
|
247
|
+
|---|---|
|
|
248
|
+
| `time` | `occurred_at` as epoch milliseconds |
|
|
249
|
+
| `metadata.version` | `"1.3.0"` |
|
|
250
|
+
| `metadata.uid` | event id (API Activity), `"<event id>/<change index>"` (Entity Management) |
|
|
251
|
+
| `metadata.correlation_uid` | event id |
|
|
252
|
+
| `metadata.log_name` | `app` |
|
|
253
|
+
| `metadata.product` | `{name: "Provenance", vendor_name: "Provenance", version}` |
|
|
254
|
+
| `actor.app_name` | `app` |
|
|
255
|
+
| `actor.user.uid` / `.name` / `.type` | `actor.id` / `actor.display` / `actor.type` |
|
|
256
|
+
| `actor.user.groups[].name` | `actor.roles` |
|
|
257
|
+
| `status_id` / `status` | `success` → 1 Success; `failure`, `denied` → 2 Failure |
|
|
258
|
+
| `status_detail` | `outcome.result` |
|
|
259
|
+
| `severity_id` | `success` 1 Informational, `failure` 2 Low, `denied` 3 Medium |
|
|
260
|
+
| `type_uid` | `class_uid * 100 + activity_id` |
|
|
261
|
+
| `unmapped.action` | native `action` |
|
|
262
|
+
| `unmapped.metadata` | native `metadata` |
|
|
263
|
+
| `unmapped.impersonator` | `actor.impersonator` |
|
|
264
|
+
| `unmapped.integrity` | native `integrity` |
|
|
265
|
+
|
|
266
|
+
API Activity (6003):
|
|
267
|
+
|
|
268
|
+
| OCSF | Source |
|
|
269
|
+
|---|---|
|
|
270
|
+
| `activity_id` | POST 1 Create, GET/HEAD 2 Read, PUT/PATCH 3 Update, DELETE 4 Delete; non-HTTP actions use the change operations when they agree, else 99 Other |
|
|
271
|
+
| `api.operation` | action name |
|
|
272
|
+
| `api.request.uid` | request id (event id for non-HTTP actions) |
|
|
273
|
+
| `api.response.code` | HTTP status |
|
|
274
|
+
| `api.response.error` / `.error_message` | `outcome.error.class` / `.message` |
|
|
275
|
+
| `src_endpoint.ip` | `request.ip` |
|
|
276
|
+
| `http_request.http_method` / `.url.path` / `.user_agent` / `.uid` | `request.method` / `.path` / `.user_agent` / `.id` |
|
|
277
|
+
| `http_response.code` | `request.status` |
|
|
278
|
+
| `resources[]` | `{type: entity, uid: entity_id}` per change |
|
|
279
|
+
|
|
280
|
+
Entity Management (3004):
|
|
281
|
+
|
|
282
|
+
| OCSF | Source |
|
|
283
|
+
|---|---|
|
|
284
|
+
| `activity_id` | `create`, `bulk_insert` 1 Create; `update`, `bulk_update` 3 Update; `destroy`, `bulk_delete` 4 Delete; `link`, `unlink` 99 Other (`activity_name` "Link"/"Unlink") |
|
|
285
|
+
| `entity.name`, `entity.type` | `entity` |
|
|
286
|
+
| `entity.uid` | `entity_id` |
|
|
287
|
+
| `entity.data` | `diff` |
|
|
288
|
+
| `unmapped.change` | `operation`, `association`, `target`, `count`, `where`, `ids`, `truncated` |
|
|
289
|
+
|
|
290
|
+
## Integrity
|
|
291
|
+
|
|
292
|
+
With `c.integrity.key` set (requires the outbox; configuration raises otherwise), each
|
|
293
|
+
event gets
|
|
294
|
+
|
|
295
|
+
```json
|
|
296
|
+
"integrity": { "seq": 118, "prev": "<mac of seq 117>", "mac": "<hex>" }
|
|
297
|
+
```
|
|
298
|
+
|
|
299
|
+
where `seq` increases monotonically per app, and
|
|
300
|
+
`mac = HMAC-SHA256(key, prev + canonical_json(event without "integrity"))`. Canonical
|
|
301
|
+
JSON has sorted object keys and no whitespace; `prev` is `null` for the first event and
|
|
302
|
+
counts as an empty string in the MAC input. Sequence numbers are allocated from the outbox table (an advisory transaction
|
|
303
|
+
lock on PostgreSQL, a unique index plus retry elsewhere).
|
|
304
|
+
|
|
305
|
+
```ruby
|
|
306
|
+
Provenance::Integrity.verify(events, key: ENV["PROVENANCE_HMAC_KEY"]) # => nil or first broken seq
|
|
307
|
+
```
|
|
308
|
+
|
|
309
|
+
`bin/rails provenance:verify` checks the chain stored in the outbox and exits non-zero
|
|
310
|
+
on a broken chain. Forward events to an append-only store to keep evidence beyond the
|
|
311
|
+
outbox retention window.
|
|
312
|
+
|
|
313
|
+
## Testing
|
|
314
|
+
|
|
315
|
+
```ruby
|
|
316
|
+
# spec/rails_helper.rb
|
|
317
|
+
require "provenance/rspec"
|
|
318
|
+
```
|
|
319
|
+
|
|
320
|
+
```ruby
|
|
321
|
+
events = Provenance::Testing.capture { post "/invoices", params: { ... } }
|
|
322
|
+
|
|
323
|
+
expect { patch invoice_path(invoice), params: { invoice: { status: "sent" } } }
|
|
324
|
+
.to emit_provenance_event(action: "invoices#update", outcome: "success")
|
|
325
|
+
.with_change(entity: "Invoice", operation: "update", diff: including(status: ["draft", "sent"]))
|
|
326
|
+
|
|
327
|
+
expect { get invoices_path }.not_to emit_provenance_event
|
|
328
|
+
```
|
|
329
|
+
|
|
330
|
+
While capturing, tracking is enabled even if `c.enabled` is false, and events go to the
|
|
331
|
+
capture instead of the outbox or sinks. The matcher accepts `action`, `kind`,
|
|
332
|
+
`caused_by`, `outcome`, `error`, `actor`, `actor_id`, `request`, `status`, `metadata`
|
|
333
|
+
and `app`, any RSpec composable matcher as a value, `.with_change(...)` (repeatable)
|
|
334
|
+
and `.exactly(n)` / `.once`.
|
|
335
|
+
|
|
336
|
+
## Comparison
|
|
337
|
+
|
|
338
|
+
| | Provenance | paper_trail | audited | logidze |
|
|
339
|
+
|---|---|---|---|---|
|
|
340
|
+
| Unit of record | one event per action (request, job, task, block) | one version per record change | one audit per record change | record history in a JSONB column |
|
|
341
|
+
| Stored in | external sinks (via outbox) | app database | app database | app database (triggers) |
|
|
342
|
+
| Undo / reify | no | yes | yes (revisions) | yes |
|
|
343
|
+
| Request context and outcome (failure/denied) | yes | via whodunnit/metadata | via request uuid/comment | via meta |
|
|
344
|
+
| Bulk operations | summarized per call | no | no | yes (triggers) |
|
|
345
|
+
| Tamper evidence | HMAC chain | no | no | no |
|
|
346
|
+
| Wire formats | native, OCSF, CloudEvents | — | — | — |
|
|
347
|
+
|
|
348
|
+
Use paper_trail, audited or logidze when the application itself needs record history.
|
|
349
|
+
Use Provenance when security or compliance tooling needs to know who did what. They
|
|
350
|
+
can be combined.
|
|
351
|
+
|
|
352
|
+
## License
|
|
353
|
+
|
|
354
|
+
MIT, see [LICENSE](LICENSE).
|
|
355
|
+
|
|
356
|
+
[paper_trail]: https://github.com/paper-trail-gem/paper_trail
|
|
357
|
+
[audited]: https://github.com/collectiveidea/audited
|
|
358
|
+
[logidze]: https://github.com/palkan/logidze
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "rails/generators"
|
|
4
|
+
require "rails/generators/active_record"
|
|
5
|
+
|
|
6
|
+
module Provenance
|
|
7
|
+
# Rails generators shipped with Provenance.
|
|
8
|
+
module Generators
|
|
9
|
+
# +rails g provenance:install+: creates the outbox migration and an initializer.
|
|
10
|
+
class InstallGenerator < Rails::Generators::Base
|
|
11
|
+
include ActiveRecord::Generators::Migration
|
|
12
|
+
|
|
13
|
+
source_root File.expand_path("templates", __dir__)
|
|
14
|
+
|
|
15
|
+
desc "Creates the provenance_outbox migration and config/initializers/provenance.rb"
|
|
16
|
+
|
|
17
|
+
# @return [void]
|
|
18
|
+
def create_migration_file
|
|
19
|
+
migration_template "create_provenance_outbox.rb.tt", "db/migrate/create_provenance_outbox.rb"
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
# @return [void]
|
|
23
|
+
def create_initializer
|
|
24
|
+
template "initializer.rb.tt", "config/initializers/provenance.rb"
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
private
|
|
28
|
+
|
|
29
|
+
def migration_version
|
|
30
|
+
"[#{ActiveRecord::VERSION::MAJOR}.#{ActiveRecord::VERSION::MINOR}]"
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
end
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
class CreateProvenanceOutbox < ActiveRecord::Migration<%= migration_version %>
|
|
2
|
+
def change
|
|
3
|
+
create_table :provenance_outbox do |t|
|
|
4
|
+
t.string :app, null: false
|
|
5
|
+
t.string :event_id, null: false
|
|
6
|
+
t.bigint :seq
|
|
7
|
+
t.string :mac
|
|
8
|
+
t.text :payload, null: false
|
|
9
|
+
t.string :status, null: false, default: "pending"
|
|
10
|
+
t.integer :attempts, null: false, default: 0
|
|
11
|
+
t.datetime :next_attempt_at
|
|
12
|
+
t.datetime :delivered_at
|
|
13
|
+
t.text :last_error
|
|
14
|
+
t.timestamps
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
add_index :provenance_outbox, [:status, :id]
|
|
18
|
+
add_index :provenance_outbox, [:app, :seq], unique: true
|
|
19
|
+
add_index :provenance_outbox, :delivered_at
|
|
20
|
+
end
|
|
21
|
+
end
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
Provenance.configure do |c|
|
|
2
|
+
c.enabled = !Rails.env.test?
|
|
3
|
+
c.actor { |ctx| ctx.controller.try(:current_user) }
|
|
4
|
+
c.format = :native
|
|
5
|
+
c.sink :logger
|
|
6
|
+
|
|
7
|
+
# Reliable delivery through the provenance_outbox table (run the generated migration).
|
|
8
|
+
c.outbox = true
|
|
9
|
+
|
|
10
|
+
# Tamper evidence; requires the outbox.
|
|
11
|
+
# c.integrity.key = ENV["PROVENANCE_HMAC_KEY"]
|
|
12
|
+
end
|
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Provenance
|
|
4
|
+
# A unit of user intent: one HTTP request, job, rake task or explicit block.
|
|
5
|
+
class Action
|
|
6
|
+
# Accepted kinds.
|
|
7
|
+
KINDS = %w[http job task custom].freeze
|
|
8
|
+
# HTTP methods treated as reads.
|
|
9
|
+
READ_METHODS = %w[GET HEAD].freeze
|
|
10
|
+
# Maximum length of a recorded exception message.
|
|
11
|
+
MESSAGE_LIMIT = 500
|
|
12
|
+
|
|
13
|
+
# @return [String] UUIDv7
|
|
14
|
+
attr_reader :id
|
|
15
|
+
# @return [String]
|
|
16
|
+
attr_reader :kind
|
|
17
|
+
# @return [String, nil]
|
|
18
|
+
attr_accessor :name
|
|
19
|
+
# @return [String, nil] id of the action that caused this one
|
|
20
|
+
attr_accessor :caused_by
|
|
21
|
+
# @return [Context]
|
|
22
|
+
attr_reader :context
|
|
23
|
+
# @return [Hash] request attributes for HTTP actions
|
|
24
|
+
attr_reader :request
|
|
25
|
+
# @return [Hash{String => Object}]
|
|
26
|
+
attr_reader :metadata
|
|
27
|
+
# @return [Array<EntityChange>]
|
|
28
|
+
attr_reader :changes
|
|
29
|
+
# @return [Time]
|
|
30
|
+
attr_reader :started_at
|
|
31
|
+
# @return [Exception, nil]
|
|
32
|
+
attr_reader :exception
|
|
33
|
+
# @return [Actor, nil] actor set explicitly, without running the resolver
|
|
34
|
+
attr_reader :explicit_actor
|
|
35
|
+
|
|
36
|
+
# @param kind [Symbol, String]
|
|
37
|
+
# @param name [String, nil]
|
|
38
|
+
# @param actor [Object, nil]
|
|
39
|
+
# @param metadata [Hash]
|
|
40
|
+
# @param explicit [Boolean] explicit actions are always auditable
|
|
41
|
+
# @param caused_by [String, nil]
|
|
42
|
+
# @param context [Context, nil]
|
|
43
|
+
def initialize(kind:, name:, actor: nil, metadata: {}, explicit: false, caused_by: nil, context: nil)
|
|
44
|
+
@kind = kind.to_s
|
|
45
|
+
raise ArgumentError, "unknown action kind #{kind.inspect}" unless KINDS.include?(@kind)
|
|
46
|
+
|
|
47
|
+
@started_at = Time.now.utc
|
|
48
|
+
@id = UUID.v7(@started_at)
|
|
49
|
+
@name = name
|
|
50
|
+
@explicit_actor = Actor.wrap(actor)
|
|
51
|
+
@metadata = {}
|
|
52
|
+
annotate(**metadata)
|
|
53
|
+
@explicit = explicit
|
|
54
|
+
@caused_by = caused_by
|
|
55
|
+
@context = context || Context.new
|
|
56
|
+
@request = {}
|
|
57
|
+
@changes = []
|
|
58
|
+
@lock = Mutex.new
|
|
59
|
+
@skipped = false
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
# Merges data into the action metadata.
|
|
63
|
+
#
|
|
64
|
+
# @param data [Hash]
|
|
65
|
+
# @return [self]
|
|
66
|
+
def annotate(**data)
|
|
67
|
+
@metadata.merge!(data.deep_stringify_keys)
|
|
68
|
+
self
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
# @param value [Object, Hash, Actor, nil] anything {Actor.wrap} accepts
|
|
72
|
+
def actor=(value)
|
|
73
|
+
@explicit_actor = Actor.wrap(value)
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
# The actor, resolved through the configured resolver when not set explicitly.
|
|
77
|
+
# Resolver failures are reported to +on_error+ and yield +nil+.
|
|
78
|
+
#
|
|
79
|
+
# @return [Actor, nil]
|
|
80
|
+
def actor
|
|
81
|
+
return @explicit_actor if @explicit_actor
|
|
82
|
+
|
|
83
|
+
resolver = Provenance.config.actor_resolver
|
|
84
|
+
return nil unless resolver
|
|
85
|
+
|
|
86
|
+
resolved = Actor.wrap(resolver.call(context))
|
|
87
|
+
@explicit_actor = resolved if resolved
|
|
88
|
+
resolved
|
|
89
|
+
rescue => e
|
|
90
|
+
Provenance.config.report_error(e)
|
|
91
|
+
nil
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
# @param change [EntityChange]
|
|
95
|
+
# @return [void]
|
|
96
|
+
def add_change(change)
|
|
97
|
+
@lock.synchronize { @changes << change }
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
# Records an exception as the outcome. Only the first exception is kept.
|
|
101
|
+
#
|
|
102
|
+
# @param exception [Exception]
|
|
103
|
+
# @return [void]
|
|
104
|
+
def fail!(exception)
|
|
105
|
+
@exception ||= exception
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
# Excludes this action from emission.
|
|
109
|
+
#
|
|
110
|
+
# @return [void]
|
|
111
|
+
def skip!
|
|
112
|
+
@skipped = true
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
# @return [Boolean]
|
|
116
|
+
def skipped?
|
|
117
|
+
@skipped
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
# @return [Boolean]
|
|
121
|
+
def explicit?
|
|
122
|
+
@explicit
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
# @return [String] "success", "failure" or "denied"
|
|
126
|
+
def result
|
|
127
|
+
if exception
|
|
128
|
+
Provenance.config.denied?(exception) ? "denied" : "failure"
|
|
129
|
+
else
|
|
130
|
+
case request["status"].to_i
|
|
131
|
+
when 401, 403 then "denied"
|
|
132
|
+
when 500..599 then "failure"
|
|
133
|
+
else "success"
|
|
134
|
+
end
|
|
135
|
+
end
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
# @return [Hash, nil] {"class", "message"} of the recorded exception
|
|
139
|
+
def error
|
|
140
|
+
return nil unless exception
|
|
141
|
+
|
|
142
|
+
{"class" => exception.class.name, "message" => exception.message.to_s.truncate(MESSAGE_LIMIT)}
|
|
143
|
+
end
|
|
144
|
+
|
|
145
|
+
# @return [Boolean] whether an HTTP action used GET or HEAD
|
|
146
|
+
def read?
|
|
147
|
+
kind == "http" && READ_METHODS.include?(request["method"].to_s.upcase)
|
|
148
|
+
end
|
|
149
|
+
|
|
150
|
+
# Whether the action produces an event (section 4.4).
|
|
151
|
+
#
|
|
152
|
+
# @return [Boolean]
|
|
153
|
+
def auditable?
|
|
154
|
+
return false if skipped?
|
|
155
|
+
|
|
156
|
+
changes.any? || result != "success" || (kind == "http" && !read?) ||
|
|
157
|
+
Provenance.config.audit_reads || explicit?
|
|
158
|
+
end
|
|
159
|
+
end
|
|
160
|
+
end
|