event_engine-delivery 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +23 -0
- data/MIT-LICENSE +20 -0
- data/README.md +493 -0
- data/Rakefile +9 -0
- data/app/assets/config/event_engine_manifest.js +1 -0
- data/app/assets/stylesheets/event_engine/application.css +15 -0
- data/app/controllers/event_engine/application_controller.rb +4 -0
- data/app/helpers/event_engine/application_helper.rb +4 -0
- data/app/jobs/event_engine/application_job.rb +4 -0
- data/app/jobs/event_engine/outbox_cleanup_job.rb +13 -0
- data/app/jobs/event_engine/publish_outbox_events_job.rb +16 -0
- data/app/mailers/event_engine/application_mailer.rb +6 -0
- data/app/models/event_engine/application_record.rb +5 -0
- data/app/models/event_engine/outbox_event.rb +123 -0
- data/app/views/layouts/event_engine/application.html.erb +15 -0
- data/config/routes.rb +2 -0
- data/db/migrate/20251216190654_create_event_engine_outbox_events.rb +9 -0
- data/db/migrate/20251216194009_add_event_type_to_event_engine_outbox_events.rb +5 -0
- data/db/migrate/20251216201941_add_payload_to_event_engine_outbox_events.rb +9 -0
- data/db/migrate/20251216203339_add_published_at_to_event_engine_outbox_events.rb +5 -0
- data/db/migrate/20251216213819_add_idempotency_key_to_event_engine_outbox_events.rb +6 -0
- data/db/migrate/20251217155422_add_attempts_to_event_engine_outbox_events.rb +5 -0
- data/db/migrate/20251217174013_add_dead_lettered_at_to_event_engine_outbox_events.rb +5 -0
- data/db/migrate/20251220213944_add_event_version_to_outbox_events.rb +20 -0
- data/db/migrate/20251221193107_add_occurred_at_and_metadata_to_outbox_events.rb +6 -0
- data/db/migrate/20260127000000_add_indexes_to_event_engine_outbox_events.rb +8 -0
- data/db/migrate/20260212000001_add_constraints_and_indexes_to_event_engine_outbox_events.rb +6 -0
- data/db/migrate/20260212000002_add_error_context_to_event_engine_outbox_events.rb +6 -0
- data/db/migrate/20260212000003_add_aggregate_columns_to_event_engine_outbox_events.rb +9 -0
- data/db/migrate/20260212000005_add_process_type_to_event_engine_outbox_events.rb +5 -0
- data/db/migrate/20260921000000_add_subject_and_domain_to_event_engine_outbox_events.rb +6 -0
- data/lib/event_engine/cloud/api_client.rb +62 -0
- data/lib/event_engine/cloud/batch.rb +50 -0
- data/lib/event_engine/cloud/reporter.rb +155 -0
- data/lib/event_engine/cloud/serializer.rb +56 -0
- data/lib/event_engine/cloud/subscribers.rb +46 -0
- data/lib/event_engine/definition_transport_check.rb +35 -0
- data/lib/event_engine/delivery/configuration.rb +65 -0
- data/lib/event_engine/delivery/engine.rb +25 -0
- data/lib/event_engine/delivery/handler.rb +52 -0
- data/lib/event_engine/delivery/version.rb +5 -0
- data/lib/event_engine/delivery.rb +61 -0
- data/lib/event_engine/kafka_producer.rb +16 -0
- data/lib/event_engine/locking_strategy.rb +31 -0
- data/lib/event_engine/outbox_publisher.rb +89 -0
- data/lib/event_engine/outbox_router.rb +58 -0
- data/lib/event_engine/outbox_writer.rb +7 -0
- data/lib/event_engine/transports/in_memory_transport.rb +27 -0
- data/lib/event_engine/transports/kafka.rb +45 -0
- data/lib/event_engine/transports/null_transport.rb +27 -0
- data/lib/event_engine-delivery.rb +20 -0
- data/lib/generators/event_engine/install_generator.rb +45 -0
- data/lib/generators/event_engine/templates/event_schema.rb +10 -0
- data/lib/generators/event_engine/templates/initializer.rb +14 -0
- data/lib/tasks/event_engine_dead_letters.rake +65 -0
- data/lib/tasks/event_engine_outbox.rake +25 -0
- data/lib/tasks/event_engine_schema.rake +49 -0
- data/lib/tasks/event_engine_schema_check.rake +20 -0
- data/lib/tasks/event_engine_tasks.rake +4 -0
- metadata +183 -0
checksums.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
---
|
|
2
|
+
SHA256:
|
|
3
|
+
metadata.gz: 11a0ab2cbc92fee65fae05651dc9908743196fae615427bb0bfbcfab5b2c97d0
|
|
4
|
+
data.tar.gz: 0104a00e5c2ca4434500ddd21690b08188cfe66fb6b298db34ace0276af8bd66
|
|
5
|
+
SHA512:
|
|
6
|
+
metadata.gz: 9fba2892fbf51bf6b50ce2cfd5b7dc585b71053e02a2e06ee83c7facff638171b829d58cbf04538545e0a4656d64b36ddabb3a2fd3e8ac38d7375bc98995f215
|
|
7
|
+
data.tar.gz: 8066c109de6d3e75875ca276eff90e302a24f2c3eaf0695a9ef6cecf3c399414fc13899261d0d6568389e2398506ba7fb2c93d7cfa37caea0f955b45118fad8a
|
data/CHANGELOG.md
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to this project are documented here, following
|
|
4
|
+
[Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and
|
|
5
|
+
[Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
6
|
+
|
|
7
|
+
## [Unreleased]
|
|
8
|
+
|
|
9
|
+
### Added
|
|
10
|
+
- Initial gem scaffold: mountable `EventEngine::Delivery` Rails engine depending on
|
|
11
|
+
`event_engine`.
|
|
12
|
+
- Verbatim copy of `event_engine` as the starting point for the delivery layer
|
|
13
|
+
(standalone, depends only on Rails); full suite green. Inherited RuboCop offenses
|
|
14
|
+
grandfathered in `.rubocop_todo.yml` so the lint gate still checks new code.
|
|
15
|
+
|
|
16
|
+
### Changed
|
|
17
|
+
- Connected to `event_engine` core: deleted the duplicated core files (definitions,
|
|
18
|
+
schema, registry, `Event`, `EventBuilder`, subscribers, etc.) and now depend on the
|
|
19
|
+
`event_engine` gem for them. Delivery registers its pipeline as a handler via the new
|
|
20
|
+
`EventEngine::Delivery::Handler` (built `Event` → level 1 subscribers / 2 job / 3+
|
|
21
|
+
outbox+transport). Delivery configuration moved to `EventEngine::Delivery.configure`
|
|
22
|
+
(`transport`, `delivery_adapter`, `batch_size`, `retention_period`, `cloud_*`). The
|
|
23
|
+
dashboard mounts on `EventEngine::Delivery::Engine`.
|
data/MIT-LICENSE
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
Copyright tylercschneider
|
|
2
|
+
|
|
3
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
|
4
|
+
a copy of this software and associated documentation files (the
|
|
5
|
+
"Software"), to deal in the Software without restriction, including
|
|
6
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
|
7
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
|
8
|
+
permit persons to whom the Software is furnished to do so, subject to
|
|
9
|
+
the following conditions:
|
|
10
|
+
|
|
11
|
+
The above copyright notice and this permission notice shall be
|
|
12
|
+
included in all copies or substantial portions of the Software.
|
|
13
|
+
|
|
14
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
|
15
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
|
16
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
|
17
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
|
18
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
|
19
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
|
20
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
|
@@ -0,0 +1,493 @@
|
|
|
1
|
+
# EventEngine::Delivery
|
|
2
|
+
|
|
3
|
+
The **delivery layer** for [EventEngine](https://github.com/tylercschneider/event_engine).
|
|
4
|
+
|
|
5
|
+
`event_engine` (the core) declares events with a schema-first DSL, compiles them to
|
|
6
|
+
a committed schema, and **dispatches** built events to registered handlers. It does
|
|
7
|
+
not deliver anything on its own. `event_engine-delivery` is the handler that takes an
|
|
8
|
+
emitted event and **gets it delivered reliably**:
|
|
9
|
+
|
|
10
|
+
- the durability **level ladder** (1 sync → 2 job → 3 outbox → 4 broker),
|
|
11
|
+
- a transactional **outbox** (`event_engine_outbox_events`),
|
|
12
|
+
- **retries** and **dead-letter** handling,
|
|
13
|
+
- pluggable **transports** (in-memory, Kafka, or your own),
|
|
14
|
+
- an observability **dashboard**,
|
|
15
|
+
- an optional **cloud reporter**.
|
|
16
|
+
|
|
17
|
+
Apps that only need to *declare* events depend on `event_engine` alone. Add this gem
|
|
18
|
+
when you need durable, reliable delivery.
|
|
19
|
+
|
|
20
|
+
> **Namespacing note (transitional).** This gem is being extracted from `event_engine`
|
|
21
|
+
> via copy-then-subtract. Most of its classes still live under the top-level
|
|
22
|
+
> `EventEngine::` namespace (e.g. `EventEngine::Transports::Kafka`,
|
|
23
|
+
> `EventEngine::OutboxEvent`, `EventEngine::OutboxPublisher`). The delivery-specific
|
|
24
|
+
> entry points are `EventEngine::Delivery` (config, engine, handler). The examples
|
|
25
|
+
> below use the namespaces as they exist today.
|
|
26
|
+
|
|
27
|
+
---
|
|
28
|
+
|
|
29
|
+
## Table of contents
|
|
30
|
+
|
|
31
|
+
- [Installation](#installation)
|
|
32
|
+
- [How it hooks into the core](#how-it-hooks-into-the-core)
|
|
33
|
+
- [Level routing](#level-routing)
|
|
34
|
+
- [The outbox table](#the-outbox-table)
|
|
35
|
+
- [Configuration](#configuration)
|
|
36
|
+
- [Delivery adapters](#delivery-adapters)
|
|
37
|
+
- [Transports](#transports)
|
|
38
|
+
- [Writing a custom transport](#writing-a-custom-transport)
|
|
39
|
+
- [Customizing Kafka topics / payloads](#customizing-kafka-topics--payloads)
|
|
40
|
+
- [Idempotency](#idempotency)
|
|
41
|
+
- [Dead-letter recovery](#dead-letter-recovery)
|
|
42
|
+
- [Outbox cleanup](#outbox-cleanup)
|
|
43
|
+
- [Instrumentation](#instrumentation)
|
|
44
|
+
- [Dashboard](#dashboard)
|
|
45
|
+
- [Cloud reporter](#cloud-reporter)
|
|
46
|
+
- [Customization recipes](#customization-recipes)
|
|
47
|
+
- [License](#license)
|
|
48
|
+
|
|
49
|
+
---
|
|
50
|
+
|
|
51
|
+
## Installation
|
|
52
|
+
|
|
53
|
+
```ruby
|
|
54
|
+
# Gemfile
|
|
55
|
+
gem "event_engine"
|
|
56
|
+
gem "event_engine-delivery"
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
```bash
|
|
60
|
+
bundle install
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
Install the outbox migration and run it:
|
|
64
|
+
|
|
65
|
+
```bash
|
|
66
|
+
bin/rails event_engine:install:migrations # copies the outbox migrations into your app
|
|
67
|
+
bin/rails db:migrate
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
Then add an initializer:
|
|
71
|
+
|
|
72
|
+
```ruby
|
|
73
|
+
# config/initializers/event_engine_delivery.rb
|
|
74
|
+
EventEngine::Delivery.configure do |config|
|
|
75
|
+
config.delivery_adapter = :inline # :inline | :active_job | :manual
|
|
76
|
+
config.transport = EventEngine::Transports::InMemoryTransport.new
|
|
77
|
+
config.batch_size = 100
|
|
78
|
+
config.max_attempts = 5
|
|
79
|
+
end
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
> **Configure delivery via `EventEngine::Delivery.configure`.** Delivery options
|
|
83
|
+
> (`delivery_adapter`, `transport`, …) live on `EventEngine::Delivery::Configuration`.
|
|
84
|
+
> The core `EventEngine.configure` only knows about `logger`.
|
|
85
|
+
|
|
86
|
+
---
|
|
87
|
+
|
|
88
|
+
## How it hooks into the core
|
|
89
|
+
|
|
90
|
+
At Rails boot, the delivery engine registers a single handler with the core, for all
|
|
91
|
+
levels:
|
|
92
|
+
|
|
93
|
+
```ruby
|
|
94
|
+
# lib/event_engine/delivery/engine.rb
|
|
95
|
+
initializer "event_engine.delivery.register_handler" do
|
|
96
|
+
config.after_initialize do
|
|
97
|
+
EventEngine.register_handler(Handler.new, levels: :all)
|
|
98
|
+
Engine.send(:start_cloud_reporter!)
|
|
99
|
+
end
|
|
100
|
+
end
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
From then on, every `EventEngine.<event>` call dispatches into
|
|
104
|
+
`EventEngine::Delivery::Handler#call`, which routes by `event_level`.
|
|
105
|
+
|
|
106
|
+
---
|
|
107
|
+
|
|
108
|
+
## Level routing
|
|
109
|
+
|
|
110
|
+
`EventEngine::Delivery::Handler` is the brain. It maps each event's level to a
|
|
111
|
+
delivery strategy:
|
|
112
|
+
|
|
113
|
+
```ruby
|
|
114
|
+
def call(event)
|
|
115
|
+
case event.event_level
|
|
116
|
+
when 1 then dispatch_synchronously(event) # invoke subscribers now, in-process
|
|
117
|
+
when 2 then dispatch_in_background(event) # enqueue DispatchSubscribersJob
|
|
118
|
+
else write_and_publish(event) # 3, 4, and nil → outbox + publish
|
|
119
|
+
end
|
|
120
|
+
end
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
- **Level 1** — calls every `EventEngine::Subscriber` registered for the event,
|
|
124
|
+
synchronously, in the caller's stack.
|
|
125
|
+
- **Level 2** — enqueues `DispatchSubscribersJob`, which invokes the same subscribers
|
|
126
|
+
in a background job.
|
|
127
|
+
- **Levels 3+ (and `nil`)** — writes an `OutboxEvent` row, fires
|
|
128
|
+
`event_engine.event_emitted`, and triggers publishing per the configured
|
|
129
|
+
[delivery adapter](#delivery-adapters). When the outbox drains,
|
|
130
|
+
`EventEngine::OutboxRouter` differentiates:
|
|
131
|
+
- **level 3** → invoke in-app subscribers,
|
|
132
|
+
- **level 4** → publish to the configured **transport** (raises
|
|
133
|
+
`MissingTransportError` if none/Null),
|
|
134
|
+
- **level 5** → raises `UnsupportedLevelError` (reserved, unsupported).
|
|
135
|
+
|
|
136
|
+
> **`nil` levels fall into the outbox path** (the `else` branch). If you didn't set
|
|
137
|
+
> `event_level` on a definition, its events behave like level 3+. Set it explicitly.
|
|
138
|
+
|
|
139
|
+
---
|
|
140
|
+
|
|
141
|
+
## The outbox table
|
|
142
|
+
|
|
143
|
+
Events at level 3+ are persisted to `event_engine_outbox_events` before delivery, so
|
|
144
|
+
they survive a crash and are written atomically with your transaction. The
|
|
145
|
+
`EventEngine::OutboxEvent` model is **append-only for its core fields**
|
|
146
|
+
(`attr_readonly`), with these columns:
|
|
147
|
+
|
|
148
|
+
| Column | Type | Notes |
|
|
149
|
+
|---|---|---|
|
|
150
|
+
| `event_name` | string | NOT NULL |
|
|
151
|
+
| `event_type` | string | NOT NULL |
|
|
152
|
+
| `event_version` | integer | NOT NULL |
|
|
153
|
+
| `payload` | json | NOT NULL |
|
|
154
|
+
| `metadata` | json | optional context |
|
|
155
|
+
| `idempotency_key` | string | **unique index** |
|
|
156
|
+
| `occurred_at` | datetime | NOT NULL |
|
|
157
|
+
| `published_at` | datetime | set when delivered |
|
|
158
|
+
| `dead_lettered_at` | datetime | set when attempts exhausted |
|
|
159
|
+
| `attempts` | integer | default 0 |
|
|
160
|
+
| `last_error_message` | text | last failure message |
|
|
161
|
+
| `last_error_class` | string | last failure class |
|
|
162
|
+
| `aggregate_type` / `aggregate_id` / `aggregate_version` | string / string / integer | aggregate tracking |
|
|
163
|
+
| `event_level` | integer | the dispatched level |
|
|
164
|
+
| `created_at` / `updated_at` | datetime | standard |
|
|
165
|
+
|
|
166
|
+
Useful scopes and methods:
|
|
167
|
+
|
|
168
|
+
```ruby
|
|
169
|
+
OutboxEvent.unpublished # published_at IS NULL
|
|
170
|
+
OutboxEvent.active # not dead-lettered
|
|
171
|
+
OutboxEvent.dead_lettered # dead_lettered_at IS NOT NULL
|
|
172
|
+
OutboxEvent.retryable(max) # attempts < max
|
|
173
|
+
OutboxEvent.cleanable # published and not dead-lettered
|
|
174
|
+
OutboxEvent.for_aggregate(t,i) # ordered events for an aggregate
|
|
175
|
+
|
|
176
|
+
event.retry! # reset attempts + clear dead-letter + clear last_error
|
|
177
|
+
event.dead_letter! # mark dead-lettered now
|
|
178
|
+
event.mark_published! # stamp published_at
|
|
179
|
+
OutboxEvent.next_aggregate_version(type, id)
|
|
180
|
+
```
|
|
181
|
+
|
|
182
|
+
---
|
|
183
|
+
|
|
184
|
+
## Configuration
|
|
185
|
+
|
|
186
|
+
All on `EventEngine::Delivery.configure`:
|
|
187
|
+
|
|
188
|
+
| Option | Default | Purpose |
|
|
189
|
+
|---|---|---|
|
|
190
|
+
| `delivery_adapter` | `:inline` | `:inline`, `:active_job`, or `:manual` |
|
|
191
|
+
| `transport` | `NullTransport` | Object responding to `#publish(event)` |
|
|
192
|
+
| `batch_size` | `100` | Max events per `OutboxPublisher` batch |
|
|
193
|
+
| `max_attempts` | `5` | Publish attempts before dead-lettering |
|
|
194
|
+
| `retention_period` | `nil` | Age after which published events are cleanable (`nil` = keep) |
|
|
195
|
+
| `dashboard_auth` | `nil` | Callable `->(controller) { bool }` gating the dashboard |
|
|
196
|
+
| `logger` | `Rails.logger` | Where delivery logs |
|
|
197
|
+
| `cloud_api_key` | `nil` | Enables the cloud reporter when set |
|
|
198
|
+
| `cloud_endpoint` | `https://api.eventengine.dev/v1/ingest` | Cloud ingest URL |
|
|
199
|
+
| `cloud_environment` | `nil` | Environment label |
|
|
200
|
+
| `cloud_app_name` | `nil` | App label |
|
|
201
|
+
| `cloud_batch_size` | `50` | Cloud entries per flush |
|
|
202
|
+
| `cloud_flush_interval` | `10` | Seconds between cloud flushes |
|
|
203
|
+
|
|
204
|
+
`EventEngine::Delivery::Configuration#validate!` enforces the invariants you'd want:
|
|
205
|
+
`delivery_adapter` must be one of `:inline/:active_job/:manual`; `:active_job`
|
|
206
|
+
requires a real transport (not `NullTransport`); a transport must respond to
|
|
207
|
+
`#publish`; `batch_size`/`max_attempts` must be positive integers. It raises
|
|
208
|
+
`InvalidConfigurationError` otherwise.
|
|
209
|
+
|
|
210
|
+
---
|
|
211
|
+
|
|
212
|
+
## Delivery adapters
|
|
213
|
+
|
|
214
|
+
`delivery_adapter` decides *when* the outbox is drained after a level-3+ write:
|
|
215
|
+
|
|
216
|
+
- **`:inline`** (default) — drains in-process. Inside an open transaction it
|
|
217
|
+
registers an after-commit callback (so publishing happens only once your data is
|
|
218
|
+
committed); outside a transaction it publishes immediately. Great for monoliths and
|
|
219
|
+
tests.
|
|
220
|
+
- **`:active_job`** — enqueues `EventEngine::PublishOutboxEventsJob`, which runs
|
|
221
|
+
`OutboxPublisher`. Use this in production so request latency isn't tied to delivery.
|
|
222
|
+
Requires a real transport (validated).
|
|
223
|
+
- **`:manual`** — does nothing automatically. You drain the outbox yourself — a cron
|
|
224
|
+
job, a rake task, or an operator action calling `OutboxPublisher`. Choose this when
|
|
225
|
+
you want full control over publish timing (e.g. batch windows, maintenance pauses).
|
|
226
|
+
|
|
227
|
+
Draining manually:
|
|
228
|
+
|
|
229
|
+
```ruby
|
|
230
|
+
EventEngine::OutboxPublisher.new(
|
|
231
|
+
router: EventEngine::OutboxRouter.new(transport: EventEngine::Delivery.configuration.transport),
|
|
232
|
+
batch_size: EventEngine::Delivery.configuration.batch_size,
|
|
233
|
+
max_attempts: EventEngine::Delivery.configuration.max_attempts
|
|
234
|
+
).call
|
|
235
|
+
```
|
|
236
|
+
|
|
237
|
+
---
|
|
238
|
+
|
|
239
|
+
## Transports
|
|
240
|
+
|
|
241
|
+
A transport is any object that responds to `publish(event)` and raises on failure
|
|
242
|
+
(so the publisher can retry). Three ship with the gem:
|
|
243
|
+
|
|
244
|
+
| Transport | Use |
|
|
245
|
+
|---|---|
|
|
246
|
+
| `EventEngine::Transports::NullTransport` | Default. Logs a warning and discards. Counts as "no transport" for level-4 checks. |
|
|
247
|
+
| `EventEngine::Transports::InMemoryTransport` | Dev/test. Collects events in `#events` for assertions. |
|
|
248
|
+
| `EventEngine::Transports::Kafka` | Production. Wraps a producer; publishes to topics `events.{event_name}`. |
|
|
249
|
+
|
|
250
|
+
```ruby
|
|
251
|
+
# dev/test
|
|
252
|
+
config.transport = EventEngine::Transports::InMemoryTransport.new
|
|
253
|
+
|
|
254
|
+
# Kafka — you bring the client; EventEngine never manages Kafka
|
|
255
|
+
kafka = Kafka.new(seed_brokers: ENV["KAFKA_BROKERS"])
|
|
256
|
+
producer = EventEngine::KafkaProducer.new(client: kafka)
|
|
257
|
+
config.transport = EventEngine::Transports::Kafka.new(producer: producer)
|
|
258
|
+
```
|
|
259
|
+
|
|
260
|
+
### Writing a custom transport
|
|
261
|
+
|
|
262
|
+
The contract is one method. Raise on failure to trigger retry/dead-lettering.
|
|
263
|
+
|
|
264
|
+
```ruby
|
|
265
|
+
class SqsTransport
|
|
266
|
+
def initialize(client:, queue_url:)
|
|
267
|
+
@client = client
|
|
268
|
+
@queue_url = queue_url
|
|
269
|
+
end
|
|
270
|
+
|
|
271
|
+
# `event` is the persisted EventEngine::OutboxEvent. It exposes:
|
|
272
|
+
# event_name, event_type, event_version, idempotency_key,
|
|
273
|
+
# payload, metadata, occurred_at, aggregate_type/id/version
|
|
274
|
+
def publish(event)
|
|
275
|
+
@client.send_message(
|
|
276
|
+
queue_url: @queue_url,
|
|
277
|
+
message_body: JSON.generate(
|
|
278
|
+
name: event.event_name,
|
|
279
|
+
version: event.event_version,
|
|
280
|
+
key: event.idempotency_key,
|
|
281
|
+
payload: event.payload,
|
|
282
|
+
meta: event.metadata
|
|
283
|
+
)
|
|
284
|
+
)
|
|
285
|
+
# raise on a non-success response so it retries
|
|
286
|
+
end
|
|
287
|
+
end
|
|
288
|
+
|
|
289
|
+
EventEngine::Delivery.configure { |c| c.transport = SqsTransport.new(client: sqs, queue_url: url) }
|
|
290
|
+
```
|
|
291
|
+
|
|
292
|
+
**Why** a custom transport: target a broker EventEngine doesn't ship (SQS, SNS,
|
|
293
|
+
RabbitMQ, Redis Streams, an internal HTTP bus), add tracing/headers, or fan out to
|
|
294
|
+
multiple destinations from one `publish`.
|
|
295
|
+
|
|
296
|
+
### Customizing Kafka topics / payloads
|
|
297
|
+
|
|
298
|
+
The built-in Kafka transport hardcodes the topic as `events.{event_name}` and a
|
|
299
|
+
fixed JSON shape. To change either (e.g. environment-namespaced topics like
|
|
300
|
+
`prod.events.cow_fed`, a partition key, or a different envelope), write a thin
|
|
301
|
+
transport instead of using the built-in one:
|
|
302
|
+
|
|
303
|
+
```ruby
|
|
304
|
+
class NamespacedKafka
|
|
305
|
+
def initialize(producer:, prefix: Rails.env)
|
|
306
|
+
@producer = producer
|
|
307
|
+
@prefix = prefix
|
|
308
|
+
end
|
|
309
|
+
|
|
310
|
+
def publish(event)
|
|
311
|
+
topic = "#{@prefix}.events.#{event.event_name}"
|
|
312
|
+
@producer.publish(topic, envelope(event), key: event.aggregate_id)
|
|
313
|
+
end
|
|
314
|
+
|
|
315
|
+
private
|
|
316
|
+
|
|
317
|
+
def envelope(event)
|
|
318
|
+
{ name: event.event_name, version: event.event_version,
|
|
319
|
+
key: event.idempotency_key, payload: event.payload,
|
|
320
|
+
occurred_at: event.occurred_at }
|
|
321
|
+
end
|
|
322
|
+
end
|
|
323
|
+
```
|
|
324
|
+
|
|
325
|
+
**Why:** a shared Kafka cluster across environments needs topic namespacing to avoid
|
|
326
|
+
cross-environment contamination; ordered consumers need a deterministic partition
|
|
327
|
+
key (usually the aggregate id).
|
|
328
|
+
|
|
329
|
+
---
|
|
330
|
+
|
|
331
|
+
## Idempotency
|
|
332
|
+
|
|
333
|
+
Every outbox event has an `idempotency_key` — an auto-generated UUID, or one you
|
|
334
|
+
supply at emit time:
|
|
335
|
+
|
|
336
|
+
```ruby
|
|
337
|
+
EventEngine.cow_fed(cow: cow, idempotency_key: "cow-#{cow.id}-fed-#{Date.current}")
|
|
338
|
+
```
|
|
339
|
+
|
|
340
|
+
- The column has a **unique index**, so the same logical event can't be written
|
|
341
|
+
twice.
|
|
342
|
+
- The key is passed through to transports so **consumers can deduplicate**.
|
|
343
|
+
- EventEngine stores and transmits the key but does **not** enforce idempotent
|
|
344
|
+
*processing* downstream — consumers must dedupe on their end.
|
|
345
|
+
|
|
346
|
+
Override the auto-UUID when you want domain-level dedup (one feed per cow per day),
|
|
347
|
+
to make user-action retries safe, or to correlate across systems.
|
|
348
|
+
|
|
349
|
+
---
|
|
350
|
+
|
|
351
|
+
## Dead-letter recovery
|
|
352
|
+
|
|
353
|
+
After `max_attempts` failed publishes, an event is dead-lettered (its
|
|
354
|
+
`dead_lettered_at`, `last_error_message`, and `last_error_class` are set).
|
|
355
|
+
|
|
356
|
+
```bash
|
|
357
|
+
bin/rails event_engine:dead_letters:list # list dead-lettered events
|
|
358
|
+
bin/rails event_engine:dead_letters:retry[123] # retry one by id
|
|
359
|
+
bin/rails event_engine:dead_letters:retry:all # retry every dead-lettered event
|
|
360
|
+
```
|
|
361
|
+
|
|
362
|
+
Programmatically:
|
|
363
|
+
|
|
364
|
+
```ruby
|
|
365
|
+
event = EventEngine::OutboxEvent.dead_lettered.find(123)
|
|
366
|
+
event.retry! # attempts → 0, dead_lettered_at → nil, last_error_* cleared
|
|
367
|
+
```
|
|
368
|
+
|
|
369
|
+
Typical loop: `dead_letters:list` → diagnose via `last_error_*` → fix the cause →
|
|
370
|
+
`dead_letters:retry:all`.
|
|
371
|
+
|
|
372
|
+
---
|
|
373
|
+
|
|
374
|
+
## Outbox cleanup
|
|
375
|
+
|
|
376
|
+
Published events accumulate. Configure a retention window and the cleaner deletes
|
|
377
|
+
**only** events that are both published and not dead-lettered:
|
|
378
|
+
|
|
379
|
+
```ruby
|
|
380
|
+
config.retention_period = 30.days # nil disables cleanup entirely
|
|
381
|
+
```
|
|
382
|
+
|
|
383
|
+
```bash
|
|
384
|
+
bin/rails event_engine:outbox:cleanup
|
|
385
|
+
```
|
|
386
|
+
|
|
387
|
+
Or schedule `EventEngine::OutboxCleanupJob` (it no-ops when `retention_period` is
|
|
388
|
+
nil):
|
|
389
|
+
|
|
390
|
+
```ruby
|
|
391
|
+
# e.g. sidekiq-cron
|
|
392
|
+
Sidekiq::Cron::Job.create(name: "EventEngine cleanup", cron: "0 3 * * *",
|
|
393
|
+
class: "EventEngine::OutboxCleanupJob")
|
|
394
|
+
```
|
|
395
|
+
|
|
396
|
+
---
|
|
397
|
+
|
|
398
|
+
## Instrumentation
|
|
399
|
+
|
|
400
|
+
Delivery emits `ActiveSupport::Notifications` you can subscribe to for APM/logging:
|
|
401
|
+
|
|
402
|
+
| Notification | When | Key payload |
|
|
403
|
+
|---|---|---|
|
|
404
|
+
| `event_engine.event_emitted` | written to outbox | `event_name`, `event_version`, `event_id`, `idempotency_key`, `aggregate_*` |
|
|
405
|
+
| `event_engine.event_published` | sent to transport | `event_name`, `event_version`, `event_id` |
|
|
406
|
+
| `event_engine.event_dead_lettered` | attempts exhausted | `+ attempts`, `error_message`, `error_class` |
|
|
407
|
+
| `event_engine.publish_batch` | a publish batch finished | `count` |
|
|
408
|
+
|
|
409
|
+
```ruby
|
|
410
|
+
ActiveSupport::Notifications.subscribe("event_engine.event_dead_lettered") do |*, payload|
|
|
411
|
+
Alerting.notify("Dead-lettered #{payload[:event_name]} after #{payload[:attempts]} attempts")
|
|
412
|
+
end
|
|
413
|
+
```
|
|
414
|
+
|
|
415
|
+
---
|
|
416
|
+
|
|
417
|
+
## Dashboard
|
|
418
|
+
|
|
419
|
+
A small observability UI for the outbox.
|
|
420
|
+
|
|
421
|
+
```ruby
|
|
422
|
+
# config/initializers/event_engine_delivery.rb
|
|
423
|
+
EventEngine::Delivery.configure do |config|
|
|
424
|
+
config.dashboard_auth = ->(controller) { controller.current_user&.admin? }
|
|
425
|
+
end
|
|
426
|
+
```
|
|
427
|
+
|
|
428
|
+
```ruby
|
|
429
|
+
# config/routes.rb
|
|
430
|
+
mount EventEngine::Delivery::Engine => "/event_engine", as: :event_engine
|
|
431
|
+
```
|
|
432
|
+
|
|
433
|
+
Then visit `/event_engine/dashboard`:
|
|
434
|
+
|
|
435
|
+
- **Overview** — totals (all / published / unpublished / dead-lettered).
|
|
436
|
+
- **Events** — paginated list (20/page) with status, plus a detail view of payload &
|
|
437
|
+
metadata.
|
|
438
|
+
- **Dead letters** — list with single and bulk **retry** buttons.
|
|
439
|
+
|
|
440
|
+
Access is gated by `dashboard_auth`. If it's `nil` or returns false, the dashboard
|
|
441
|
+
returns **403 Forbidden** (and logs a warning when unconfigured). The gem ships
|
|
442
|
+
functional but unstyled HTML — bring your own CSS if you want it pretty.
|
|
443
|
+
|
|
444
|
+
> Mount **`EventEngine::Delivery::Engine`** (this gem), not `EventEngine::Engine`
|
|
445
|
+
> (core). The dashboard controllers live here.
|
|
446
|
+
|
|
447
|
+
---
|
|
448
|
+
|
|
449
|
+
## Cloud reporter
|
|
450
|
+
|
|
451
|
+
Optionally stream lightweight **metadata** (never payloads or business data) to
|
|
452
|
+
EventEngine Cloud for real-time observability.
|
|
453
|
+
|
|
454
|
+
```ruby
|
|
455
|
+
config.cloud_api_key = ENV["EVENT_ENGINE_CLOUD_KEY"]
|
|
456
|
+
```
|
|
457
|
+
|
|
458
|
+
That's it — the reporter starts at boot when a key is present, and is a zero-overhead
|
|
459
|
+
no-op when absent. It hooks the same `ActiveSupport::Notifications` above, batches
|
|
460
|
+
entries (`cloud_batch_size`, default 50), and flushes on a timer
|
|
461
|
+
(`cloud_flush_interval`, default 10s). What's sent: event name, type, version,
|
|
462
|
+
status (emitted/published/dead-lettered), timestamps, attempt counts, and error
|
|
463
|
+
classes for dead letters.
|
|
464
|
+
|
|
465
|
+
**Failure isolation:** all calls are fire-and-forget with a 5s timeout over
|
|
466
|
+
`Net::HTTP`; errors are logged, never raised — the reporter can never affect your
|
|
467
|
+
app.
|
|
468
|
+
|
|
469
|
+
---
|
|
470
|
+
|
|
471
|
+
## Customization recipes
|
|
472
|
+
|
|
473
|
+
**Run delivery and a durable log together.** Add `event_engine-store`. Both register
|
|
474
|
+
handlers at `levels: :all`; every event is delivered *and* recorded. Order is the
|
|
475
|
+
registration order at boot.
|
|
476
|
+
|
|
477
|
+
**Pause delivery during maintenance.** Set `delivery_adapter = :manual`; events keep
|
|
478
|
+
landing in the outbox safely and you drain them with `OutboxPublisher` when ready.
|
|
479
|
+
|
|
480
|
+
**Different transports per destination.** Write one transport whose `publish`
|
|
481
|
+
fans out to several brokers, or branch on `event.event_type` / `event.event_name`
|
|
482
|
+
inside `publish`.
|
|
483
|
+
|
|
484
|
+
**Tune durability per event.** Change `event_level` in the definition (1→2→3→4). No
|
|
485
|
+
producer code changes; re-dump the schema (level isn't fingerprinted, so it won't
|
|
486
|
+
bump the version).
|
|
487
|
+
|
|
488
|
+
---
|
|
489
|
+
|
|
490
|
+
## License
|
|
491
|
+
|
|
492
|
+
Available as open source under the terms of the
|
|
493
|
+
[MIT License](https://opensource.org/licenses/MIT).
|
data/Rakefile
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
//= link_directory ../stylesheets/event_engine .css
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* This is a manifest file that'll be compiled into application.css, which will include all the files
|
|
3
|
+
* listed below.
|
|
4
|
+
*
|
|
5
|
+
* Any CSS and SCSS file within this directory, lib/assets/stylesheets, vendor/assets/stylesheets,
|
|
6
|
+
* or any plugin's vendor/assets/stylesheets directory can be referenced here using a relative path.
|
|
7
|
+
*
|
|
8
|
+
* You're free to add application-wide styles to this file and they'll appear at the bottom of the
|
|
9
|
+
* compiled file so the styles you add here take precedence over styles defined in any other CSS/SCSS
|
|
10
|
+
* files in this directory. Styles in this file should be added after the last require_* statement.
|
|
11
|
+
* It is generally better to create a new file per style scope.
|
|
12
|
+
*
|
|
13
|
+
*= require_tree .
|
|
14
|
+
*= require_self
|
|
15
|
+
*/
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
module EventEngine
|
|
2
|
+
class OutboxCleanupJob < ApplicationJob
|
|
3
|
+
queue_as :default
|
|
4
|
+
|
|
5
|
+
def perform
|
|
6
|
+
retention_period = EventEngine::Delivery.configuration.retention_period
|
|
7
|
+
return unless retention_period
|
|
8
|
+
|
|
9
|
+
cutoff = Time.current - retention_period
|
|
10
|
+
OutboxEvent.cleanable.published_before(cutoff).delete_all
|
|
11
|
+
end
|
|
12
|
+
end
|
|
13
|
+
end
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
module EventEngine
|
|
2
|
+
class PublishOutboxEventsJob < ApplicationJob
|
|
3
|
+
queue_as :default
|
|
4
|
+
|
|
5
|
+
def perform
|
|
6
|
+
config = EventEngine::Delivery.configuration
|
|
7
|
+
raise "EventEngine transport not configured" unless config&.transport
|
|
8
|
+
|
|
9
|
+
OutboxPublisher.new(
|
|
10
|
+
router: OutboxRouter.new(transport: config.transport),
|
|
11
|
+
batch_size: config.batch_size,
|
|
12
|
+
max_attempts: config.max_attempts
|
|
13
|
+
).call
|
|
14
|
+
end
|
|
15
|
+
end
|
|
16
|
+
end
|