@prosody-events/prosody 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +481 -284
- package/bindings.d.ts +481 -5
- package/bindings.js +81 -66
- package/index.d.ts +494 -7
- package/index.js +915 -27
- package/package.json +19 -16
package/README.md
CHANGED
|
@@ -21,51 +21,58 @@ strategies, and integrated OpenTelemetry support for distributed tracing.
|
|
|
21
21
|
npm install @prosody-events/prosody
|
|
22
22
|
```
|
|
23
23
|
|
|
24
|
+
The package ships TypeScript declarations for the public API.
|
|
25
|
+
`EventHandler<P>` carries an application payload type into `Message<P>`, and
|
|
26
|
+
keyed-state definitions carry their item types through `context.state()`.
|
|
27
|
+
Unparameterized handlers, messages, definitions, and state handles default to
|
|
28
|
+
`JsonValue`. See the [strict TypeScript examples](examples/) for IDE-ready
|
|
29
|
+
projects compiled by the repository typecheck.
|
|
30
|
+
|
|
24
31
|
## Quick Start
|
|
25
32
|
|
|
26
33
|
```javascript
|
|
27
|
-
const {ProsodyClient} = require(
|
|
34
|
+
const { ProsodyClient } = require("@prosody-events/prosody");
|
|
28
35
|
|
|
29
36
|
// Initialize the client with Kafka bootstrap servers, consumer group, and topics
|
|
30
37
|
const client = new ProsodyClient({
|
|
31
|
-
|
|
32
|
-
|
|
38
|
+
// Bootstrap servers should normally be set using the PROSODY_BOOTSTRAP_SERVERS environment variable
|
|
39
|
+
bootstrapServers: "localhost:9092",
|
|
33
40
|
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
41
|
+
// To allow loopbacks, sourceSystem must be different from groupId.
|
|
42
|
+
// Normally, sourceSystem is omitted and defaults to groupId.
|
|
43
|
+
sourceSystem: "my-application-source",
|
|
37
44
|
|
|
38
|
-
|
|
39
|
-
|
|
45
|
+
// groupId should be set to the name of your application
|
|
46
|
+
groupId: "my-consumer-group",
|
|
40
47
|
|
|
41
|
-
|
|
42
|
-
|
|
48
|
+
// Topics the client should subscribe to
|
|
49
|
+
subscribedTopics: "my-topic",
|
|
43
50
|
});
|
|
44
51
|
|
|
45
52
|
// Define a message handler
|
|
46
53
|
const messageHandler = {
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
}
|
|
56
|
-
},
|
|
57
|
-
|
|
58
|
-
onTimer: async (context, timer, signal) => {
|
|
59
|
-
// Handle timer firing
|
|
60
|
-
console.log(`Timer fired for key: ${timer.key} at ${timer.time}`);
|
|
54
|
+
onMessage: async (context, message, signal) => {
|
|
55
|
+
// Process the received message
|
|
56
|
+
console.log(`Received message: ${JSON.stringify(message)}`);
|
|
57
|
+
|
|
58
|
+
// Schedule a timer for delayed processing
|
|
59
|
+
if (message.payload.scheduleFollowup) {
|
|
60
|
+
const followupTime = new Date(Date.now() + 30000); // 30 seconds from now
|
|
61
|
+
await context.schedule(followupTime);
|
|
61
62
|
}
|
|
63
|
+
},
|
|
64
|
+
|
|
65
|
+
onTimer: async (context, timer, signal) => {
|
|
66
|
+
// Handle timer firing
|
|
67
|
+
console.log(`Timer fired for key: ${timer.key} at ${timer.time}`);
|
|
68
|
+
},
|
|
62
69
|
};
|
|
63
70
|
|
|
64
71
|
// Subscribe to messages using the message handler
|
|
65
72
|
client.subscribe(messageHandler);
|
|
66
73
|
|
|
67
74
|
// Send a message to a topic
|
|
68
|
-
await client.send("my-topic", "message-key", {content: "Hello, Kafka!"});
|
|
75
|
+
await client.send("my-topic", "message-key", { content: "Hello, Kafka!" });
|
|
69
76
|
|
|
70
77
|
// Ensure proper shutdown when done
|
|
71
78
|
await client.unsubscribe();
|
|
@@ -110,12 +117,12 @@ When the timer fires, reload the message from Kafka and retry.
|
|
|
110
117
|
```javascript
|
|
111
118
|
// Configure defer behavior
|
|
112
119
|
const client = new ProsodyClient({
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
120
|
+
groupId: "my-consumer-group",
|
|
121
|
+
subscribedTopics: "my-topic",
|
|
122
|
+
deferEnabled: true, // Enable deferral (default: true)
|
|
123
|
+
deferBaseMs: 1000, // Wait 1s before first retry
|
|
124
|
+
deferMaxDelayMs: 86400000, // Cap at 24 hours
|
|
125
|
+
deferFailureThreshold: 0.9, // Disable when >90% failing
|
|
119
126
|
});
|
|
120
127
|
```
|
|
121
128
|
|
|
@@ -132,11 +139,11 @@ with a transient error, routing them through defer.
|
|
|
132
139
|
```javascript
|
|
133
140
|
// Configure monopolization detection
|
|
134
141
|
const client = new ProsodyClient({
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
142
|
+
groupId: "my-consumer-group",
|
|
143
|
+
subscribedTopics: "my-topic",
|
|
144
|
+
monopolizationEnabled: true, // Enable detection (default: true)
|
|
145
|
+
monopolizationThreshold: 0.9, // Reject keys using >90% of window
|
|
146
|
+
monopolizationWindowMs: 300000, // 5-minute window
|
|
140
147
|
});
|
|
141
148
|
```
|
|
142
149
|
|
|
@@ -146,10 +153,10 @@ Handlers are automatically cancelled if they exceed a deadline:
|
|
|
146
153
|
|
|
147
154
|
```javascript
|
|
148
155
|
const client = new ProsodyClient({
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
156
|
+
groupId: "my-consumer-group",
|
|
157
|
+
subscribedTopics: "my-topic",
|
|
158
|
+
timeoutMs: 30000, // Cancel after 30 seconds
|
|
159
|
+
stallThresholdMs: 60000, // Report unhealthy after 60 seconds
|
|
153
160
|
});
|
|
154
161
|
```
|
|
155
162
|
|
|
@@ -162,105 +169,128 @@ Configure via constructor options or environment variables. Options fall back to
|
|
|
162
169
|
|
|
163
170
|
### Core
|
|
164
171
|
|
|
165
|
-
| Option / Environment Variable
|
|
166
|
-
|
|
167
|
-
| `bootstrapServers` / `PROSODY_BOOTSTRAP_SERVERS` | Kafka servers to connect to
|
|
168
|
-
| `groupId` / `PROSODY_GROUP_ID`
|
|
169
|
-
| `subscribedTopics` / `PROSODY_SUBSCRIBED_TOPICS` | Topics to read from
|
|
170
|
-
| `allowedEvents` / `PROSODY_ALLOWED_EVENTS`
|
|
171
|
-
| `sourceSystem` / `PROSODY_SOURCE_SYSTEM`
|
|
172
|
-
| `mock` / `PROSODY_MOCK`
|
|
172
|
+
| Option / Environment Variable | Description | Default |
|
|
173
|
+
| ------------------------------------------------ | ------------------------------------------------- | ----------- |
|
|
174
|
+
| `bootstrapServers` / `PROSODY_BOOTSTRAP_SERVERS` | Kafka servers to connect to | - |
|
|
175
|
+
| `groupId` / `PROSODY_GROUP_ID` | Consumer group name | - |
|
|
176
|
+
| `subscribedTopics` / `PROSODY_SUBSCRIBED_TOPICS` | Topics to read from | - |
|
|
177
|
+
| `allowedEvents` / `PROSODY_ALLOWED_EVENTS` | Only process events matching these prefixes | (all) |
|
|
178
|
+
| `sourceSystem` / `PROSODY_SOURCE_SYSTEM` | Tag for outgoing messages (prevents reprocessing) | `<groupId>` |
|
|
179
|
+
| `mock` / `PROSODY_MOCK` | Use in-memory Kafka for testing | false |
|
|
173
180
|
|
|
174
181
|
### Consumer
|
|
175
182
|
|
|
176
|
-
| Option / Environment Variable
|
|
177
|
-
|
|
178
|
-
| `maxConcurrency` / `PROSODY_MAX_CONCURRENCY`
|
|
179
|
-
| `maxUncommitted` / `PROSODY_MAX_UNCOMMITTED`
|
|
180
|
-
| `timeoutMs` / `PROSODY_TIMEOUT`
|
|
181
|
-
| `commitIntervalMs` / `PROSODY_COMMIT_INTERVAL`
|
|
182
|
-
| `pollIntervalMs` / `PROSODY_POLL_INTERVAL`
|
|
183
|
-
| `shutdownTimeoutMs` / `PROSODY_SHUTDOWN_TIMEOUT`
|
|
184
|
-
| `stallThresholdMs` / `PROSODY_STALL_THRESHOLD`
|
|
185
|
-
| `probePort` / `PROSODY_PROBE_PORT`
|
|
186
|
-
| `failureTopic` / `PROSODY_FAILURE_TOPIC`
|
|
187
|
-
| `idempotenceCacheSize` / `PROSODY_IDEMPOTENCE_CACHE_SIZE` | Global shared cache capacity across all partitions for deduplicating messages.
|
|
188
|
-
| `idempotenceVersion` / `PROSODY_IDEMPOTENCE_VERSION`
|
|
189
|
-
| `idempotenceTtlS` / `PROSODY_IDEMPOTENCE_TTL`
|
|
190
|
-
| `slabSizeMs` / `PROSODY_SLAB_SIZE`
|
|
191
|
-
| `messageSpans` / `PROSODY_MESSAGE_SPANS`
|
|
192
|
-
| `timerSpans` / `PROSODY_TIMER_SPANS`
|
|
183
|
+
| Option / Environment Variable | Description | Default |
|
|
184
|
+
| --------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------- | ---------------------- |
|
|
185
|
+
| `maxConcurrency` / `PROSODY_MAX_CONCURRENCY` | Max messages being processed simultaneously | 32 |
|
|
186
|
+
| `maxUncommitted` / `PROSODY_MAX_UNCOMMITTED` | Max queued messages before pausing consumption | 64 |
|
|
187
|
+
| `timeoutMs` / `PROSODY_TIMEOUT` | Cancel handler if it runs longer than this | 80% of stall threshold |
|
|
188
|
+
| `commitIntervalMs` / `PROSODY_COMMIT_INTERVAL` | How often to save progress to Kafka | 1s |
|
|
189
|
+
| `pollIntervalMs` / `PROSODY_POLL_INTERVAL` | How often to fetch new messages from Kafka | 100ms |
|
|
190
|
+
| `shutdownTimeoutMs` / `PROSODY_SHUTDOWN_TIMEOUT` | Wait this long for in-flight work before force-quit | 30s |
|
|
191
|
+
| `stallThresholdMs` / `PROSODY_STALL_THRESHOLD` | Report unhealthy if no progress for this long | 5m |
|
|
192
|
+
| `probePort` / `PROSODY_PROBE_PORT` | HTTP port for health checks (null to disable) | 8000 |
|
|
193
|
+
| `failureTopic` / `PROSODY_FAILURE_TOPIC` | Send unprocessable messages here (dead letter queue) | - |
|
|
194
|
+
| `idempotenceCacheSize` / `PROSODY_IDEMPOTENCE_CACHE_SIZE` | Global shared cache capacity across all partitions for deduplicating messages. Must be greater than 0 (a value of 0 is rejected). | 8192 |
|
|
195
|
+
| `idempotenceVersion` / `PROSODY_IDEMPOTENCE_VERSION` | Version string for cache-busting dedup hashes | `"1"` |
|
|
196
|
+
| `idempotenceTtlS` / `PROSODY_IDEMPOTENCE_TTL` | TTL for dedup records in Cassandra in seconds | 604800 |
|
|
197
|
+
| `slabSizeMs` / `PROSODY_SLAB_SIZE` | Timer storage granularity (rarely needs changing) | 1h |
|
|
198
|
+
| `messageSpans` / `PROSODY_MESSAGE_SPANS` | Span linking for message execution: `child` (child-of) or `follows_from` | `child` |
|
|
199
|
+
| `timerSpans` / `PROSODY_TIMER_SPANS` | Span linking for timer execution: `child` (child-of) or `follows_from` | `follows_from` |
|
|
193
200
|
|
|
194
201
|
### Producer
|
|
195
202
|
|
|
196
|
-
| Option / Environment Variable
|
|
197
|
-
|
|
203
|
+
| Option / Environment Variable | Description | Default |
|
|
204
|
+
| ---------------------------------------- | ------------------------------- | ------- |
|
|
198
205
|
| `sendTimeoutMs` / `PROSODY_SEND_TIMEOUT` | Give up sending after this long | 1s |
|
|
199
206
|
|
|
200
207
|
### Retry
|
|
201
208
|
|
|
202
209
|
When a handler fails, retry with exponential backoff:
|
|
203
210
|
|
|
204
|
-
| Option / Environment Variable
|
|
205
|
-
|
|
206
|
-
| `maxRetries` / `PROSODY_MAX_RETRIES`
|
|
207
|
-
| `retryBaseMs` / `PROSODY_RETRY_BASE`
|
|
208
|
-
| `maxRetryDelayMs` / `PROSODY_RETRY_MAX_DELAY` | Never wait longer than this
|
|
211
|
+
| Option / Environment Variable | Description | Default |
|
|
212
|
+
| --------------------------------------------- | --------------------------------- | ------- |
|
|
213
|
+
| `maxRetries` / `PROSODY_MAX_RETRIES` | Give up after this many attempts | 3 |
|
|
214
|
+
| `retryBaseMs` / `PROSODY_RETRY_BASE` | Wait this long before first retry | 20ms |
|
|
215
|
+
| `maxRetryDelayMs` / `PROSODY_RETRY_MAX_DELAY` | Never wait longer than this | 5m |
|
|
209
216
|
|
|
210
217
|
### Deferral (Pipeline Mode)
|
|
211
218
|
|
|
212
|
-
| Option / Environment Variable
|
|
213
|
-
|
|
214
|
-
| `deferEnabled` / `PROSODY_DEFER_ENABLED`
|
|
215
|
-
| `deferBaseMs` / `PROSODY_DEFER_BASE`
|
|
216
|
-
| `deferMaxDelayMs` / `PROSODY_DEFER_MAX_DELAY`
|
|
217
|
-
| `deferFailureThreshold` / `PROSODY_DEFER_FAILURE_THRESHOLD`
|
|
218
|
-
| `deferFailureWindowMs` / `PROSODY_DEFER_FAILURE_WINDOW`
|
|
219
|
-
| `deferCacheSize` / `
|
|
220
|
-
| `deferStoreCacheSize` / `PROSODY_DEFER_STORE_CACHE_SIZE`
|
|
221
|
-
| `deferSeekTimeoutMs` / `
|
|
222
|
-
| `deferDiscardThreshold` / `
|
|
219
|
+
| Option / Environment Variable | Description | Default |
|
|
220
|
+
| ------------------------------------------------------------ | -------------------------------------------------------------- | ------- |
|
|
221
|
+
| `deferEnabled` / `PROSODY_DEFER_ENABLED` | Enable deferral for new messages | true |
|
|
222
|
+
| `deferBaseMs` / `PROSODY_DEFER_BASE` | Wait this long before first deferred retry | 1s |
|
|
223
|
+
| `deferMaxDelayMs` / `PROSODY_DEFER_MAX_DELAY` | Never wait longer than this | 24h |
|
|
224
|
+
| `deferFailureThreshold` / `PROSODY_DEFER_FAILURE_THRESHOLD` | Disable deferral when failure rate exceeds this | 0.9 |
|
|
225
|
+
| `deferFailureWindowMs` / `PROSODY_DEFER_FAILURE_WINDOW` | Measure failure rate over this time window | 5m |
|
|
226
|
+
| `deferCacheSize` / `PROSODY_LOADER_CACHE_SIZE` | Track this many deferred keys in memory | 1024 |
|
|
227
|
+
| `deferStoreCacheSize` / `PROSODY_DEFER_STORE_CACHE_SIZE` | Maximum deferred store cache entries per Cassandra defer store | 8192 |
|
|
228
|
+
| `deferSeekTimeoutMs` / `PROSODY_LOADER_SEEK_TIMEOUT` | Timeout when loading deferred messages | 30s |
|
|
229
|
+
| `deferDiscardThreshold` / `PROSODY_LOADER_DISCARD_THRESHOLD` | Read optimization (rarely needs changing) | 100 |
|
|
223
230
|
|
|
224
231
|
### Monopolization Detection (Pipeline Mode)
|
|
225
232
|
|
|
226
|
-
| Option / Environment Variable
|
|
227
|
-
|
|
228
|
-
| `monopolizationEnabled` / `PROSODY_MONOPOLIZATION_ENABLED`
|
|
229
|
-
| `monopolizationThreshold` / `PROSODY_MONOPOLIZATION_THRESHOLD`
|
|
230
|
-
| `monopolizationWindowMs` / `PROSODY_MONOPOLIZATION_WINDOW`
|
|
231
|
-
| `monopolizationCacheSize` / `PROSODY_MONOPOLIZATION_CACHE_SIZE` | Max distinct keys to track
|
|
233
|
+
| Option / Environment Variable | Description | Default |
|
|
234
|
+
| --------------------------------------------------------------- | -------------------------------------- | ------- |
|
|
235
|
+
| `monopolizationEnabled` / `PROSODY_MONOPOLIZATION_ENABLED` | Enable hot key protection | true |
|
|
236
|
+
| `monopolizationThreshold` / `PROSODY_MONOPOLIZATION_THRESHOLD` | Max handler time as fraction of window | 0.9 |
|
|
237
|
+
| `monopolizationWindowMs` / `PROSODY_MONOPOLIZATION_WINDOW` | Measurement window | 5m |
|
|
238
|
+
| `monopolizationCacheSize` / `PROSODY_MONOPOLIZATION_CACHE_SIZE` | Max distinct keys to track | 8192 |
|
|
232
239
|
|
|
233
240
|
### Fair Scheduling (All Modes)
|
|
234
241
|
|
|
235
|
-
| Option / Environment Variable
|
|
236
|
-
|
|
237
|
-
| `schedulerFailureWeight` / `PROSODY_SCHEDULER_FAILURE_WEIGHT` | Fraction of processing time reserved for retries
|
|
238
|
-
| `schedulerMaxWaitMs` / `PROSODY_SCHEDULER_MAX_WAIT`
|
|
239
|
-
| `schedulerWaitWeight` / `PROSODY_SCHEDULER_WAIT_WEIGHT`
|
|
240
|
-
| `schedulerCacheSize` / `PROSODY_SCHEDULER_CACHE_SIZE`
|
|
242
|
+
| Option / Environment Variable | Description | Default |
|
|
243
|
+
| ------------------------------------------------------------- | -------------------------------------------------------------- | ------- |
|
|
244
|
+
| `schedulerFailureWeight` / `PROSODY_SCHEDULER_FAILURE_WEIGHT` | Fraction of processing time reserved for retries | 0.3 |
|
|
245
|
+
| `schedulerMaxWaitMs` / `PROSODY_SCHEDULER_MAX_WAIT` | Messages waiting this long get maximum priority | 2m |
|
|
246
|
+
| `schedulerWaitWeight` / `PROSODY_SCHEDULER_WAIT_WEIGHT` | Priority boost for waiting messages (higher = more aggressive) | 200.0 |
|
|
247
|
+
| `schedulerCacheSize` / `PROSODY_SCHEDULER_CACHE_SIZE` | Max distinct keys to track | 8192 |
|
|
241
248
|
|
|
242
249
|
### Telemetry Emitter
|
|
243
250
|
|
|
244
251
|
Produces message and timer lifecycle events to a Kafka topic for observability:
|
|
245
252
|
|
|
246
|
-
| Option / Environment Variable
|
|
247
|
-
|
|
248
|
-
| `telemetryEnabled` / `PROSODY_TELEMETRY_ENABLED` | Produce lifecycle events to Kafka
|
|
249
|
-
| `telemetryTopic` / `PROSODY_TELEMETRY_TOPIC`
|
|
253
|
+
| Option / Environment Variable | Description | Default |
|
|
254
|
+
| ------------------------------------------------ | --------------------------------- | ------------------------ |
|
|
255
|
+
| `telemetryEnabled` / `PROSODY_TELEMETRY_ENABLED` | Produce lifecycle events to Kafka | true |
|
|
256
|
+
| `telemetryTopic` / `PROSODY_TELEMETRY_TOPIC` | Kafka topic for telemetry events | prosody.telemetry-events |
|
|
250
257
|
|
|
251
258
|
### Cassandra
|
|
252
259
|
|
|
253
260
|
Persistent storage for timers and deferred retries (not needed if `mock: true`):
|
|
254
261
|
|
|
255
|
-
| Option / Environment Variable
|
|
256
|
-
|
|
257
|
-
| `cassandraNodes` / `PROSODY_CASSANDRA_NODES`
|
|
258
|
-
| `cassandraKeyspace` / `PROSODY_CASSANDRA_KEYSPACE`
|
|
259
|
-
| `cassandraUser` / `PROSODY_CASSANDRA_USER`
|
|
260
|
-
| `cassandraPassword` / `PROSODY_CASSANDRA_PASSWORD`
|
|
261
|
-
| `cassandraDatacenter` / `PROSODY_CASSANDRA_DATACENTER`
|
|
262
|
-
| `cassandraRack` / `PROSODY_CASSANDRA_RACK`
|
|
263
|
-
| `cassandraRetentionSeconds` / `PROSODY_CASSANDRA_RETENTION` | Delete data older than this
|
|
262
|
+
| Option / Environment Variable | Description | Default |
|
|
263
|
+
| ----------------------------------------------------------- | ---------------------------------- | ------- |
|
|
264
|
+
| `cassandraNodes` / `PROSODY_CASSANDRA_NODES` | Servers to connect to (host:port) | - |
|
|
265
|
+
| `cassandraKeyspace` / `PROSODY_CASSANDRA_KEYSPACE` | Keyspace name | prosody |
|
|
266
|
+
| `cassandraUser` / `PROSODY_CASSANDRA_USER` | Username | - |
|
|
267
|
+
| `cassandraPassword` / `PROSODY_CASSANDRA_PASSWORD` | Password | - |
|
|
268
|
+
| `cassandraDatacenter` / `PROSODY_CASSANDRA_DATACENTER` | Prefer this datacenter for queries | - |
|
|
269
|
+
| `cassandraRack` / `PROSODY_CASSANDRA_RACK` | Prefer this rack for queries | - |
|
|
270
|
+
| `cassandraRetentionSeconds` / `PROSODY_CASSANDRA_RETENTION` | Delete data older than this | 1y |
|
|
271
|
+
|
|
272
|
+
### Keyed State
|
|
273
|
+
|
|
274
|
+
Register keyed-state collections before you subscribe. Persistence is backed by Cassandra and is not needed when `mock: true`. See the [Keyed State](#keyed-state-1) feature section for handler usage; the client-level knobs and per-collection fields are below. Where an option and an environment variable are paired, an explicitly set option wins; otherwise the environment variable applies, then the default.
|
|
275
|
+
|
|
276
|
+
| Option / Environment Variable | Description | Default |
|
|
277
|
+
| ------------------------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------- |
|
|
278
|
+
| `stateCollections` / - | Keyed-state collections to register before subscribe (array of collection configs; duplicate names are rejected) | (none) |
|
|
279
|
+
| `stateCacheDir` / `PROSODY_STATE_CACHE_DIR` | Disk workspace for the local keyed-state cache; each live client needs its own directory (it is locked exclusively) | per-client temp dir |
|
|
280
|
+
| `stateCacheSizeBytes` / `PROSODY_STATE_CACHE_SIZE_BYTES` | Capacity of the in-memory keyed-state cache, in bytes; must be a positive safe integer. One cache is shared by all partition keyspaces | engine default |
|
|
281
|
+
| `stateRecoveryDelaySeconds` / `PROSODY_STATE_RECOVERY_DELAY` | Delay between staging a provisional cell and the recovery sweep; every collection TTL must strictly exceed this. The option is whole seconds (e.g. `30`); the env var is a duration string (e.g. `30s`), second-granularity, min `1s`. | 30s |
|
|
282
|
+
|
|
283
|
+
Each `stateCollections` entry (a `StateCollectionConfig`) has these fields. Prefer the definition constructors (`value` / `map` / `deque` and their `message*` variants, documented below): they serialize into `stateCollections` so you declare each collection once and reuse the same object with `context.state()`.
|
|
284
|
+
|
|
285
|
+
| Field | Description | Default |
|
|
286
|
+
| ----------------- | ----------------------------------------------------------------------------------- | ---------- |
|
|
287
|
+
| `name` | Collection name; non-empty and unique within the client | (required) |
|
|
288
|
+
| `kind` | `"value"`, `"map"`, or `"deque"` | (required) |
|
|
289
|
+
| `payload` | `"json"` (JSON values) or `"message"` (the full Kafka message the handler received) | (required) |
|
|
290
|
+
| `ttlSeconds` | Per-write TTL in whole seconds (at least 1; must exceed the recovery delay) | (none) |
|
|
291
|
+
| `readUncommitted` | Opt out of transactional staging (read-uncommitted) | false |
|
|
292
|
+
| `keysetLimit` | Map-only; ordered-scan bound in `0..=4096` (`0` disables ordered-scan tracking) | 128 |
|
|
293
|
+
| `capacity` | Deque-only; bounded backlog (a whole number `>= 1`), enforced lazily on push | (none) |
|
|
264
294
|
|
|
265
295
|
## Liveness and Readiness Probes
|
|
266
296
|
|
|
@@ -277,10 +307,10 @@ Configure the probe server using either the client constructor:
|
|
|
277
307
|
|
|
278
308
|
```javascript
|
|
279
309
|
const client = new ProsodyClient({
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
310
|
+
groupId: "my-consumer-group",
|
|
311
|
+
subscribedTopics: "my-topic",
|
|
312
|
+
probePort: 8000, // Set to null to disable
|
|
313
|
+
stallThresholdMs: 15000, // 15 seconds before considering a partition stalled
|
|
284
314
|
});
|
|
285
315
|
```
|
|
286
316
|
|
|
@@ -309,7 +339,7 @@ const partitionCount = client.assignedPartitionCount;
|
|
|
309
339
|
|
|
310
340
|
// You can use these in your own health checks or monitoring
|
|
311
341
|
if (client.isStalled) {
|
|
312
|
-
|
|
342
|
+
console.warn("Consumer has stalled partitions");
|
|
313
343
|
}
|
|
314
344
|
```
|
|
315
345
|
|
|
@@ -320,26 +350,27 @@ if (client.isStalled) {
|
|
|
320
350
|
All messages must be processed. Retries indefinitely. Uses defer and monopolization detection.
|
|
321
351
|
|
|
322
352
|
**Middleware stack:**
|
|
353
|
+
|
|
323
354
|
```
|
|
324
355
|
Kafka → Deduplication → Retry → Defer → Monopolization → Shutdown → Scheduler → Timeout → Telemetry → Handler
|
|
325
356
|
```
|
|
326
357
|
|
|
327
|
-
| Layer | Purpose
|
|
328
|
-
|
|
329
|
-
| Deduplication | Skips messages whose ID was already processed
|
|
330
|
-
| Retry | Retries transient errors indefinitely
|
|
331
|
-
| Defer | Stores failing messages for timer-based retry
|
|
332
|
-
| Monopolization | Rejects keys exceeding execution time threshold
|
|
333
|
-
| Shutdown | Drains in-flight work on partition revocation
|
|
334
|
-
| Scheduler | Enforces concurrency limits and VT-based priority
|
|
335
|
-
| Timeout | Cancels handlers exceeding deadline
|
|
336
|
-
| Telemetry | Emits handler lifecycle events
|
|
358
|
+
| Layer | Purpose |
|
|
359
|
+
| -------------- | ------------------------------------------------- |
|
|
360
|
+
| Deduplication | Skips messages whose ID was already processed |
|
|
361
|
+
| Retry | Retries transient errors indefinitely |
|
|
362
|
+
| Defer | Stores failing messages for timer-based retry |
|
|
363
|
+
| Monopolization | Rejects keys exceeding execution time threshold |
|
|
364
|
+
| Shutdown | Drains in-flight work on partition revocation |
|
|
365
|
+
| Scheduler | Enforces concurrency limits and VT-based priority |
|
|
366
|
+
| Timeout | Cancels handlers exceeding deadline |
|
|
367
|
+
| Telemetry | Emits handler lifecycle events |
|
|
337
368
|
|
|
338
369
|
```javascript
|
|
339
370
|
const client = new ProsodyClient({
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
371
|
+
mode: Mode.Pipeline, // Default mode
|
|
372
|
+
groupId: "my-consumer-group",
|
|
373
|
+
subscribedTopics: "my-topic",
|
|
343
374
|
});
|
|
344
375
|
```
|
|
345
376
|
|
|
@@ -353,11 +384,11 @@ Tries a few times, then routes failures to a dead letter topic.
|
|
|
353
384
|
|
|
354
385
|
```javascript
|
|
355
386
|
const client = new ProsodyClient({
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
387
|
+
mode: Mode.LowLatency,
|
|
388
|
+
groupId: "my-consumer-group",
|
|
389
|
+
subscribedTopics: "my-topic",
|
|
390
|
+
failureTopic: "failed-messages", // Required for low-latency mode
|
|
391
|
+
maxRetries: 3, // Give up after 3 attempts
|
|
361
392
|
});
|
|
362
393
|
```
|
|
363
394
|
|
|
@@ -371,9 +402,9 @@ Logs failures and moves on.
|
|
|
371
402
|
|
|
372
403
|
```javascript
|
|
373
404
|
const client = new ProsodyClient({
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
405
|
+
mode: Mode.BestEffort,
|
|
406
|
+
groupId: "my-consumer-group",
|
|
407
|
+
subscribedTopics: "my-topic",
|
|
377
408
|
});
|
|
378
409
|
```
|
|
379
410
|
|
|
@@ -385,9 +416,9 @@ of events:
|
|
|
385
416
|
```javascript
|
|
386
417
|
// Process only events with types starting with "user." or "account."
|
|
387
418
|
const client = new ProsodyClient({
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
419
|
+
groupId: "my-consumer-group",
|
|
420
|
+
subscribedTopics: "my-topic",
|
|
421
|
+
allowedEvents: ["user.", "account."],
|
|
391
422
|
});
|
|
392
423
|
```
|
|
393
424
|
|
|
@@ -421,9 +452,9 @@ Prosody prevents processing loops in distributed systems by tracking the source
|
|
|
421
452
|
```javascript
|
|
422
453
|
// Consumer and producer in one application
|
|
423
454
|
const client = new ProsodyClient({
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
455
|
+
groupId: "my-service",
|
|
456
|
+
sourceSystem: "my-service-producer", // Must differ from groupId to allow loopbacks; defaults to groupId
|
|
457
|
+
subscribedTopics: "my-topic",
|
|
427
458
|
});
|
|
428
459
|
```
|
|
429
460
|
|
|
@@ -453,50 +484,34 @@ Deduplication uses a two-tier approach:
|
|
|
453
484
|
- **Cassandra-backed persistent store**: Survives restarts and rebalances across instances. TTL controlled by
|
|
454
485
|
`idempotenceTtlS` (default 7 days, i.e. 604800s).
|
|
455
486
|
|
|
456
|
-
|
|
457
|
-
|
|
487
|
+
Deduplication is always active. `idempotenceCacheSize` must be greater than `0`; a value of `0` (via either the option
|
|
488
|
+
or `PROSODY_IDEMPOTENCE_CACHE_SIZE=0`) is rejected when the client is constructed.
|
|
458
489
|
|
|
459
490
|
```javascript
|
|
460
491
|
// Messages with IDs are deduplicated per key
|
|
461
492
|
await client.send("my-topic", "key1", {
|
|
462
|
-
|
|
463
|
-
|
|
493
|
+
id: "msg-123", // Message will be processed
|
|
494
|
+
content: "Hello!",
|
|
464
495
|
});
|
|
465
496
|
|
|
466
497
|
await client.send("my-topic", "key1", {
|
|
467
|
-
|
|
468
|
-
|
|
498
|
+
id: "msg-123", // Message will be skipped (duplicate)
|
|
499
|
+
content: "Hello again!",
|
|
469
500
|
});
|
|
470
501
|
|
|
471
502
|
await client.send("my-topic", "key2", {
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
});
|
|
475
|
-
```
|
|
476
|
-
|
|
477
|
-
The entire deduplication middleware (both in-memory cache and Cassandra persistent store) can be disabled by setting `idempotenceCacheSize: 0`:
|
|
478
|
-
|
|
479
|
-
```javascript
|
|
480
|
-
const client = new ProsodyClient({
|
|
481
|
-
groupId: "my-consumer-group",
|
|
482
|
-
subscribedTopics: "my-topic",
|
|
483
|
-
idempotenceCacheSize: 0 // Disable deduplication entirely
|
|
503
|
+
id: "msg-123", // Message will be processed (different key)
|
|
504
|
+
content: "Hello!",
|
|
484
505
|
});
|
|
485
506
|
```
|
|
486
507
|
|
|
487
|
-
Or via environment variable:
|
|
488
|
-
|
|
489
|
-
```bash
|
|
490
|
-
PROSODY_IDEMPOTENCE_CACHE_SIZE=0
|
|
491
|
-
```
|
|
492
|
-
|
|
493
508
|
To invalidate all previously recorded dedup entries (forcing reprocessing of messages), change the version:
|
|
494
509
|
|
|
495
510
|
```javascript
|
|
496
511
|
const client = new ProsodyClient({
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
512
|
+
groupId: "my-consumer-group",
|
|
513
|
+
subscribedTopics: "my-topic",
|
|
514
|
+
idempotenceVersion: "2", // Changing this invalidates all previously recorded entries
|
|
500
515
|
});
|
|
501
516
|
```
|
|
502
517
|
|
|
@@ -512,27 +527,27 @@ Prosody supports timer-based delayed execution within message handlers. When a t
|
|
|
512
527
|
|
|
513
528
|
```javascript
|
|
514
529
|
const messageHandler = {
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
530
|
+
onMessage: async (context, message, signal) => {
|
|
531
|
+
// Schedule a timer to fire in 30 seconds
|
|
532
|
+
const futureTime = new Date(Date.now() + 30000);
|
|
533
|
+
await context.schedule(futureTime);
|
|
534
|
+
|
|
535
|
+
// Schedule multiple timers
|
|
536
|
+
const oneMinute = new Date(Date.now() + 60000);
|
|
537
|
+
const twoMinutes = new Date(Date.now() + 120000);
|
|
538
|
+
await context.schedule(oneMinute);
|
|
539
|
+
await context.schedule(twoMinutes);
|
|
540
|
+
|
|
541
|
+
// Check what's scheduled
|
|
542
|
+
const scheduled = await context.scheduled();
|
|
543
|
+
console.log(`Scheduled timers: ${scheduled.length}`);
|
|
544
|
+
},
|
|
545
|
+
|
|
546
|
+
onTimer: async (context, timer, signal) => {
|
|
547
|
+
console.log("Timer fired!");
|
|
548
|
+
console.log(`Key: ${timer.key}`);
|
|
549
|
+
console.log(`Scheduled time: ${timer.time}`);
|
|
550
|
+
},
|
|
536
551
|
};
|
|
537
552
|
```
|
|
538
553
|
|
|
@@ -567,10 +582,10 @@ Or programmatically when creating the client:
|
|
|
567
582
|
|
|
568
583
|
```javascript
|
|
569
584
|
const client = new ProsodyClient({
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
585
|
+
bootstrapServers: "localhost:9092",
|
|
586
|
+
groupId: "my-application",
|
|
587
|
+
subscribedTopics: "my-topic",
|
|
588
|
+
cassandraNodes: "localhost:9042", // Required unless mock: true
|
|
574
589
|
});
|
|
575
590
|
```
|
|
576
591
|
|
|
@@ -579,10 +594,10 @@ For testing, you can use mock mode to avoid Cassandra dependency:
|
|
|
579
594
|
```javascript
|
|
580
595
|
// Mock mode for testing (timers work but aren't persisted)
|
|
581
596
|
const client = new ProsodyClient({
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
597
|
+
bootstrapServers: "localhost:9092",
|
|
598
|
+
groupId: "my-application",
|
|
599
|
+
subscribedTopics: "my-topic",
|
|
600
|
+
mock: true, // No Cassandra required in mock mode
|
|
586
601
|
});
|
|
587
602
|
```
|
|
588
603
|
|
|
@@ -597,15 +612,15 @@ If you're using TypeScript or a JavaScript environment that supports decorators,
|
|
|
597
612
|
to classify exceptions that should not be retried:
|
|
598
613
|
|
|
599
614
|
```javascript
|
|
600
|
-
import {permanent, ProsodyClient} from
|
|
615
|
+
import { permanent, ProsodyClient } from "@prosody-events/prosody";
|
|
601
616
|
|
|
602
617
|
class MyHandler {
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
618
|
+
@permanent(TypeError, AttributeError)
|
|
619
|
+
async onMessage(context, message, signal) {
|
|
620
|
+
// Your message handling logic here
|
|
621
|
+
// TypeError and AttributeError will be treated as permanent
|
|
622
|
+
// All other exceptions will be treated as transient (default behavior)
|
|
623
|
+
}
|
|
609
624
|
}
|
|
610
625
|
|
|
611
626
|
const client = new ProsodyClient(config);
|
|
@@ -617,20 +632,20 @@ client.subscribe(new MyHandler());
|
|
|
617
632
|
If you're not using decorators, you can still classify errors as permanent by throwing a `PermanentError`:
|
|
618
633
|
|
|
619
634
|
```javascript
|
|
620
|
-
import {PermanentError, ProsodyClient} from
|
|
635
|
+
import { PermanentError, ProsodyClient } from "@prosody-events/prosody";
|
|
621
636
|
|
|
622
637
|
const messageHandler = {
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
}
|
|
638
|
+
onMessage: async (context, message, signal) => {
|
|
639
|
+
try {
|
|
640
|
+
// Your message handling logic here
|
|
641
|
+
} catch (error) {
|
|
642
|
+
if (error instanceof TypeError || error instanceof AttributeError) {
|
|
643
|
+
throw new PermanentError(error.message);
|
|
644
|
+
}
|
|
645
|
+
// All other exceptions will be treated as transient (default behavior)
|
|
646
|
+
throw error;
|
|
633
647
|
}
|
|
648
|
+
},
|
|
634
649
|
};
|
|
635
650
|
|
|
636
651
|
const client = new ProsodyClient(config);
|
|
@@ -644,6 +659,109 @@ client.subscribe(messageHandler);
|
|
|
644
659
|
- Be cautious with permanent errors as they prevent retries and can result in data loss.
|
|
645
660
|
- Consider system reliability and data consistency when classifying errors.
|
|
646
661
|
|
|
662
|
+
## Keyed State
|
|
663
|
+
|
|
664
|
+
Keyed state gives every Kafka key its own durable working memory. Prosody automatically uses the current message or timer key, so a handler can relate the current event to earlier events for that key. State survives restarts and rebalances. By default, changes become visible only when the event succeeds.
|
|
665
|
+
|
|
666
|
+
Use keyed state for time-aware stream processing: counters, deduplication, rolling aggregates, pending work, and per-key workflows. Keep your relational database as the source of truth for business data and for work that needs joins or ad hoc queries. Reconstructing stream state with repeated database queries can be slow and expensive; keyed state is built for that job.
|
|
667
|
+
|
|
668
|
+
Most collections should have a TTL. Set it comfortably beyond the longest timer or workflow that uses the state; Prosody validates the minimum supported TTL. Omit it only when keeping inactive keys forever is intentional.
|
|
669
|
+
|
|
670
|
+
### A counter for each key
|
|
671
|
+
|
|
672
|
+
Declare each collection once, register it on the client, and ask the event context for the current key's state:
|
|
673
|
+
|
|
674
|
+
```typescript
|
|
675
|
+
const COUNT = value<number>("count", { ttlSeconds: 30 * 24 * 60 * 60 });
|
|
676
|
+
|
|
677
|
+
const client = new ProsodyClient({
|
|
678
|
+
...config,
|
|
679
|
+
stateCollections: [COUNT],
|
|
680
|
+
});
|
|
681
|
+
|
|
682
|
+
client.subscribe({
|
|
683
|
+
async onMessage(context) {
|
|
684
|
+
const count = context.state(COUNT);
|
|
685
|
+
await count.set(((await count.get()) ?? 0) + 1);
|
|
686
|
+
},
|
|
687
|
+
});
|
|
688
|
+
```
|
|
689
|
+
|
|
690
|
+
Here, counters expire after 30 days without an update.
|
|
691
|
+
|
|
692
|
+
### Window activity into one notification
|
|
693
|
+
|
|
694
|
+
This example turns a burst of activity into two useful notifications. It sends the first event immediately, collects later events for five minutes, then sends one summary. Because the user ID is the Kafka key, every user gets an independent window.
|
|
695
|
+
|
|
696
|
+
```typescript
|
|
697
|
+
const WINDOW = value<boolean>("window", { ttlSeconds: 24 * 60 * 60 });
|
|
698
|
+
const PENDING = messageDeque<Activity>("pending", {
|
|
699
|
+
capacity: 100,
|
|
700
|
+
ttlSeconds: 24 * 60 * 60,
|
|
701
|
+
});
|
|
702
|
+
|
|
703
|
+
const handler = {
|
|
704
|
+
async onMessage(context, message) {
|
|
705
|
+
const window = context.state(WINDOW);
|
|
706
|
+
const pending = context.state(PENDING);
|
|
707
|
+
|
|
708
|
+
if (await window.get()) {
|
|
709
|
+
await pending.push(message);
|
|
710
|
+
return;
|
|
711
|
+
}
|
|
712
|
+
|
|
713
|
+
await notify(message.key, [message]);
|
|
714
|
+
await window.set(true);
|
|
715
|
+
await context.clearAndSchedule(new Date(Date.now() + 5 * 60_000));
|
|
716
|
+
},
|
|
717
|
+
|
|
718
|
+
async onTimer(context, timer) {
|
|
719
|
+
const pending = context.state(PENDING);
|
|
720
|
+
const batch: Message<Activity>[] = [];
|
|
721
|
+
for await (const message of pending.values()) batch.push(message);
|
|
722
|
+
|
|
723
|
+
if (batch.length > 0) await notify(timer.key, batch);
|
|
724
|
+
await pending.clear();
|
|
725
|
+
await context.state(WINDOW).clear();
|
|
726
|
+
},
|
|
727
|
+
} satisfies EventHandler<Activity>;
|
|
728
|
+
```
|
|
729
|
+
|
|
730
|
+
See the complete, type-checked example for imports, types, client setup, and `notify`: [`examples/windowing.ts`](examples/windowing.ts).
|
|
731
|
+
|
|
732
|
+
Why this works:
|
|
733
|
+
|
|
734
|
+
- Register both definitions in `stateCollections` before subscribing. Keyed state uses Cassandra unless `mock: true`.
|
|
735
|
+
- Use `clearAndSchedule`, not `schedule`, so a retried event does not add another timer for the same key.
|
|
736
|
+
- `capacity: 100` and the one-day TTL prevent an inactive or unusually busy key from retaining an unlimited backlog. Since this example only pushes, overflow drops the oldest saved message.
|
|
737
|
+
- A `messageDeque` requires the original Kafka messages to remain available for the whole window. Use a plain `deque` of payloads if topic retention or compaction cannot guarantee that.
|
|
738
|
+
- Prosody runs one handler at a time for each key, so a user's message and timer handlers cannot overlap.
|
|
739
|
+
- Sending a notification is outside Prosody's state transaction and may happen again after a retry. Give notifications a stable idempotency key, or send them through an outbox, when duplicates matter.
|
|
740
|
+
|
|
741
|
+
### Collections and handles
|
|
742
|
+
|
|
743
|
+
A definition gives a collection a stable name, kind, and options. Register it once on the client, then pass the same definition to `context.state` to access the current key. Do not reuse a persisted name for a different collection kind or payload type.
|
|
744
|
+
|
|
745
|
+
Create handles inside the handler and do not retain them or their iterators afterward.
|
|
746
|
+
|
|
747
|
+
| Collection | JSON payload | Kafka message | Main operations |
|
|
748
|
+
| ------------------ | ------------ | ----------------- | -------------------------------------------------------------------- |
|
|
749
|
+
| Value | `value<T>` | `messageValue<P>` | `get`, `set`, `clear` |
|
|
750
|
+
| Ordered string map | `map<V>` | `messageMap<P>` | `get`, `getMany`, `has`, `set`, `delete`, `entries`, `keys`, `clear` |
|
|
751
|
+
| Deque | `deque<T>` | `messageDeque<P>` | `push`, `unshift`, `pop`, `shift`, `at`, `length`, `values`, `clear` |
|
|
752
|
+
|
|
753
|
+
All operations are async. Map and deque scans are async iterables; a `for await` loop may stop early safely. Map keys are strings. `null` and `undefined` mean absence and cannot be stored—use `clear()` or `delete()` instead.
|
|
754
|
+
|
|
755
|
+
### When changes become visible
|
|
756
|
+
|
|
757
|
+
Reads inside a handler see its earlier writes. The default behavior is the safest choice for most handlers: Prosody buffers those changes and publishes them together when the event succeeds. If the handler throws, none of its pending changes become visible.
|
|
758
|
+
|
|
759
|
+
Each collection also offers explicit controls for workflows that need different behavior:
|
|
760
|
+
|
|
761
|
+
- `readUncommitted: true` writes that collection's changes after the handler succeeds but before the event is recorded as complete. A crash in between can leave the changes visible even though the event is retried. Use it only for idempotent changes, where processing the same event again produces the same stored result.
|
|
762
|
+
- `commit()` immediately publishes this collection's pending changes. They remain visible even if the handler later throws and the event is retried.
|
|
763
|
+
- `rollback()` discards this collection's pending changes since its last `commit()`. It cannot undo changes that were already committed.
|
|
764
|
+
|
|
647
765
|
## OpenTelemetry Tracing
|
|
648
766
|
|
|
649
767
|
Prosody supports OpenTelemetry tracing, allowing you to monitor and analyze the performance of your Kafka-based
|
|
@@ -666,19 +784,21 @@ npm install @opentelemetry/api @opentelemetry/sdk-node @opentelemetry/exporter-t
|
|
|
666
784
|
To initialize tracing in your application:
|
|
667
785
|
|
|
668
786
|
```javascript
|
|
669
|
-
const opentelemetry = require(
|
|
670
|
-
const {NodeSDK} = require(
|
|
671
|
-
const {
|
|
787
|
+
const opentelemetry = require("@opentelemetry/api");
|
|
788
|
+
const { NodeSDK } = require("@opentelemetry/sdk-node");
|
|
789
|
+
const {
|
|
790
|
+
OTLPTraceExporter,
|
|
791
|
+
} = require("@opentelemetry/exporter-trace-otlp-http");
|
|
672
792
|
|
|
673
793
|
const sdk = new NodeSDK({
|
|
674
|
-
|
|
675
|
-
|
|
794
|
+
traceExporter: new OTLPTraceExporter(),
|
|
795
|
+
serviceName: "my-service-name",
|
|
676
796
|
});
|
|
677
797
|
|
|
678
798
|
sdk.start();
|
|
679
799
|
|
|
680
800
|
// Creates a tracer from the global tracer provider
|
|
681
|
-
const tracer = opentelemetry.trace.getTracer(
|
|
801
|
+
const tracer = opentelemetry.trace.getTracer("my-service-name");
|
|
682
802
|
```
|
|
683
803
|
|
|
684
804
|
### Setting OpenTelemetry Environment Variables
|
|
@@ -700,28 +820,28 @@ After initializing tracing, you can define spans in your application, and they w
|
|
|
700
820
|
Kafka:
|
|
701
821
|
|
|
702
822
|
```javascript
|
|
703
|
-
const {ProsodyClient} = require(
|
|
704
|
-
const opentelemetry = require(
|
|
823
|
+
const { ProsodyClient } = require("@prosody-events/prosody");
|
|
824
|
+
const opentelemetry = require("@opentelemetry/api");
|
|
705
825
|
|
|
706
|
-
const tracer = opentelemetry.trace.getTracer(
|
|
826
|
+
const tracer = opentelemetry.trace.getTracer("my-service-name");
|
|
707
827
|
|
|
708
828
|
const client = new ProsodyClient({
|
|
709
|
-
|
|
710
|
-
|
|
829
|
+
groupId: "my-consumer-group",
|
|
830
|
+
subscribedTopics: "my-topic",
|
|
711
831
|
});
|
|
712
832
|
|
|
713
833
|
const messageHandler = {
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
}
|
|
834
|
+
onMessage: async (context, message, signal) => {
|
|
835
|
+
const span = tracer.startSpan("process-message");
|
|
836
|
+
try {
|
|
837
|
+
// Process the received message
|
|
838
|
+
span.addEvent("message.received", {
|
|
839
|
+
"message.payload": JSON.stringify(message),
|
|
840
|
+
});
|
|
841
|
+
} finally {
|
|
842
|
+
span.end();
|
|
724
843
|
}
|
|
844
|
+
},
|
|
725
845
|
};
|
|
726
846
|
|
|
727
847
|
client.subscribe(messageHandler);
|
|
@@ -794,37 +914,37 @@ This ensures:
|
|
|
794
914
|
Implement shutdown handling in your application:
|
|
795
915
|
|
|
796
916
|
```javascript
|
|
797
|
-
const {ProsodyClient} = require(
|
|
917
|
+
const { ProsodyClient } = require("@prosody-events/prosody");
|
|
798
918
|
|
|
799
919
|
async function main() {
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
920
|
+
const client = new ProsodyClient({
|
|
921
|
+
groupId: "my-consumer-group",
|
|
922
|
+
subscribedTopics: "my-topic",
|
|
923
|
+
});
|
|
924
|
+
|
|
925
|
+
const messageHandler = {
|
|
926
|
+
onMessage: async (context, message, signal) => {
|
|
927
|
+
// Process the message
|
|
928
|
+
},
|
|
929
|
+
};
|
|
810
930
|
|
|
811
|
-
|
|
931
|
+
client.subscribe(messageHandler);
|
|
812
932
|
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
933
|
+
// Create a promise that resolves when shutdown is signaled
|
|
934
|
+
const shutdownPromise = new Promise((resolve) => {
|
|
935
|
+
const shutdown = async (signal) => {
|
|
936
|
+
console.log(`Received ${signal}. Initiating shutdown...`);
|
|
937
|
+
await client.unsubscribe();
|
|
938
|
+
resolve();
|
|
939
|
+
};
|
|
820
940
|
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
941
|
+
process.on("SIGTERM", () => shutdown("SIGTERM"));
|
|
942
|
+
process.on("SIGINT", () => shutdown("SIGINT"));
|
|
943
|
+
process.on("SIGHUP", () => shutdown("SIGHUP"));
|
|
944
|
+
});
|
|
825
945
|
|
|
826
|
-
|
|
827
|
-
|
|
946
|
+
// Wait for shutdown to be signaled
|
|
947
|
+
await shutdownPromise;
|
|
828
948
|
}
|
|
829
949
|
|
|
830
950
|
main().catch(console.error);
|
|
@@ -852,19 +972,23 @@ Example of using AbortSignal in message processing:
|
|
|
852
972
|
|
|
853
973
|
```javascript
|
|
854
974
|
const messageHandler = {
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
975
|
+
onMessage: async (context, message, signal) => {
|
|
976
|
+
// Pass the signal to fetch calls
|
|
977
|
+
const response = await fetch("https://api.example.com", { signal });
|
|
978
|
+
const data = await response.json();
|
|
979
|
+
|
|
980
|
+
// Pass the signal to database operations
|
|
981
|
+
await db.query(
|
|
982
|
+
"INSERT INTO messages (payload) VALUES ($1)",
|
|
983
|
+
[message.payload],
|
|
984
|
+
{ signal },
|
|
985
|
+
);
|
|
986
|
+
|
|
987
|
+
// Process the data...
|
|
988
|
+
|
|
989
|
+
// Send a message, passing the abort signal
|
|
990
|
+
await client.send("topic", "key", { data: "value" }, signal);
|
|
991
|
+
},
|
|
868
992
|
};
|
|
869
993
|
```
|
|
870
994
|
|
|
@@ -885,14 +1009,14 @@ Prosody uses an automated release process managed by GitHub Actions. Here's an o
|
|
|
885
1009
|
1. **Trigger**: The release process is triggered automatically on pushes to the `main` branch.
|
|
886
1010
|
|
|
887
1011
|
2. **Release Please**: The process starts with the "Release Please" action, which:
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
1012
|
+
- Analyzes commit messages since the last release.
|
|
1013
|
+
- Creates or updates a release pull request with changelog updates and version bumps.
|
|
1014
|
+
- When the PR is merged, it creates a GitHub release and a git tag.
|
|
891
1015
|
|
|
892
1016
|
3. **Build Process**: If a new release is created, the following build jobs are triggered:
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
1017
|
+
- Linux builds for x86_64 and aarch64 (glibc).
|
|
1018
|
+
- Windows build for x64.
|
|
1019
|
+
- macOS build for aarch64 (Apple Silicon).
|
|
896
1020
|
|
|
897
1021
|
4. **Testing**: The built binaries are tested on Linux (x86_64 and aarch64) with Node.js 20 and 22.
|
|
898
1022
|
|
|
@@ -925,18 +1049,19 @@ your changes before merging to `main`.
|
|
|
925
1049
|
### ProsodyClient
|
|
926
1050
|
|
|
927
1051
|
- `constructor(config: Configuration)`: Initialize a new ProsodyClient with the given configuration.
|
|
928
|
-
- `send(topic: string, key: string, payload:
|
|
1052
|
+
- `send<P>(topic: string, key: string, payload: P & JsonCompatible<P>, signal?: AbortSignal): Promise<void>`: Send a statically checked JSON-compatible message to a specified
|
|
929
1053
|
topic.
|
|
930
1054
|
- `consumerState: ConsumerState`: Get the current state of the consumer.
|
|
931
1055
|
- `sourceSystem: string`: Get the source system identifier configured for the client.
|
|
932
|
-
- `subscribe(eventHandler: EventHandler): void
|
|
1056
|
+
- `subscribe<P = JsonValue>(eventHandler: EventHandler<P>): Promise<void>`: Subscribe using a handler whose payload type flows into `Message<P>`.
|
|
933
1057
|
- `unsubscribe(): Promise<void>`: Unsubscribe from messages and shut down the consumer.
|
|
934
1058
|
|
|
935
1059
|
### EventHandler
|
|
936
1060
|
|
|
937
1061
|
Interface for handling messages and timers:
|
|
938
1062
|
|
|
939
|
-
- `
|
|
1063
|
+
- `EventHandler<P = JsonValue>` carries the application payload type through to the message callback.
|
|
1064
|
+
- `onMessage?: (context: Context, message: Message<P>, signal: AbortSignal) => Promise<void>`: Handles incoming messages
|
|
940
1065
|
- `onTimer?: (context: Context, timer: Timer, signal: AbortSignal) => Promise<void>`: Handles timer events
|
|
941
1066
|
|
|
942
1067
|
### Message
|
|
@@ -945,10 +1070,17 @@ Represents a Kafka message with the following properties:
|
|
|
945
1070
|
|
|
946
1071
|
- `topic: string`: The name of the topic.
|
|
947
1072
|
- `partition: number`: The partition number.
|
|
948
|
-
- `offset:
|
|
1073
|
+
- `offset: bigint`: The message offset within the partition.
|
|
949
1074
|
- `timestamp: Date`: The timestamp when the message was created or sent.
|
|
950
1075
|
- `key: string`: The message key.
|
|
951
|
-
- `payload:
|
|
1076
|
+
- `payload: P`: The statically typed message payload.
|
|
1077
|
+
|
|
1078
|
+
`Message` takes an optional payload type parameter, `Message<P>`, used by handlers and message-backed state collections to type `payload`. Unparameterized `Message` is `Message<JsonValue>`, preserving useful JSON safety without requiring an application-specific payload type.
|
|
1079
|
+
|
|
1080
|
+
`JsonValue` describes arbitrary JSON data. `JsonCompatible<T>` checks a known
|
|
1081
|
+
application type recursively, so ordinary interfaces work with `send()` while
|
|
1082
|
+
functions, `undefined`, `Date`, symbols, bigints, and invalid nested fields are
|
|
1083
|
+
reported by TypeScript before the message reaches the serializer.
|
|
952
1084
|
|
|
953
1085
|
### Context
|
|
954
1086
|
|
|
@@ -965,6 +1097,10 @@ Timer scheduling methods:
|
|
|
965
1097
|
- `clearScheduled(): Promise<void>`: Removes all scheduled timers
|
|
966
1098
|
- `scheduled(): Promise<Date[]>`: Returns an array of all scheduled timer times
|
|
967
1099
|
|
|
1100
|
+
Keyed-state binding:
|
|
1101
|
+
|
|
1102
|
+
- `state(definition): ValueState<T> | MapState<V> | DequeState<T>`: Binds a registered collection for the current event attempt, returning a typed handle (message definitions vend `*State<Message<P>>`). Throws `PermanentStateError` when the name was never registered, or when the definition's `kind`/`payload` disagrees with the collection's durably-registered schema. See the [Keyed State](#keyed-state-2) API reference below.
|
|
1103
|
+
|
|
968
1104
|
### Timer
|
|
969
1105
|
|
|
970
1106
|
Represents a timer that has fired, provided to the `onTimer` method:
|
|
@@ -972,6 +1108,67 @@ Represents a timer that has fired, provided to the `onTimer` method:
|
|
|
972
1108
|
- `key: string`: The entity key identifying what this timer belongs to
|
|
973
1109
|
- `time: Date`: The time when this timer was scheduled to fire
|
|
974
1110
|
|
|
1111
|
+
### Keyed State
|
|
1112
|
+
|
|
1113
|
+
Definition constructors (each returns a frozen definition object used both in `Configuration.stateCollections` and with `context.state()`):
|
|
1114
|
+
|
|
1115
|
+
- `value<T = JsonValue>(name: string, options?: StateDefinitionOptions): ValueDefinition<T>`
|
|
1116
|
+
- `map<V = JsonValue>(name: string, options?: MapDefinitionOptions): MapDefinition<V>`
|
|
1117
|
+
- `deque<T = JsonValue>(name: string, options?: StateDefinitionOptions): DequeDefinition<T>`
|
|
1118
|
+
- `messageValue<P = JsonValue>(name: string, options?: StateDefinitionOptions): MessageValueDefinition<P>`
|
|
1119
|
+
- `messageMap<P = JsonValue>(name: string, options?: MapDefinitionOptions): MessageMapDefinition<P>`
|
|
1120
|
+
- `messageDeque<P = JsonValue>(name: string, options?: StateDefinitionOptions): MessageDequeDefinition<P>`
|
|
1121
|
+
|
|
1122
|
+
`StateDefinitionOptions`: `{ ttlSeconds?: number; readUncommitted?: boolean }`. `MapDefinitionOptions` extends it with `keysetLimit?: number`.
|
|
1123
|
+
|
|
1124
|
+
`ValueState<T>`:
|
|
1125
|
+
|
|
1126
|
+
- `get(): Promise<T | null>`
|
|
1127
|
+
- `set(value: T): Promise<void>`
|
|
1128
|
+
- `clear(): Promise<void>`
|
|
1129
|
+
- `commit(): Promise<void>`
|
|
1130
|
+
- `rollback(): Promise<void>`
|
|
1131
|
+
|
|
1132
|
+
`MapState<V>` (keys are `string`):
|
|
1133
|
+
|
|
1134
|
+
- `get(key: string): Promise<V | null>`
|
|
1135
|
+
- `getMany(keys: readonly string[]): Promise<(V | null)[]>`
|
|
1136
|
+
- `has(key: string): Promise<boolean>`
|
|
1137
|
+
- `set(key: string, value: V): Promise<void>`
|
|
1138
|
+
- `delete(key: string): Promise<void>`
|
|
1139
|
+
- `clear(): Promise<void>`
|
|
1140
|
+
- `entries(direction?: ScanDirection): AsyncIterableIterator<[string, V]>`
|
|
1141
|
+
- `keys(direction?: ScanDirection): AsyncIterableIterator<string>`
|
|
1142
|
+
- `values(direction?: ScanDirection): AsyncIterableIterator<V>`
|
|
1143
|
+
- `[Symbol.asyncIterator](): AsyncIterableIterator<[string, V]>`
|
|
1144
|
+
- `commit(): Promise<void>`
|
|
1145
|
+
- `rollback(): Promise<void>`
|
|
1146
|
+
|
|
1147
|
+
`DequeState<T>`:
|
|
1148
|
+
|
|
1149
|
+
- `push(item: T): Promise<void>`
|
|
1150
|
+
- `unshift(item: T): Promise<void>`
|
|
1151
|
+
- `pop(): Promise<T | null>`
|
|
1152
|
+
- `shift(): Promise<T | null>`
|
|
1153
|
+
- `length(): Promise<number>`
|
|
1154
|
+
- `isEmpty(): Promise<boolean>`
|
|
1155
|
+
- `clear(): Promise<void>`
|
|
1156
|
+
- `at(index: number): Promise<T | null>`
|
|
1157
|
+
- `values(direction?: ScanDirection): AsyncIterableIterator<T>`
|
|
1158
|
+
- `[Symbol.asyncIterator](): AsyncIterableIterator<T>`
|
|
1159
|
+
- `commit(): Promise<void>`
|
|
1160
|
+
- `rollback(): Promise<void>`
|
|
1161
|
+
|
|
1162
|
+
`ScanDirection`: `"forward" | "backward"`.
|
|
1163
|
+
|
|
1164
|
+
`StateCollectionConfig` (a `stateCollections` entry): `{ name: string; kind: "value" | "map" | "deque"; payload: "json" | "message"; ttlSeconds?: number; readUncommitted?: boolean; keysetLimit?: number }`. The definition constructors produce objects assignable to this shape, so prefer them.
|
|
1165
|
+
|
|
1166
|
+
Errors:
|
|
1167
|
+
|
|
1168
|
+
- `TransientStateError extends TransientError`: the default — a temporary store read/write failure, or any caller mistake (a `null`/unrepresentable write, item-shape mismatch, non-integer deque index, invalid scan direction), rejected transient so it retries rather than discarding the message.
|
|
1169
|
+
- `PermanentStateError extends PermanentError`: reserved for failures a retry cannot resolve in-process (unregistered/identity-mismatched collection, duplicate registration), or one a handler throws explicitly.
|
|
1170
|
+
- `isStateError(error: unknown): error is PermanentStateError | TransientStateError`: type-guard narrowing an error to either state error class.
|
|
1171
|
+
|
|
975
1172
|
## License
|
|
976
1173
|
|
|
977
1174
|
This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.
|