durability 2.1.0 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +300 -116
- package/dist/durability-alarms.d.ts +90 -0
- package/dist/durability-alarms.js +165 -0
- package/dist/durability.d.ts +158 -0
- package/dist/durability.js +260 -0
- package/dist/errors.d.ts +83 -0
- package/dist/errors.js +184 -0
- package/dist/fanout.d.ts +61 -0
- package/dist/fanout.js +528 -0
- package/dist/index.d.ts +10 -435
- package/dist/index.js +7 -1420
- package/dist/kv-storage.d.ts +2 -0
- package/dist/kv-storage.js +421 -0
- package/dist/lifecycle.d.ts +70 -0
- package/dist/lifecycle.js +36 -0
- package/dist/load.d.ts +40 -0
- package/dist/load.js +68 -0
- package/dist/migrations.d.ts +50 -0
- package/dist/migrations.js +379 -0
- package/dist/policy.d.ts +74 -0
- package/dist/policy.js +49 -0
- package/dist/routing-client.d.ts +28 -0
- package/dist/routing-client.js +109 -0
- package/dist/routing.d.ts +33 -0
- package/dist/routing.js +58 -0
- package/dist/scheduler.d.ts +149 -0
- package/dist/scheduler.js +424 -0
- package/dist/serialization.d.ts +2 -0
- package/dist/serialization.js +13 -0
- package/dist/sqlite-storage.d.ts +2 -0
- package/dist/sqlite-storage.js +251 -0
- package/dist/storage.d.ts +134 -0
- package/dist/storage.js +69 -0
- package/dist/stored-message.d.ts +29 -0
- package/dist/stored-message.js +1 -0
- package/package.json +7 -3
package/README.md
CHANGED
|
@@ -1,24 +1,48 @@
|
|
|
1
1
|
# Durability
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
Composable capabilities for application-owned Cloudflare Durable Objects:
|
|
4
4
|
|
|
5
|
-
|
|
5
|
+
- **`Durability`** — effectively-once operations with retries, idempotency, and result lookup.
|
|
6
|
+
- **`DurabilityAlarms`** — named logical alarms with retries and stable idempotency keys.
|
|
7
|
+
- **`DurabilityFanout`** — durable batch acceptance and independent delivery to static consumer or storage targets.
|
|
8
|
+
- **`DurabilityRouting`** — a DO-side load observer and client-side shard selector (`durability/routing`), supporting location hints and caller transforms.
|
|
9
|
+
|
|
10
|
+
Durable capabilities own their storage and migrations. Routing observations are in-memory and advisory. When an object composes durable capabilities, one `DurabilityScheduler` shares its physical alarm and concurrency pool. No queue factory, generated DO class, or special consumer superclass is required.
|
|
11
|
+
|
|
12
|
+
Durability is not an event-sourcing, replay-log, or workflow-orchestration engine. Operation methods return `Promise<void>` after registration commits; they do not wait for the handler result. Failed work is retried from later alarms. Both SQLite-backed and KV-backed Durable Object classes are supported.
|
|
6
13
|
|
|
7
14
|
## Migrating to v3
|
|
8
15
|
|
|
9
|
-
|
|
16
|
+
The factory API is replaced by classes:
|
|
17
|
+
|
|
18
|
+
| v2 | v3 |
|
|
19
|
+
| -------------------------------------------------------- | ------------------------------------------------------------------ |
|
|
20
|
+
| `createDurability(ctx, handlers, options)` | `new Durability({ context: ctx, handlers, ...options })` |
|
|
21
|
+
| `options.alarms` / `options.alarmMethods` | `new DurabilityAlarms({ context: ctx, handlers, methods })` |
|
|
22
|
+
| `durability.alarm.cleanup(time)` | `alarms.cleanup(time)` |
|
|
23
|
+
| `migrateDurability(ctx, target)` | `Durability.migrate` / `DurabilityAlarms.migrate` |
|
|
24
|
+
| `durabilityMigrations` | `durabilityOperationMigrations` / `durabilityNamedAlarmMigrations` |
|
|
25
|
+
| `purgeBefore(ts)` → `{ operations, namedAlarms, total }` | `purgeBefore(ts)` → `number` per helper |
|
|
26
|
+
|
|
27
|
+
Existing SQLite databases are upgraded transparently on first construction: no schema statement runs, the history table gains `capability` and `down` columns, and per-capability rows are recorded next to the combined v2 `durability_0004_generation_and_created_at` row. That row is kept so a rollback to 2.x also sees a complete history. Objects first created on 3.x with only one helper cannot be rolled back to 2.x, which expects both tables. `workers-qb` is no longer a dependency. The `alarmConcurrency` pool is now shared through a `DurabilityScheduler` when both helpers are used; standalone helpers each own a pool.
|
|
28
|
+
|
|
29
|
+
## Durable operations
|
|
10
30
|
|
|
11
31
|
```ts
|
|
12
32
|
import { DurableObject } from 'cloudflare:workers';
|
|
13
|
-
import {
|
|
33
|
+
import { Durability, type DurableCall } from 'durability';
|
|
14
34
|
|
|
15
35
|
type ResizeInput = { imageId: string };
|
|
16
36
|
|
|
17
37
|
export class ImageJobs extends DurableObject<Env> {
|
|
18
|
-
private readonly durability =
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
38
|
+
private readonly durability = new Durability({
|
|
39
|
+
context: this.ctx,
|
|
40
|
+
handlers: {
|
|
41
|
+
resizeImage: async ({ id, payload }: DurableCall<ResizeInput>) =>
|
|
42
|
+
this.env.IMAGES.resize(payload.imageId, { idempotencyKey: id }),
|
|
43
|
+
},
|
|
44
|
+
retries: { maxAttempts: 3 },
|
|
45
|
+
methods: { resizeImage: { attemptTimeoutMs: 60_000 } },
|
|
22
46
|
});
|
|
23
47
|
|
|
24
48
|
resize(imageId: string) {
|
|
@@ -28,79 +52,114 @@ export class ImageJobs extends DurableObject<Env> {
|
|
|
28
52
|
});
|
|
29
53
|
}
|
|
30
54
|
|
|
31
|
-
|
|
32
|
-
return this.durability.
|
|
55
|
+
result(imageId: string) {
|
|
56
|
+
return this.durability.resizeImage.getResult(`resize:${imageId}`);
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
alarm(info?: AlarmInvocationInfo) {
|
|
60
|
+
return this.durability.alarm(info);
|
|
33
61
|
}
|
|
34
62
|
}
|
|
35
63
|
```
|
|
36
64
|
|
|
37
|
-
|
|
65
|
+
Every handler becomes a method on the instance. Handler names that collide with instance members (`alarm`, `purgeBefore`) are rejected at construction.
|
|
38
66
|
|
|
39
|
-
|
|
67
|
+
### Reading results
|
|
40
68
|
|
|
41
69
|
```ts
|
|
42
|
-
|
|
43
|
-
import { createDurability } from 'durability';
|
|
44
|
-
import { exponential, jitter } from 'durability/utils';
|
|
70
|
+
const result = await this.durability.resizeImage.getResult(`resize:${imageId}`);
|
|
45
71
|
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
72
|
+
switch (result.status) {
|
|
73
|
+
case 'not_found':
|
|
74
|
+
break;
|
|
75
|
+
case 'pending':
|
|
76
|
+
console.log(result.attempt, result.nextAttemptAt, result.lastError);
|
|
77
|
+
break;
|
|
78
|
+
case 'failed':
|
|
79
|
+
console.error(result.error.name, result.error.message);
|
|
80
|
+
break;
|
|
81
|
+
case 'completed':
|
|
82
|
+
console.log(result.result);
|
|
83
|
+
break;
|
|
84
|
+
}
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
The completed result type is inferred from the handler. Looking up a key that belongs to another operation throws `DuplicateDurableCallError`. Payloads and results must be JSON-serializable.
|
|
88
|
+
|
|
89
|
+
## Named alarms
|
|
90
|
+
|
|
91
|
+
```ts
|
|
92
|
+
import { DurableObject } from 'cloudflare:workers';
|
|
93
|
+
import { DurabilityAlarms } from 'durability';
|
|
94
|
+
|
|
95
|
+
export class Subscription extends DurableObject<Env> {
|
|
96
|
+
private readonly alarms = new DurabilityAlarms({
|
|
97
|
+
context: this.ctx,
|
|
98
|
+
handlers: {
|
|
99
|
+
renew: async ({ idempotencyKey, signal }) => {
|
|
100
|
+
await this.env.BILLING.fetch('https://billing.internal/renew', {
|
|
101
|
+
method: 'POST',
|
|
102
|
+
headers: { 'Idempotency-Key': idempotencyKey },
|
|
59
103
|
signal,
|
|
60
|
-
|
|
61
|
-
}) => {
|
|
62
|
-
await this.env.CLEANUP.fetch('https://cleanup.internal/run', {
|
|
63
|
-
method: 'POST',
|
|
64
|
-
headers: { 'Idempotency-Key': idempotencyKey },
|
|
65
|
-
signal,
|
|
66
|
-
});
|
|
67
|
-
console.log({ scheduledTime, attempt, platform });
|
|
68
|
-
},
|
|
69
|
-
},
|
|
70
|
-
alarmMethods: {
|
|
71
|
-
cleanup: {
|
|
72
|
-
attemptTimeoutMs: 60_000,
|
|
73
|
-
retries: {
|
|
74
|
-
maxAttempts: 5,
|
|
75
|
-
delay: (attempt) => jitter(exponential(attempt)),
|
|
76
|
-
},
|
|
77
|
-
retryTimeouts: true,
|
|
78
|
-
},
|
|
104
|
+
});
|
|
79
105
|
},
|
|
80
|
-
}
|
|
81
|
-
|
|
106
|
+
},
|
|
107
|
+
methods: { renew: { retryTimeouts: true, retries: { maxAttempts: 5 } } },
|
|
108
|
+
});
|
|
82
109
|
|
|
83
|
-
|
|
84
|
-
return this.
|
|
110
|
+
scheduleRenewal(at: number) {
|
|
111
|
+
return this.alarms.renew(at);
|
|
85
112
|
}
|
|
86
113
|
|
|
87
|
-
alarm(
|
|
88
|
-
return this.
|
|
114
|
+
alarm(info?: AlarmInvocationInfo) {
|
|
115
|
+
return this.alarms.alarm(info);
|
|
89
116
|
}
|
|
90
117
|
}
|
|
91
118
|
```
|
|
92
119
|
|
|
93
|
-
Scheduling
|
|
120
|
+
Scheduling a name again replaces its pending occurrence. If that name is already running, the current handler continues and the replacement runs afterward. Different names may run concurrently, but one name never has more than one active handler in the same instance.
|
|
121
|
+
|
|
122
|
+
Each occurrence receives a new internal generation. Its `idempotencyKey` stays stable across retries while `attempt` increments. A successful handler removes only the occurrence it executed, so it cannot delete a replacement scheduled while it was running.
|
|
123
|
+
|
|
124
|
+
A timeout aborts the handler's `signal` and is terminal by default because the external outcome is unknown. Set `retryTimeouts: true` only when the handler's side effects use the idempotency key or reconcile their outcome before retrying. Named alarms remain at-least-once across eviction or restart.
|
|
125
|
+
|
|
126
|
+
## Using both in one Durable Object
|
|
94
127
|
|
|
95
|
-
|
|
128
|
+
A Durable Object has one physical alarm. When one object uses both helpers, create a `DurabilityScheduler` and pass it to each so they share the alarm, the storage backend, and the concurrency pool:
|
|
129
|
+
|
|
130
|
+
```ts
|
|
131
|
+
import { Durability, DurabilityAlarms, DurabilityScheduler } from 'durability';
|
|
132
|
+
|
|
133
|
+
export class ImageJobs extends DurableObject<Env> {
|
|
134
|
+
private readonly scheduler = new DurabilityScheduler({
|
|
135
|
+
context: this.ctx,
|
|
136
|
+
alarmConcurrency: 10,
|
|
137
|
+
});
|
|
138
|
+
|
|
139
|
+
private readonly durability = new Durability({
|
|
140
|
+
scheduler: this.scheduler,
|
|
141
|
+
handlers: {
|
|
142
|
+
resizeImage: async ({ payload }) =>
|
|
143
|
+
this.env.IMAGES.resize(payload.imageId),
|
|
144
|
+
},
|
|
145
|
+
});
|
|
146
|
+
|
|
147
|
+
private readonly alarms = new DurabilityAlarms({
|
|
148
|
+
scheduler: this.scheduler,
|
|
149
|
+
handlers: { cleanup: async () => this.env.IMAGES.cleanup() },
|
|
150
|
+
});
|
|
96
151
|
|
|
97
|
-
|
|
152
|
+
alarm(info?: AlarmInvocationInfo) {
|
|
153
|
+
return this.scheduler.alarm(info);
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
```
|
|
98
157
|
|
|
99
|
-
|
|
158
|
+
A helper configured with `context` instead of `scheduler` creates a private scheduler and accepts the scheduler options (`storageBackend`, `alarmConcurrency`, `alarmHandoffMs`) in the same object. Two helpers on the same object must share one scheduler; otherwise each would reconcile the physical alarm against only its own records.
|
|
100
159
|
|
|
101
|
-
##
|
|
160
|
+
## Storage backends
|
|
102
161
|
|
|
103
|
-
|
|
162
|
+
SQLite is the default. Configure the Durable Object class with `new_sqlite_classes`; no option is required:
|
|
104
163
|
|
|
105
164
|
```jsonc
|
|
106
165
|
{
|
|
@@ -108,51 +167,52 @@ The Durable Object class must use SQLite storage:
|
|
|
108
167
|
}
|
|
109
168
|
```
|
|
110
169
|
|
|
111
|
-
`
|
|
170
|
+
`Durability` creates and migrates `durability_calls`; `DurabilityAlarms` does the same for `durability_alarms`. Both histories are tracked in the shared `durability_migrations` table and only the constructed helpers' tables exist.
|
|
112
171
|
|
|
113
|
-
|
|
172
|
+
For a class backed only by the KV storage API, configure it with `new_classes` and select the backend:
|
|
114
173
|
|
|
115
|
-
```
|
|
116
|
-
|
|
174
|
+
```jsonc
|
|
175
|
+
{
|
|
176
|
+
"migrations": [{ "tag": "v1", "new_classes": ["ImageJobs"] }],
|
|
177
|
+
}
|
|
178
|
+
```
|
|
117
179
|
|
|
118
|
-
|
|
119
|
-
|
|
180
|
+
```ts
|
|
181
|
+
const durability = new Durability({
|
|
182
|
+
context: this.ctx,
|
|
183
|
+
handlers,
|
|
184
|
+
storageBackend: 'kv',
|
|
185
|
+
});
|
|
120
186
|
```
|
|
121
187
|
|
|
122
|
-
|
|
188
|
+
KV mode stores records and ordered pending/created indexes in the object's private key-value storage, never touches `storage.sql`, and creates no migration records. Records are not converted between backends; deploy a separate class or copy records explicitly before switching.
|
|
123
189
|
|
|
124
|
-
|
|
190
|
+
### Explicit migrations
|
|
125
191
|
|
|
126
|
-
|
|
192
|
+
Rollbacks are explicit because reverting the initial migration deletes records:
|
|
127
193
|
|
|
128
194
|
```ts
|
|
129
|
-
|
|
195
|
+
import { Durability, DurabilityAlarms } from 'durability';
|
|
130
196
|
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
case 'pending':
|
|
135
|
-
console.log(result.attempt, result.nextAttemptAt, result.lastError);
|
|
136
|
-
break;
|
|
137
|
-
case 'failed':
|
|
138
|
-
console.error(result.error.name, result.error.message);
|
|
139
|
-
break;
|
|
140
|
-
case 'completed':
|
|
141
|
-
console.log(result.result);
|
|
142
|
-
break;
|
|
143
|
-
}
|
|
197
|
+
Durability.migrate(this.ctx, 'durability_0001_create_calls');
|
|
198
|
+
Durability.migrate(this.ctx, null); // remove the operations schema entirely
|
|
199
|
+
DurabilityAlarms.migrate(this.ctx, null); // remove the named alarm schema
|
|
144
200
|
```
|
|
145
201
|
|
|
146
|
-
|
|
202
|
+
Passing a name moves that capability's schema to exactly that version. Each batch of migrations applies or reverts in one synchronous transaction, so a failure leaves the schema untouched.
|
|
203
|
+
|
|
204
|
+
Every history row stores its migration's `down` script. If a Durable Object later runs an older `durability` version, that version reverts the migrations it does not recognize using the stored scripts, so downgrades are safe without code changes. To guard against deploying a very old version by mistake, at most two unknown migrations are reverted; beyond that, construction throws. The `durability_migrations` table is dropped once no durability history remains.
|
|
147
205
|
|
|
148
206
|
## Retries, timeouts, and terminal failures
|
|
149
207
|
|
|
150
|
-
Attempts use exponential backoff with equal jitter, stop after five attempts, and time out after five minutes by default. The retry `delay` function fully controls scheduling and can be overridden globally or per
|
|
208
|
+
Attempts use exponential backoff with equal jitter, stop after five attempts, and time out after five minutes by default. The retry `delay` function fully controls scheduling and can be overridden globally or per method on either helper:
|
|
151
209
|
|
|
152
210
|
```ts
|
|
153
211
|
import { exponential, jitter } from 'durability/utils';
|
|
154
212
|
|
|
155
|
-
const durability =
|
|
213
|
+
const durability = new Durability({
|
|
214
|
+
context: this.ctx,
|
|
215
|
+
handlers,
|
|
156
216
|
attemptTimeoutMs: 60_000,
|
|
157
217
|
retries: {
|
|
158
218
|
maxAttempts: 5,
|
|
@@ -170,66 +230,190 @@ const durability = createDurability(this.ctx, handlers, {
|
|
|
170
230
|
});
|
|
171
231
|
```
|
|
172
232
|
|
|
173
|
-
Each attempt receives its own `AbortSignal
|
|
233
|
+
Each attempt receives its own `AbortSignal`. A timed-out operation follows the normal retry policy; a timed-out named alarm is terminal unless `retryTimeouts` is set. If a handler ignores abort, its in-memory lock and concurrency permit remain held until it actually settles, so a same-isolate retry cannot overlap it.
|
|
234
|
+
|
|
235
|
+
Throw `NonRetryableError` to move work directly to `failed` without another attempt. Errors named `NonRetryableError` from other packages, such as `cloudflare:workflows`, are recognized too.
|
|
236
|
+
|
|
237
|
+
## Delivery semantics and concurrency
|
|
238
|
+
|
|
239
|
+
Work is delivered at least once. A process can stop after a side effect succeeds but before its completion record commits, so handlers should be idempotent and pass the call ID or alarm `idempotencyKey` to external services.
|
|
240
|
+
|
|
241
|
+
`alarmConcurrency` defaults to 10 and is one FIFO limit shared by eager operation handlers, alarm-driven operation handlers, and named alarm handlers attached to the same scheduler. Newly registered operations start eagerly only when a permit is immediately available; otherwise the reconciled alarm picks them up.
|
|
242
|
+
|
|
243
|
+
Completion records are retained indefinitely so IDs remain deduplicated. The scheduler owns the Durable Object's alarm; compose unrelated scheduled work as named alarms instead of replacing the physical alarm.
|
|
244
|
+
|
|
245
|
+
## Destructive retention purge
|
|
246
|
+
|
|
247
|
+
`purgeBefore(timestamp)` on either helper deletes that helper's records created strictly before `timestamp`, regardless of status, and returns the count. Records created exactly at the cutoff are retained.
|
|
174
248
|
|
|
175
249
|
```ts
|
|
176
|
-
const
|
|
177
|
-
|
|
178
|
-
fetch(payload.url, { method: 'POST', signal }),
|
|
179
|
-
};
|
|
250
|
+
const removedOperations = await this.durability.purgeBefore(retentionCutoff);
|
|
251
|
+
const removedAlarms = await this.alarms.purgeBefore(retentionCutoff);
|
|
180
252
|
```
|
|
181
253
|
|
|
182
|
-
|
|
254
|
+
SQLite purges in one transaction. KV purges in bounded transactions until no matching record remains, so retry if the invocation is interrupted. The purge best-effort aborts matching active handlers, but cannot force an abort-ignoring handler to stop. Operation IDs can be reused immediately; a late old generation cannot overwrite the replacement record.
|
|
183
255
|
|
|
184
|
-
|
|
256
|
+
## Lifecycle metrics hook
|
|
185
257
|
|
|
186
|
-
|
|
187
|
-
import { NonRetryableError } from 'durability';
|
|
258
|
+
Each helper accepts `onLifecycleEvent`, which receives compact events for registration or scheduling, attempt start and settlement, pre-invocation attempt exhaustion, and purges:
|
|
188
259
|
|
|
189
|
-
|
|
260
|
+
```ts
|
|
261
|
+
const durability = new Durability({
|
|
262
|
+
context: this.ctx,
|
|
263
|
+
handlers,
|
|
264
|
+
onLifecycleEvent: (event) => recordDurabilityMetric(event),
|
|
265
|
+
});
|
|
190
266
|
```
|
|
191
267
|
|
|
192
|
-
|
|
268
|
+
The hook is best-effort and non-blocking, is not a durable journal, and may duplicate or drop events across crashes. Hook failures are reported to `console.error` and never change queue state. Returned promises are attached to `waitUntil` when the context exposes it.
|
|
193
269
|
|
|
194
|
-
##
|
|
270
|
+
## Routing and fanout
|
|
195
271
|
|
|
196
|
-
|
|
272
|
+
A queue is a composition, not another exported abstraction. An application owns its DO class and attaches the capabilities it needs:
|
|
197
273
|
|
|
198
|
-
|
|
274
|
+
```ts
|
|
275
|
+
import { DurableObject } from 'cloudflare:workers';
|
|
276
|
+
import {
|
|
277
|
+
DurabilityFanout,
|
|
278
|
+
DurabilityScheduler,
|
|
279
|
+
type FanoutInput,
|
|
280
|
+
type StoredMessage,
|
|
281
|
+
} from 'durability';
|
|
282
|
+
import { DurabilityRouting } from 'durability/routing';
|
|
283
|
+
|
|
284
|
+
export class Mailbox extends DurableObject<Env> {
|
|
285
|
+
private readonly scheduler = new DurabilityScheduler({ context: this.ctx });
|
|
286
|
+
private readonly routing = new DurabilityRouting();
|
|
287
|
+
private readonly fanout = new DurabilityFanout<Job>({
|
|
288
|
+
scheduler: this.scheduler,
|
|
289
|
+
routing: this.routing,
|
|
290
|
+
targets: {
|
|
291
|
+
consumer: { deliver: (messages) => this.env.CONSUMER.consume(messages) },
|
|
292
|
+
archive: { storage: (message) => this.store('archive/', message) },
|
|
293
|
+
},
|
|
294
|
+
dlq: (message) => this.store('dead-letters/', message),
|
|
295
|
+
});
|
|
199
296
|
|
|
200
|
-
|
|
297
|
+
private async store(prefix: string, message: StoredMessage<Job>) {
|
|
298
|
+
await this.env.ARCHIVE.put(
|
|
299
|
+
`${prefix}${encodeURIComponent(message.id)}`,
|
|
300
|
+
JSON.stringify(message),
|
|
301
|
+
{ onlyIf: { etagDoesNotMatch: '*' } }
|
|
302
|
+
);
|
|
303
|
+
}
|
|
201
304
|
|
|
202
|
-
|
|
305
|
+
enqueue(messages: FanoutInput<Job>[]) {
|
|
306
|
+
return this.fanout.enqueue(messages);
|
|
307
|
+
}
|
|
203
308
|
|
|
204
|
-
|
|
309
|
+
alarm(info?: AlarmInvocationInfo) {
|
|
310
|
+
return this.scheduler.alarm(info);
|
|
311
|
+
}
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
const routing = DurabilityRouting.client({
|
|
315
|
+
target: Mailbox,
|
|
316
|
+
invoke: (stub, messages: FanoutInput<Job>[]) => stub.enqueue(messages),
|
|
317
|
+
sharding: (messages, context: { tenant?: string } | undefined) => ({
|
|
318
|
+
shard: context?.tenant ?? messages[0]?.body.customerId ?? 'default',
|
|
319
|
+
locationHint: 'weur',
|
|
320
|
+
}),
|
|
321
|
+
});
|
|
322
|
+
|
|
323
|
+
const result = await routing.push([{ id: 'job:42', body: job }], {
|
|
324
|
+
tenant: 'acme',
|
|
325
|
+
});
|
|
326
|
+
```
|
|
327
|
+
|
|
328
|
+
The application chooses the RPC method via `invoke`; routing does not know about messages or fanout. A single push selects one shard for the entire input. Static fanout target IDs identify independent delivery obligations; each accepted message snapshots its recipients. Targets added later do not receive old work, and a removed target's work remains durable with an error and backoff until that ID is restored. Target callbacks/storage references are fixed at capability construction.
|
|
329
|
+
|
|
330
|
+
### Acceptance, not completion
|
|
331
|
+
|
|
332
|
+
`fanout.enqueue` accepts a message or batch and returns:
|
|
205
333
|
|
|
206
334
|
```ts
|
|
207
|
-
|
|
208
|
-
|
|
335
|
+
type EnqueueResult<T> =
|
|
336
|
+
| { success: true; value: T; load: LoadSnapshot }
|
|
337
|
+
| {
|
|
338
|
+
success: false;
|
|
339
|
+
error: { name: string; message: string };
|
|
340
|
+
load: LoadSnapshot;
|
|
341
|
+
};
|
|
209
342
|
```
|
|
210
343
|
|
|
211
|
-
|
|
344
|
+
Fanout's value is the message ID array; an application RPC can return its own value with the same envelope. Success means the entire batch and its target obligations were committed, not that consumers ran. Definite validation/conflict rejection returns `success: false` with no partial batch. Storage/commit uncertainty throws, as does an RPC transport failure: the caller must not infer non-acceptance. Routing never silently retries or switches shards on a failed call. The caller decides whether to retry on the same or a different shard.
|
|
212
345
|
|
|
213
|
-
|
|
346
|
+
`id` defaults to a UUID. `deduplicationKey` defaults to `id`; while a registration remains active, repeating that key with identical ID, payload, and recipient set does not redeliver acknowledged siblings. Reusing it for different work rejects the batch. After the final target settles, the message manifest is removed and the key may be reused. This is not permanent or cross-shard deduplication. Consumers must make side effects idempotent for at-least-once delivery.
|
|
347
|
+
|
|
348
|
+
### Load and adaptive routing
|
|
349
|
+
|
|
350
|
+
A snapshot contains `observedAt`, `windowMs: 60000`, and:
|
|
351
|
+
|
|
352
|
+
- `inbound`: instantaneous `inFlight`, rolling `completed`, and `averageProcessingMs` for enqueue calls through acceptance/rejection.
|
|
353
|
+
- `outbound`: the same metrics for target delivery attempts (one batch to one target is one attempt), plus exact durable `pendingDeliveries` counting outstanding message-target pairs.
|
|
354
|
+
|
|
355
|
+
Counts and averages share a rolling window with one-second buckets. No samples means `null`, not zero; telemetry resets on eviction while backlog survives. A DO cannot measure requests waiting in the platform before its handler starts. Snapshots are advisory, not capacity reservations or global metrics.
|
|
356
|
+
|
|
357
|
+
`sharding(input, context, observations)` can inspect recent per-shard snapshots. Caller context can contain arbitrary local values; it is not persisted or sent to consumers. A location hint influences initial shard placement, not existing DO placement.
|
|
358
|
+
|
|
359
|
+
Without custom sharding, routing starts with one shard, samples two candidate shard keys, and compares backlog followed by in-flight processing pressure. It widens under backlog pressure (`softBacklogLimit`, default 1000) and narrows slowly when observations are shallow. Observations expire and are bounded per isolate; this is a best-effort adaptive router, not a globally coordinated autoscaler. Stable-key sharding gives affinity; random routing does not guarantee ordering between separate pushes.
|
|
360
|
+
|
|
361
|
+
### Independent delivery
|
|
362
|
+
|
|
363
|
+
First deliveries are ordered by enqueue sequence per shard and target. Waiting retries do not block later first deliveries; completion order is not guaranteed. Each scheduler pass advances a bounded batch per target (`maxBatchSize`, default 10) with the shared concurrency pool. Different targets progress independently subject to that capacity.
|
|
364
|
+
|
|
365
|
+
Consumer callbacks may forward the batch to any ordinary WorkerEntrypoint RPC method. Messages include `id`, `deliveryId`, `target`, `body`, `attempt`, `enqueuedAt`, and callable settlement capabilities:
|
|
366
|
+
|
|
367
|
+
- `ack()` settles this target only.
|
|
368
|
+
- `retry(10)` schedules eligibility after 10 ms, overriding the policy delay. It is not a real-time delivery guarantee.
|
|
369
|
+
- `deadLetter()` records durable terminal intent before writing to the DLQ. Exhaustion does the same automatically.
|
|
370
|
+
- Returning, throwing, or timing out leaves unsettled messages eligible for retry. A second or late settlement call rejects with `FanoutSettlementError`.
|
|
214
371
|
|
|
215
|
-
|
|
372
|
+
Await settlement calls. Timed-out RPCs cannot be forcibly cancelled, so side effects may overlap later attempts. Storage targets automatically acknowledge successful writes; failed writes retry independently of consumer targets. Their write contract must be idempotent by the supplied delivery identity.
|
|
373
|
+
|
|
374
|
+
### DLQ storage and per-target redrive
|
|
375
|
+
|
|
376
|
+
Fanout only ever **writes** to storage, so a storage target and the `dlq` are both just a write function:
|
|
216
377
|
|
|
217
378
|
```ts
|
|
218
|
-
|
|
219
|
-
onLifecycleEvent: (event) => recordDurabilityMetric(event),
|
|
220
|
-
});
|
|
379
|
+
type MessageWrite<Body> = (message: StoredMessage<Body>) => Promise<void>;
|
|
221
380
|
```
|
|
222
381
|
|
|
223
|
-
|
|
382
|
+
No adapter class or storage interface ships with the library: R2, DO storage, D1, or an HTTP endpoint are all one callback. What separates an archive from a dead letter is the **role in the flow**, not the type — a storage target receives every message on the success path, while `dlq` receives only what a target gave up on. Both may write to the same bucket under different prefixes.
|
|
224
383
|
|
|
225
|
-
|
|
384
|
+
A stored entry carries `id` (unique per registration and target), `messageId`, `target`, `body`, `enqueuedAt`, `storedAt`, and `attempts`; only dead letters also carry `failure: { reason, error }`.
|
|
385
|
+
|
|
386
|
+
Two write invariants are the application's to uphold:
|
|
387
|
+
|
|
388
|
+
- **Key each entry by `id` and refuse to overwrite it.** `id` is stable across delivery attempts, so a retried write collapses onto the same entry (`onlyIf: { etagDoesNotMatch: '*' }` on R2) and a redrive deletes exactly what it re-enqueued. `attempts` and `storedAt` advance between attempts, so entries are not byte-identical.
|
|
389
|
+
- **Delete only after acceptance.** Reading entries back for replay or redrive is application policy, so nothing in the library lists or removes them.
|
|
390
|
+
|
|
391
|
+
Retention is likewise external: filter on `storedAt` when selecting entries to redrive, and configure a **lifecycle deletion rule** on the prefix (for example 30 days on a DLQ prefix, unbounded for an archive) for physical expiry. A failed dead-letter write retains the terminal intent with a 60-second backoff rather than redelivering to the consumer, and omitting `dlq` retains terminal work rather than silently deleting it.
|
|
226
392
|
|
|
227
|
-
|
|
393
|
+
Redrive is explicit application code, not queue sugar. Walk your own storage, select what to replay, then enqueue only the failed target:
|
|
228
394
|
|
|
229
395
|
```ts
|
|
230
|
-
const
|
|
231
|
-
|
|
396
|
+
const result = await routing.push({
|
|
397
|
+
messages: [
|
|
398
|
+
{ id: entry.messageId, body: entry.body, deduplicationKey: entry.id },
|
|
399
|
+
],
|
|
400
|
+
options: { targets: [entry.target] },
|
|
232
401
|
});
|
|
402
|
+
if (result.success) {
|
|
403
|
+
await dlq.remove(entry.id);
|
|
404
|
+
}
|
|
233
405
|
```
|
|
234
406
|
|
|
235
|
-
|
|
407
|
+
Here the application RPC forwards `messages` and `options` to `fanout.enqueue`. A fresh registration resets attempts; using the DLQ entry ID as the key avoids colliding with the original message's still-running siblings. Never remove an entry on rejection or transport uncertainty. A crash between re-enqueue and removal may still duplicate delivery.
|
|
408
|
+
|
|
409
|
+
### Transforms and build-time wiring
|
|
410
|
+
|
|
411
|
+
Routing uses the actual `@durability/transforms` runtime. For example, `routing.with(timeout, 5000).with(tenant, 'acme').push(input)` composes existing caller transforms. `tenant` is created with `defineTransform<RoutingSession<Input, Value, Context>, Context>().caller(...)`; context contributions shallow-merge, with transform fields overriding matching positional fields. An RPC timeout is not cancellation. Retrying a push may select a new shard, so choose stable keys when retry affinity matters.
|
|
412
|
+
|
|
413
|
+
`doTransforms` from `@durability/transforms/vite` maps `DurabilityRouting.client({ target: Mailbox, ... })` to its same-module exported class name and validates it against Wrangler. It injects `exportName`, not a generated DO or a magic application method. Without the plugin, set `exportName: 'Mailbox'` explicitly; `namespace: () => env.MAILBOXES` is an escape hatch for explicit or remote bindings.
|
|
414
|
+
|
|
415
|
+
Loopback namespaces need the application DO declared in Wrangler migrations (or the platform's supported class configuration), but no redundant DO binding block is needed. Consumer transport is explicitly wired by the application, such as a service binding or loopback entrypoint. See the typechecked [email queue composition](../../examples/email-queue).
|
|
416
|
+
|
|
417
|
+
## Long-running alarm calls
|
|
418
|
+
|
|
419
|
+
Alarm invocations have a 15-minute wall-time limit. If work is still pending after `alarmHandoffMs` (default 14 minutes), the scheduler retains its promise in memory, arms an immediate alarm, and returns from the current invocation. The next alarm attaches to the same promise instead of starting the handler again. If the object is evicted during a handoff, the next alarm reconstructs and executes the persisted pending work.
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
import { type LifecycleHook } from './lifecycle.js';
|
|
2
|
+
import { type DurabilityMigrationResult } from './migrations.js';
|
|
3
|
+
import { type DurabilityAlarmMethodOptions, type ExecutionPolicyOptions } from './policy.js';
|
|
4
|
+
import { type SchedulerAttachment } from './scheduler.js';
|
|
5
|
+
/** Information passed to a named alarm handler for one execution attempt. */
|
|
6
|
+
export type DurableAlarmInfo = {
|
|
7
|
+
/** Configured name of the logical alarm. */
|
|
8
|
+
name: string;
|
|
9
|
+
/** Timestamp originally passed to the named alarm method. */
|
|
10
|
+
scheduledTime: number;
|
|
11
|
+
/** One-based execution attempt for this scheduled occurrence. */
|
|
12
|
+
attempt: number;
|
|
13
|
+
/** Whether this scheduled occurrence has run before. */
|
|
14
|
+
isRetry: boolean;
|
|
15
|
+
/** Number of previous attempts for this scheduled occurrence. */
|
|
16
|
+
retryCount: number;
|
|
17
|
+
/** Stable key shared by every attempt of this scheduled occurrence. */
|
|
18
|
+
idempotencyKey: string;
|
|
19
|
+
/** Aborts when the attempt exceeds its configured timeout. */
|
|
20
|
+
signal: AbortSignal;
|
|
21
|
+
/** Invocation information for the shared physical Durable Object alarm. */
|
|
22
|
+
platform: AlarmInvocationInfo | undefined;
|
|
23
|
+
};
|
|
24
|
+
/** A handler for one named logical alarm. */
|
|
25
|
+
export type DurableAlarmHandler = (info: DurableAlarmInfo) => unknown | Promise<unknown>;
|
|
26
|
+
/** Policy configuration for named alarms. */
|
|
27
|
+
export type DurabilityAlarmsOptions<Names extends string = string> = ExecutionPolicyOptions & {
|
|
28
|
+
/** Execution policy overrides keyed by alarm name. */
|
|
29
|
+
methods?: Partial<Record<NoInfer<Names>, DurabilityAlarmMethodOptions>>;
|
|
30
|
+
/** Receives best-effort, non-durable lifecycle metrics events. */
|
|
31
|
+
onLifecycleEvent?: LifecycleHook;
|
|
32
|
+
};
|
|
33
|
+
/**
|
|
34
|
+
* Constructor configuration: the alarm handlers, their policies, and either a
|
|
35
|
+
* shared `scheduler` or the Durable Object `context` (plus scheduler options)
|
|
36
|
+
* to create a private one.
|
|
37
|
+
*/
|
|
38
|
+
export type DurabilityAlarmsConfig<Names extends string = string> = DurabilityAlarmsOptions<Names> & SchedulerAttachment & {
|
|
39
|
+
/** Alarm handlers keyed by the scheduling method names they become. */
|
|
40
|
+
handlers: Record<Names, DurableAlarmHandler>;
|
|
41
|
+
};
|
|
42
|
+
/** Schedules (or replaces) the named alarm's pending occurrence. */
|
|
43
|
+
export type DurableNamedAlarm = (scheduledTime: number) => Promise<void>;
|
|
44
|
+
declare class DurabilityAlarmsCore<Names extends string> {
|
|
45
|
+
#private;
|
|
46
|
+
constructor(config: DurabilityAlarmsConfig<Names>);
|
|
47
|
+
/** Applies or reverts the `durability_alarms` schema; see {@link migrate}. */
|
|
48
|
+
static migrate(context: Pick<DurableObjectState, 'storage'>, target?: string | null): DurabilityMigrationResult;
|
|
49
|
+
/** Runs due named alarms and re-arms the physical alarm. */
|
|
50
|
+
alarm(info?: AlarmInvocationInfo): Promise<void>;
|
|
51
|
+
/**
|
|
52
|
+
* Destructively removes named alarm records created strictly before the
|
|
53
|
+
* timestamp, regardless of status, and returns how many were removed.
|
|
54
|
+
*/
|
|
55
|
+
purgeBefore(before: number): Promise<number>;
|
|
56
|
+
}
|
|
57
|
+
/**
|
|
58
|
+
* Named logical alarms sharing one Durable Object's physical alarm.
|
|
59
|
+
*
|
|
60
|
+
* Every handler becomes a typed scheduling method on the instance. Scheduling
|
|
61
|
+
* a name again replaces its pending occurrence; each occurrence has a stable
|
|
62
|
+
* `idempotencyKey` across retries.
|
|
63
|
+
*
|
|
64
|
+
* @example
|
|
65
|
+
* ```ts
|
|
66
|
+
* class Subscription extends DurableObject<Env> {
|
|
67
|
+
* private readonly alarms = new DurabilityAlarms({
|
|
68
|
+
* context: this.ctx,
|
|
69
|
+
* handlers: {
|
|
70
|
+
* renew: async ({ idempotencyKey, signal }) => renew(idempotencyKey, signal),
|
|
71
|
+
* },
|
|
72
|
+
* });
|
|
73
|
+
*
|
|
74
|
+
* scheduleRenewal(at: number) {
|
|
75
|
+
* return this.alarms.renew(at);
|
|
76
|
+
* }
|
|
77
|
+
*
|
|
78
|
+
* alarm(info?: AlarmInvocationInfo) {
|
|
79
|
+
* return this.alarms.alarm(info);
|
|
80
|
+
* }
|
|
81
|
+
* }
|
|
82
|
+
* ```
|
|
83
|
+
*/
|
|
84
|
+
export type DurabilityAlarms<Names extends string> = DurabilityAlarmsCore<Names> & Record<Names, DurableNamedAlarm>;
|
|
85
|
+
export interface DurabilityAlarmsConstructor {
|
|
86
|
+
new <const Names extends string>(config: DurabilityAlarmsConfig<Names>): DurabilityAlarms<Names>;
|
|
87
|
+
migrate: typeof DurabilityAlarmsCore.migrate;
|
|
88
|
+
}
|
|
89
|
+
export declare const DurabilityAlarms: DurabilityAlarmsConstructor;
|
|
90
|
+
export {};
|