durability 3.0.0 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +403 -27
- package/dist/client.d.ts +94 -0
- package/dist/client.js +445 -0
- package/dist/durability-alarms.d.ts +14 -1
- package/dist/durability-alarms.js +121 -22
- package/dist/durability.d.ts +32 -1
- package/dist/durability.js +119 -61
- package/dist/errors.d.ts +22 -0
- package/dist/errors.js +31 -0
- package/dist/experimental.d.ts +8 -0
- package/dist/experimental.js +7 -0
- package/dist/fanout.d.ts +25 -2
- package/dist/fanout.js +273 -93
- package/dist/index.d.ts +8 -7
- package/dist/index.js +3 -2
- package/dist/lifecycle.d.ts +8 -0
- package/dist/lifecycle.js +22 -0
- package/dist/log.d.ts +201 -0
- package/dist/log.js +705 -0
- package/dist/migrations.d.ts +28 -1
- package/dist/migrations.js +100 -0
- package/dist/scheduler.d.ts +40 -17
- package/dist/scheduler.js +150 -74
- package/dist/socket-protocol.d.ts +95 -0
- package/dist/socket-protocol.js +71 -0
- package/dist/socket.d.ts +92 -0
- package/dist/socket.js +273 -0
- package/dist/storage.d.ts +39 -128
- package/dist/storage.js +64 -64
- package/dist/tracked-source.d.ts +87 -0
- package/dist/tracked-source.js +7 -0
- package/dist/tracker.d.ts +96 -0
- package/dist/tracker.js +171 -0
- package/package.json +19 -5
- package/dist/kv-storage.d.ts +0 -2
- package/dist/kv-storage.js +0 -421
- package/dist/sqlite-storage.d.ts +0 -2
- package/dist/sqlite-storage.js +0 -251
package/dist/migrations.d.ts
CHANGED
|
@@ -28,6 +28,33 @@ export declare const durabilityFanoutMigrations: {
|
|
|
28
28
|
up: string;
|
|
29
29
|
down: string;
|
|
30
30
|
}[];
|
|
31
|
+
/**
|
|
32
|
+
* Ordered SQLite migrations for the outbound event log owned by
|
|
33
|
+
* {@link DurabilitySocket}. Applied automatically on construction.
|
|
34
|
+
*/
|
|
35
|
+
export declare const durabilitySocketMigrations: {
|
|
36
|
+
name: string;
|
|
37
|
+
up: string;
|
|
38
|
+
down: string;
|
|
39
|
+
}[];
|
|
40
|
+
/**
|
|
41
|
+
* Ordered SQLite migrations for the retained log and its consumer cursors.
|
|
42
|
+
* Applied automatically on construction.
|
|
43
|
+
*/
|
|
44
|
+
export declare const durabilityLogMigrations: {
|
|
45
|
+
name: string;
|
|
46
|
+
up: string;
|
|
47
|
+
down: string;
|
|
48
|
+
}[];
|
|
49
|
+
/**
|
|
50
|
+
* Leases let several workers consume one log in parallel without handing the
|
|
51
|
+
* same offsets to two of them. Applied automatically when leases are enabled.
|
|
52
|
+
*/
|
|
53
|
+
export declare const durabilityLogLeaseMigrations: {
|
|
54
|
+
name: string;
|
|
55
|
+
up: string;
|
|
56
|
+
down: string;
|
|
57
|
+
}[];
|
|
31
58
|
/** Migration names changed by one migrate call. */
|
|
32
59
|
export type DurabilityMigrationResult = {
|
|
33
60
|
/** Migrations applied in ascending order. */
|
|
@@ -35,7 +62,7 @@ export type DurabilityMigrationResult = {
|
|
|
35
62
|
/** Migrations reverted in descending order. */
|
|
36
63
|
rolledBack: string[];
|
|
37
64
|
};
|
|
38
|
-
export type MigrationCapability = 'operations' | 'namedAlarms' | 'fanout';
|
|
65
|
+
export type MigrationCapability = 'operations' | 'namedAlarms' | 'fanout' | 'socket' | 'log' | 'logLeases';
|
|
39
66
|
type MigrationStorage = Pick<DurableObjectStorage, 'sql' | 'transactionSync'>;
|
|
40
67
|
/**
|
|
41
68
|
* Moves one capability's schema to a target migration, applying or reverting as
|
package/dist/migrations.js
CHANGED
|
@@ -222,6 +222,106 @@ export const durabilityFanoutMigrations = [
|
|
|
222
222
|
`,
|
|
223
223
|
},
|
|
224
224
|
];
|
|
225
|
+
/**
|
|
226
|
+
* Ordered SQLite migrations for the outbound event log owned by
|
|
227
|
+
* {@link DurabilitySocket}. Applied automatically on construction.
|
|
228
|
+
*/
|
|
229
|
+
export const durabilitySocketMigrations = [
|
|
230
|
+
{
|
|
231
|
+
name: 'durability_0006_create_socket_events',
|
|
232
|
+
up: `
|
|
233
|
+
CREATE TABLE IF NOT EXISTS durability_socket_events (
|
|
234
|
+
seq INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
235
|
+
name TEXT NOT NULL,
|
|
236
|
+
payload TEXT NOT NULL,
|
|
237
|
+
created_at INTEGER NOT NULL
|
|
238
|
+
);
|
|
239
|
+
`,
|
|
240
|
+
down: 'DROP TABLE IF EXISTS durability_socket_events;',
|
|
241
|
+
},
|
|
242
|
+
];
|
|
243
|
+
/**
|
|
244
|
+
* Ordered SQLite migrations for the retained log and its consumer cursors.
|
|
245
|
+
* Applied automatically on construction.
|
|
246
|
+
*/
|
|
247
|
+
export const durabilityLogMigrations = [
|
|
248
|
+
{
|
|
249
|
+
name: 'durability_0007_create_log',
|
|
250
|
+
up: `
|
|
251
|
+
CREATE TABLE IF NOT EXISTS durability_log_records (
|
|
252
|
+
"offset" INTEGER PRIMARY KEY,
|
|
253
|
+
dedup_key TEXT NOT NULL UNIQUE,
|
|
254
|
+
payload TEXT NOT NULL,
|
|
255
|
+
bytes INTEGER NOT NULL,
|
|
256
|
+
appended_at INTEGER NOT NULL
|
|
257
|
+
);
|
|
258
|
+
CREATE INDEX IF NOT EXISTS durability_log_records_appended_idx
|
|
259
|
+
ON durability_log_records (appended_at, "offset");
|
|
260
|
+
CREATE TABLE IF NOT EXISTS durability_log_cursors (
|
|
261
|
+
consumer TEXT PRIMARY KEY,
|
|
262
|
+
"offset" INTEGER NOT NULL,
|
|
263
|
+
committed_at INTEGER NOT NULL
|
|
264
|
+
);
|
|
265
|
+
CREATE TABLE IF NOT EXISTS durability_log_segments (
|
|
266
|
+
first_offset INTEGER PRIMARY KEY,
|
|
267
|
+
last_offset INTEGER NOT NULL,
|
|
268
|
+
locator TEXT NOT NULL,
|
|
269
|
+
bytes INTEGER NOT NULL,
|
|
270
|
+
flushed_at INTEGER NOT NULL
|
|
271
|
+
);
|
|
272
|
+
CREATE TABLE IF NOT EXISTS durability_log_seq (
|
|
273
|
+
id INTEGER PRIMARY KEY CHECK (id = 1),
|
|
274
|
+
next_offset INTEGER NOT NULL,
|
|
275
|
+
total_bytes INTEGER NOT NULL
|
|
276
|
+
);
|
|
277
|
+
INSERT OR IGNORE INTO durability_log_seq (id, next_offset, total_bytes)
|
|
278
|
+
VALUES (1, 0, 0);
|
|
279
|
+
`,
|
|
280
|
+
down: `
|
|
281
|
+
DROP INDEX IF EXISTS durability_log_records_appended_idx;
|
|
282
|
+
DROP TABLE IF EXISTS durability_log_records;
|
|
283
|
+
DROP TABLE IF EXISTS durability_log_cursors;
|
|
284
|
+
DROP TABLE IF EXISTS durability_log_segments;
|
|
285
|
+
DROP TABLE IF EXISTS durability_log_seq;
|
|
286
|
+
`,
|
|
287
|
+
},
|
|
288
|
+
];
|
|
289
|
+
/**
|
|
290
|
+
* Leases let several workers consume one log in parallel without handing the
|
|
291
|
+
* same offsets to two of them. Applied automatically when leases are enabled.
|
|
292
|
+
*/
|
|
293
|
+
export const durabilityLogLeaseMigrations = [
|
|
294
|
+
{
|
|
295
|
+
name: 'durability_0008_create_log_leases',
|
|
296
|
+
up: `
|
|
297
|
+
CREATE TABLE IF NOT EXISTS durability_log_leases (
|
|
298
|
+
batch_id TEXT PRIMARY KEY,
|
|
299
|
+
consumer TEXT NOT NULL,
|
|
300
|
+
first_offset INTEGER NOT NULL,
|
|
301
|
+
last_offset INTEGER NOT NULL,
|
|
302
|
+
state TEXT NOT NULL
|
|
303
|
+
CHECK (state IN ('active', 'pending', 'acked', 'skipped')),
|
|
304
|
+
attempt INTEGER NOT NULL,
|
|
305
|
+
expires_at INTEGER NOT NULL,
|
|
306
|
+
created_at INTEGER NOT NULL
|
|
307
|
+
);
|
|
308
|
+
CREATE INDEX IF NOT EXISTS durability_log_leases_claim_idx
|
|
309
|
+
ON durability_log_leases (consumer, state, expires_at, first_offset);
|
|
310
|
+
CREATE INDEX IF NOT EXISTS durability_log_leases_order_idx
|
|
311
|
+
ON durability_log_leases (consumer, first_offset);
|
|
312
|
+
CREATE TABLE IF NOT EXISTS durability_log_allocations (
|
|
313
|
+
consumer TEXT PRIMARY KEY,
|
|
314
|
+
allocated_offset INTEGER NOT NULL
|
|
315
|
+
);
|
|
316
|
+
`,
|
|
317
|
+
down: `
|
|
318
|
+
DROP INDEX IF EXISTS durability_log_leases_claim_idx;
|
|
319
|
+
DROP INDEX IF EXISTS durability_log_leases_order_idx;
|
|
320
|
+
DROP TABLE IF EXISTS durability_log_leases;
|
|
321
|
+
DROP TABLE IF EXISTS durability_log_allocations;
|
|
322
|
+
`,
|
|
323
|
+
},
|
|
324
|
+
];
|
|
225
325
|
/**
|
|
226
326
|
* Newer library versions may leave migrations this one does not know about.
|
|
227
327
|
* On downgrade their stored `down` scripts are run, but only this many, so an
|
package/dist/scheduler.d.ts
CHANGED
|
@@ -1,17 +1,16 @@
|
|
|
1
1
|
import { type Emit, type LifecycleEntity, type LifecycleHook } from './lifecycle.js';
|
|
2
2
|
import { type MigrationCapability } from './migrations.js';
|
|
3
3
|
import { type AttemptPolicy } from './policy.js';
|
|
4
|
-
import { type
|
|
4
|
+
import { type DurableTable } from './storage.js';
|
|
5
5
|
import type { DurableMigrations } from '@durability/storage';
|
|
6
6
|
/** The Durable Object state durability needs: storage plus optional `waitUntil`. */
|
|
7
7
|
export type DurabilityContext = Pick<DurableObjectState, 'storage'> & Partial<Pick<DurableObjectState, 'waitUntil'>>;
|
|
8
|
-
export type DurabilityStorageBackend = 'sqlite' | 'kv';
|
|
9
8
|
/** Options for the scheduler shared by every durability helper in one Durable Object. */
|
|
10
9
|
export type DurabilitySchedulerOptions = {
|
|
11
|
-
/** Storage backend configured for the Durable Object class. Defaults to SQLite. */
|
|
12
|
-
storageBackend?: DurabilityStorageBackend;
|
|
13
10
|
/** Maximum handlers running at once across eager operations and alarm work. Defaults to 10. */
|
|
14
11
|
alarmConcurrency?: number;
|
|
12
|
+
/** Physical-alarm delay from 1 to 15 seconds. Defaults to 15 seconds. */
|
|
13
|
+
alarmMinDelayMs?: number;
|
|
15
14
|
/** Time before a running alarm hands unfinished work to a new alarm. Defaults to 14 minutes. */
|
|
16
15
|
alarmHandoffMs?: number;
|
|
17
16
|
};
|
|
@@ -29,11 +28,17 @@ export type SchedulerAttachment = {
|
|
|
29
28
|
} | (DurabilitySchedulerConfig & {
|
|
30
29
|
scheduler?: never;
|
|
31
30
|
});
|
|
32
|
-
/** One record
|
|
31
|
+
/** One record table attached to the scheduler: how to find and run its due work. */
|
|
33
32
|
export type Participant = {
|
|
34
|
-
|
|
33
|
+
table: DurableTable;
|
|
35
34
|
runDue(now: number, info: AlarmInvocationInfo | undefined): Promise<void>;
|
|
36
35
|
};
|
|
36
|
+
/**
|
|
37
|
+
* Called after a record may have settled. Successful completions pass the
|
|
38
|
+
* executed row because some tables delete it on success; other changes pass
|
|
39
|
+
* nothing and listeners re-read storage.
|
|
40
|
+
*/
|
|
41
|
+
export type SettlementListener = (completed?: ExecutionSpec['row']) => void;
|
|
37
42
|
export type ActiveExecution = {
|
|
38
43
|
generation: string;
|
|
39
44
|
createdAt: number;
|
|
@@ -42,9 +47,12 @@ export type ActiveExecution = {
|
|
|
42
47
|
};
|
|
43
48
|
/** Everything that differs between running a durable operation and a named alarm attempt. */
|
|
44
49
|
export type ExecutionSpec = {
|
|
45
|
-
|
|
50
|
+
table: DurableTable;
|
|
46
51
|
key: string;
|
|
47
|
-
row:
|
|
52
|
+
row: {
|
|
53
|
+
generation_id: string;
|
|
54
|
+
created_at: number;
|
|
55
|
+
};
|
|
48
56
|
label: string;
|
|
49
57
|
entity: LifecycleEntity;
|
|
50
58
|
active: Map<string, ActiveExecution>;
|
|
@@ -58,38 +66,53 @@ export type ExecutionSpec = {
|
|
|
58
66
|
* undefined when a newer generation won. May throw to turn the success into
|
|
59
67
|
* a failure.
|
|
60
68
|
*/
|
|
61
|
-
complete: (result: unknown, attempt: number) =>
|
|
69
|
+
complete: (result: unknown, attempt: number) => number | undefined;
|
|
62
70
|
/** Kind-specific terminal rule beyond non-retryable errors and exhausted attempts. */
|
|
63
71
|
isTerminal: (error: unknown) => boolean;
|
|
64
72
|
};
|
|
65
73
|
/**
|
|
66
|
-
* Shared machinery behind {@link DurabilityScheduler}:
|
|
74
|
+
* Shared machinery behind {@link DurabilityScheduler}: transactions, the physical
|
|
67
75
|
* alarm, the concurrency pool, and the attempt lifecycle. Package-internal.
|
|
68
76
|
*/
|
|
69
77
|
export declare class Engine {
|
|
70
|
-
readonly
|
|
71
|
-
private readonly backend;
|
|
78
|
+
readonly sql: SqlStorage;
|
|
72
79
|
private readonly context;
|
|
73
80
|
private readonly alarmConcurrency;
|
|
81
|
+
private readonly alarmMinDelayMs;
|
|
74
82
|
private readonly alarmHandoffMs;
|
|
75
83
|
private readonly participants;
|
|
76
84
|
private availablePermits;
|
|
77
85
|
private readonly permitWaiters;
|
|
78
86
|
private alarmRefresh;
|
|
87
|
+
private earlyAlarm;
|
|
88
|
+
private readonly watchers;
|
|
79
89
|
constructor(context: DurabilityContext, options: DurabilitySchedulerOptions);
|
|
80
90
|
register(participant: Participant): void;
|
|
81
91
|
waitUntil(promise: Promise<unknown>, onError: (error: unknown) => void): void;
|
|
82
92
|
emitter(hook: LifecycleHook | undefined): Emit;
|
|
83
|
-
/** Applies a helper's SQLite migrations
|
|
93
|
+
/** Applies a helper's SQLite migrations. */
|
|
84
94
|
migrateSchema(capability: MigrationCapability, migrations: DurableMigrations): void;
|
|
85
|
-
|
|
95
|
+
/** Runs synchronous writes atomically without touching the physical alarm. */
|
|
96
|
+
transactionSync<T>(write: (sql: SqlStorage) => T): T;
|
|
97
|
+
/**
|
|
98
|
+
* Runs the writes and reconciles the physical alarm in one storage
|
|
99
|
+
* transaction, then publishes the in-memory timer only after commit.
|
|
100
|
+
*/
|
|
101
|
+
transaction<T>(write: (sql: SqlStorage) => T): Promise<T>;
|
|
102
|
+
/** Returns the earliest pending time and maintains its bounded physical fallback. */
|
|
103
|
+
private reconcile;
|
|
104
|
+
private scheduleEarlyAlarm;
|
|
86
105
|
/** Coalesces concurrent refresh requests into one reconciliation transaction. */
|
|
87
106
|
scheduleNextAlarm(): Promise<void>;
|
|
88
107
|
/**
|
|
89
108
|
* Aborts matching active handlers, then deletes one kind's records created
|
|
90
|
-
* before the timestamp
|
|
109
|
+
* before the timestamp and reconciles the alarm in the same transaction.
|
|
91
110
|
*/
|
|
92
|
-
purge(
|
|
111
|
+
purge(table: DurableTable, before: number, active: Map<string, ActiveExecution>, abortReason: string): Promise<number>;
|
|
112
|
+
/** In-memory only: listeners never survive eviction. */
|
|
113
|
+
watch(table: DurableTable, key: string, listener: SettlementListener): () => void;
|
|
114
|
+
/** Call only after the change has committed. */
|
|
115
|
+
notify(table: DurableTable, key: string, completed?: ExecutionSpec['row']): void;
|
|
93
116
|
tryAcquirePermit(): (() => void) | undefined;
|
|
94
117
|
acquirePermit(): Promise<() => void>;
|
|
95
118
|
private createRelease;
|
|
@@ -110,7 +133,7 @@ export declare class Engine {
|
|
|
110
133
|
private settleFailure;
|
|
111
134
|
}
|
|
112
135
|
/**
|
|
113
|
-
* Owns the Durable Object's physical alarm, storage
|
|
136
|
+
* Owns the Durable Object's physical alarm, storage, and concurrency
|
|
114
137
|
* pool so {@link Durability} and {@link DurabilityAlarms} can share them.
|
|
115
138
|
*
|
|
116
139
|
* Construct one per Durable Object when using both helpers, pass it to each,
|
package/dist/scheduler.js
CHANGED
|
@@ -1,12 +1,15 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
|
-
import { DurableRetryPolicyError, isErrorInstance, isNonRetryable, serializeError, } from './errors.js';
|
|
3
|
-
import { createKvDurabilityStorage } from './kv-storage.js';
|
|
2
|
+
import { DurableRetryPolicyError, isErrorInstance, isNonRetryable, reportFailure, serializeError, } from './errors.js';
|
|
4
3
|
import { createEmitter, } from './lifecycle.js';
|
|
5
4
|
import { migrate } from './migrations.js';
|
|
6
5
|
import { assertPositiveInteger } from './policy.js';
|
|
7
|
-
import {
|
|
8
|
-
|
|
9
|
-
const
|
|
6
|
+
import { claimAttempt, exhaustAttempts, updateAttempt, } from './storage.js';
|
|
7
|
+
const minAlarmDelayMs = 1000;
|
|
8
|
+
const maxAlarmDelayMs = 15000;
|
|
9
|
+
const nextAttemptRowSchema = z.object({
|
|
10
|
+
next_attempt_at: z.number().int().nonnegative(),
|
|
11
|
+
});
|
|
12
|
+
const countRowSchema = z.object({ count: z.number().int().nonnegative() });
|
|
10
13
|
const decideRetry = (spec, attempt, error) => {
|
|
11
14
|
const terminal = isNonRetryable(error) ||
|
|
12
15
|
attempt >= spec.policy.maxAttempts ||
|
|
@@ -46,28 +49,32 @@ const decideRetry = (spec, attempt, error) => {
|
|
|
46
49
|
}
|
|
47
50
|
};
|
|
48
51
|
/**
|
|
49
|
-
* Shared machinery behind {@link DurabilityScheduler}:
|
|
52
|
+
* Shared machinery behind {@link DurabilityScheduler}: transactions, the physical
|
|
50
53
|
* alarm, the concurrency pool, and the attempt lifecycle. Package-internal.
|
|
51
54
|
*/
|
|
52
55
|
export class Engine {
|
|
53
56
|
constructor(context, options) {
|
|
54
57
|
this.participants = [];
|
|
55
58
|
this.permitWaiters = [];
|
|
59
|
+
this.watchers = new Map();
|
|
56
60
|
this.context = context;
|
|
57
|
-
this.
|
|
58
|
-
this.storage =
|
|
59
|
-
this.backend === 'kv'
|
|
60
|
-
? createKvDurabilityStorage(context.storage)
|
|
61
|
-
: createSqliteDurabilityStorage(context.storage);
|
|
61
|
+
this.sql = context.storage.sql;
|
|
62
62
|
this.alarmConcurrency = options.alarmConcurrency ?? 10;
|
|
63
|
+
this.alarmMinDelayMs = options.alarmMinDelayMs ?? 15000;
|
|
63
64
|
this.alarmHandoffMs = options.alarmHandoffMs ?? 14 * 60000;
|
|
64
65
|
assertPositiveInteger('alarmConcurrency', this.alarmConcurrency);
|
|
66
|
+
assertPositiveInteger('alarmMinDelayMs', this.alarmMinDelayMs);
|
|
67
|
+
if (this.alarmMinDelayMs < minAlarmDelayMs ||
|
|
68
|
+
this.alarmMinDelayMs > maxAlarmDelayMs) {
|
|
69
|
+
throw new RangeError(`alarmMinDelayMs must be between ${minAlarmDelayMs} and ${maxAlarmDelayMs}`);
|
|
70
|
+
}
|
|
65
71
|
assertPositiveInteger('alarmHandoffMs', this.alarmHandoffMs);
|
|
66
72
|
this.availablePermits = this.alarmConcurrency;
|
|
67
73
|
}
|
|
68
74
|
register(participant) {
|
|
69
|
-
|
|
70
|
-
|
|
75
|
+
const { name } = participant.table;
|
|
76
|
+
if (this.participants.some(({ table }) => table.name === name)) {
|
|
77
|
+
throw new Error(`A ${name} helper is already attached to this scheduler`);
|
|
71
78
|
}
|
|
72
79
|
this.participants.push(participant);
|
|
73
80
|
}
|
|
@@ -85,32 +92,94 @@ export class Engine {
|
|
|
85
92
|
emitter(hook) {
|
|
86
93
|
return createEmitter(hook, (promise) => this.context.waitUntil?.(promise));
|
|
87
94
|
}
|
|
88
|
-
/** Applies a helper's SQLite migrations
|
|
95
|
+
/** Applies a helper's SQLite migrations. */
|
|
89
96
|
migrateSchema(capability, migrations) {
|
|
90
|
-
|
|
91
|
-
|
|
97
|
+
migrate(this.context.storage, capability, migrations, undefined);
|
|
98
|
+
}
|
|
99
|
+
/** Runs synchronous writes atomically without touching the physical alarm. */
|
|
100
|
+
transactionSync(write) {
|
|
101
|
+
return this.context.storage.transactionSync(() => write(this.sql));
|
|
102
|
+
}
|
|
103
|
+
/**
|
|
104
|
+
* Runs the writes and reconciles the physical alarm in one storage
|
|
105
|
+
* transaction, then publishes the in-memory timer only after commit.
|
|
106
|
+
*/
|
|
107
|
+
async transaction(write) {
|
|
108
|
+
let nextPendingAt;
|
|
109
|
+
const result = await this.context.storage.transaction(async (transaction) => {
|
|
110
|
+
const value = write(this.sql);
|
|
111
|
+
nextPendingAt = await this.reconcile(transaction);
|
|
112
|
+
return value;
|
|
113
|
+
});
|
|
114
|
+
this.scheduleEarlyAlarm(nextPendingAt);
|
|
115
|
+
return result;
|
|
116
|
+
}
|
|
117
|
+
/** Returns the earliest pending time and maintains its bounded physical fallback. */
|
|
118
|
+
async reconcile(physicalAlarm) {
|
|
119
|
+
const scheduled = this.participants.flatMap(({ table }) => nextAttemptRowSchema
|
|
120
|
+
.array()
|
|
121
|
+
.parse(this.sql
|
|
122
|
+
.exec(`SELECT next_attempt_at FROM ${table.name}
|
|
123
|
+
WHERE status = 'pending'
|
|
124
|
+
ORDER BY next_attempt_at ASC
|
|
125
|
+
LIMIT 1`)
|
|
126
|
+
.toArray())
|
|
127
|
+
.map((row) => row.next_attempt_at));
|
|
128
|
+
const nextAlarmAt = scheduled.length === 0 ? undefined : Math.min(...scheduled);
|
|
129
|
+
const currentAlarm = await physicalAlarm.getAlarm();
|
|
130
|
+
if (nextAlarmAt === undefined) {
|
|
131
|
+
if (currentAlarm !== null) {
|
|
132
|
+
await physicalAlarm.deleteAlarm();
|
|
133
|
+
}
|
|
134
|
+
return undefined;
|
|
135
|
+
}
|
|
136
|
+
const target = Math.max(nextAlarmAt, Date.now() + this.alarmMinDelayMs);
|
|
137
|
+
// Preserve a fallback inside this range so frequent reconciliation cannot push it later.
|
|
138
|
+
if (currentAlarm === null ||
|
|
139
|
+
currentAlarm < nextAlarmAt ||
|
|
140
|
+
currentAlarm > target) {
|
|
141
|
+
await physicalAlarm.setAlarm(target);
|
|
92
142
|
}
|
|
143
|
+
return nextAlarmAt;
|
|
93
144
|
}
|
|
94
|
-
|
|
95
|
-
|
|
145
|
+
scheduleEarlyAlarm(timestamp) {
|
|
146
|
+
if (this.earlyAlarm?.timestamp === timestamp) {
|
|
147
|
+
return;
|
|
148
|
+
}
|
|
149
|
+
if (this.earlyAlarm) {
|
|
150
|
+
clearTimeout(this.earlyAlarm.timer);
|
|
151
|
+
this.earlyAlarm = undefined;
|
|
152
|
+
}
|
|
153
|
+
if (timestamp === undefined ||
|
|
154
|
+
timestamp >= Date.now() + this.alarmMinDelayMs) {
|
|
155
|
+
return;
|
|
156
|
+
}
|
|
157
|
+
const timer = setTimeout(() => {
|
|
158
|
+
if (this.earlyAlarm?.timer !== timer) {
|
|
159
|
+
return;
|
|
160
|
+
}
|
|
161
|
+
this.earlyAlarm = undefined;
|
|
162
|
+
const onError = (error) => reportFailure('durability.background_execution.failed', { scheduledTime: timestamp }, error);
|
|
163
|
+
const execution = this.alarm().catch(onError);
|
|
164
|
+
this.waitUntil(execution, onError);
|
|
165
|
+
}, Math.max(timestamp - Date.now(), 0));
|
|
166
|
+
this.earlyAlarm = { timestamp, timer };
|
|
96
167
|
}
|
|
97
168
|
/** Coalesces concurrent refresh requests into one reconciliation transaction. */
|
|
98
169
|
scheduleNextAlarm() {
|
|
99
170
|
if (this.alarmRefresh) {
|
|
100
171
|
return this.alarmRefresh;
|
|
101
172
|
}
|
|
102
|
-
this.alarmRefresh = this.
|
|
103
|
-
.transaction((transaction) => this.reconcile(transaction))
|
|
104
|
-
.finally(() => {
|
|
173
|
+
this.alarmRefresh = this.transaction(() => undefined).finally(() => {
|
|
105
174
|
this.alarmRefresh = undefined;
|
|
106
175
|
});
|
|
107
176
|
return this.alarmRefresh;
|
|
108
177
|
}
|
|
109
178
|
/**
|
|
110
179
|
* Aborts matching active handlers, then deletes one kind's records created
|
|
111
|
-
* before the timestamp
|
|
180
|
+
* before the timestamp and reconciles the alarm in the same transaction.
|
|
112
181
|
*/
|
|
113
|
-
async purge(
|
|
182
|
+
async purge(table, before, active, abortReason) {
|
|
114
183
|
if (!Number.isSafeInteger(before) || before < 0) {
|
|
115
184
|
throw new RangeError('timestamp must be a non-negative safe integer');
|
|
116
185
|
}
|
|
@@ -119,18 +188,40 @@ export class Engine {
|
|
|
119
188
|
running.controller?.abort(new Error(abortReason));
|
|
120
189
|
}
|
|
121
190
|
}
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
191
|
+
const removed = await this.transaction((sql) => {
|
|
192
|
+
const { count } = countRowSchema.parse(sql
|
|
193
|
+
.exec(`SELECT COUNT(*) AS count FROM ${table.name} WHERE created_at < ?`, before)
|
|
194
|
+
.toArray()[0]);
|
|
195
|
+
sql.exec(`DELETE FROM ${table.name} WHERE created_at < ?`, before);
|
|
196
|
+
return count;
|
|
197
|
+
});
|
|
198
|
+
if (removed > 0) {
|
|
199
|
+
for (const key of this.watchers.get(table.name)?.keys() ?? []) {
|
|
200
|
+
this.notify(table, key);
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
return removed;
|
|
204
|
+
}
|
|
205
|
+
/** In-memory only: listeners never survive eviction. */
|
|
206
|
+
watch(table, key, listener) {
|
|
207
|
+
const byKey = this.watchers.get(table.name) ??
|
|
208
|
+
new Map();
|
|
209
|
+
this.watchers.set(table.name, byKey);
|
|
210
|
+
const listeners = byKey.get(key) ?? new Set();
|
|
211
|
+
byKey.set(key, listeners);
|
|
212
|
+
listeners.add(listener);
|
|
213
|
+
return () => {
|
|
214
|
+
listeners.delete(listener);
|
|
215
|
+
if (listeners.size === 0 && byKey.get(key) === listeners) {
|
|
216
|
+
byKey.delete(key);
|
|
217
|
+
}
|
|
218
|
+
};
|
|
219
|
+
}
|
|
220
|
+
/** Call only after the change has committed. */
|
|
221
|
+
notify(table, key, completed) {
|
|
222
|
+
for (const listener of this.watchers.get(table.name)?.get(key) ?? []) {
|
|
223
|
+
listener(completed);
|
|
224
|
+
}
|
|
134
225
|
}
|
|
135
226
|
tryAcquirePermit() {
|
|
136
227
|
if (this.availablePermits === 0) {
|
|
@@ -208,10 +299,11 @@ export class Engine {
|
|
|
208
299
|
await this.scheduleNextAlarm();
|
|
209
300
|
}
|
|
210
301
|
armForHandoff(timestamp) {
|
|
211
|
-
return this.storage.transaction(async (
|
|
302
|
+
return this.context.storage.transaction(async (physicalAlarm) => {
|
|
212
303
|
const currentAlarm = await physicalAlarm.getAlarm();
|
|
213
|
-
|
|
214
|
-
|
|
304
|
+
const target = Math.max(timestamp, Date.now() + this.alarmMinDelayMs);
|
|
305
|
+
if (currentAlarm === null || currentAlarm > target) {
|
|
306
|
+
await physicalAlarm.setAlarm(target);
|
|
215
307
|
}
|
|
216
308
|
});
|
|
217
309
|
}
|
|
@@ -244,7 +336,7 @@ export class Engine {
|
|
|
244
336
|
};
|
|
245
337
|
const execution = (async () => {
|
|
246
338
|
releasePermit ?? (releasePermit = await this.acquirePermit());
|
|
247
|
-
const attempt =
|
|
339
|
+
const attempt = claimAttempt(this.sql, spec.table, key, row.generation_id, policy.maxAttempts);
|
|
248
340
|
if (attempt === undefined) {
|
|
249
341
|
await this.failExhausted(spec);
|
|
250
342
|
return;
|
|
@@ -276,8 +368,9 @@ export class Engine {
|
|
|
276
368
|
}, policy.attemptTimeoutMs);
|
|
277
369
|
});
|
|
278
370
|
const result = await Promise.race([handlerResult, timeoutResult]);
|
|
279
|
-
const timestamp =
|
|
371
|
+
const timestamp = spec.complete(result, attempt);
|
|
280
372
|
if (timestamp !== undefined) {
|
|
373
|
+
this.notify(spec.table, key, row);
|
|
281
374
|
spec.emit({
|
|
282
375
|
...spec.entity,
|
|
283
376
|
type: 'attempt_settled',
|
|
@@ -285,6 +378,7 @@ export class Engine {
|
|
|
285
378
|
attempt,
|
|
286
379
|
durationMs: Math.max(0, timestamp - startedAt),
|
|
287
380
|
outcome: 'completed',
|
|
381
|
+
result,
|
|
288
382
|
});
|
|
289
383
|
}
|
|
290
384
|
}
|
|
@@ -309,50 +403,32 @@ export class Engine {
|
|
|
309
403
|
}
|
|
310
404
|
async failExhausted(spec) {
|
|
311
405
|
const { key, row, policy } = spec;
|
|
312
|
-
const
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
current.attempt < policy.maxAttempts) {
|
|
406
|
+
const exhausted = serializeError(spec.exhaustedError());
|
|
407
|
+
const attempt = await this.transaction((sql) => exhaustAttempts(sql, spec.table, key, row.generation_id, policy.maxAttempts, exhausted));
|
|
408
|
+
if (attempt === undefined) {
|
|
316
409
|
return;
|
|
317
410
|
}
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
await this.reconcile(transaction);
|
|
411
|
+
this.notify(spec.table, key);
|
|
412
|
+
spec.emit({
|
|
413
|
+
...spec.entity,
|
|
414
|
+
type: 'terminal',
|
|
415
|
+
timestamp: Date.now(),
|
|
416
|
+
attempt,
|
|
417
|
+
reason: 'attempts_exhausted',
|
|
418
|
+
error: exhausted,
|
|
327
419
|
});
|
|
328
|
-
if (updated) {
|
|
329
|
-
spec.emit({
|
|
330
|
-
...spec.entity,
|
|
331
|
-
type: 'terminal',
|
|
332
|
-
timestamp: Date.now(),
|
|
333
|
-
attempt: current.attempt,
|
|
334
|
-
reason: 'attempts_exhausted',
|
|
335
|
-
error: exhausted,
|
|
336
|
-
});
|
|
337
|
-
}
|
|
338
420
|
}
|
|
339
421
|
async settleFailure(spec, attempt, startedAt, caught) {
|
|
340
422
|
const { key, row } = spec;
|
|
341
423
|
const decision = decideRetry(spec, attempt, caught);
|
|
342
424
|
const serializedError = serializeError(decision.error);
|
|
343
|
-
|
|
344
|
-
await this.storage.transaction(async (transaction) => {
|
|
345
|
-
updated = await transaction[spec.kind].settle(key, row.generation_id, attempt, {
|
|
346
|
-
status: decision.terminal ? 'failed' : 'pending',
|
|
347
|
-
next_attempt_at: decision.nextAttemptAt,
|
|
348
|
-
last_error: serializedError.message,
|
|
349
|
-
last_error_name: serializedError.name,
|
|
350
|
-
});
|
|
351
|
-
await this.reconcile(transaction);
|
|
352
|
-
});
|
|
425
|
+
const updated = await this.transaction((sql) => updateAttempt(sql, spec.table, { key, generation: row.generation_id, attempt }, 'status = ?, next_attempt_at = ?, last_error = ?, last_error_name = ?', decision.terminal ? 'failed' : 'pending', decision.nextAttemptAt, serializedError.message, serializedError.name));
|
|
353
426
|
if (!updated) {
|
|
354
427
|
return;
|
|
355
428
|
}
|
|
429
|
+
if (decision.terminal) {
|
|
430
|
+
this.notify(spec.table, key);
|
|
431
|
+
}
|
|
356
432
|
const settled = {
|
|
357
433
|
...spec.entity,
|
|
358
434
|
type: 'attempt_settled',
|
|
@@ -372,7 +448,7 @@ export class Engine {
|
|
|
372
448
|
}
|
|
373
449
|
const engines = new WeakMap();
|
|
374
450
|
/**
|
|
375
|
-
* Owns the Durable Object's physical alarm, storage
|
|
451
|
+
* Owns the Durable Object's physical alarm, storage, and concurrency
|
|
376
452
|
* pool so {@link Durability} and {@link DurabilityAlarms} can share them.
|
|
377
453
|
*
|
|
378
454
|
* Construct one per Durable Object when using both helpers, pass it to each,
|