@pramen/server 0.0.9 → 0.0.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/durable-object.d.ts +20 -0
- package/dist/durable-object.js +88 -4
- package/dist/index.d.ts +1 -1
- package/dist/pramen.d.ts +8 -2
- package/dist/pramen.js +5 -2
- package/dist/runtime/dispatch.d.ts +9 -1
- package/dist/runtime/dispatch.js +21 -2
- package/dist/runtime/outbox.d.ts +57 -0
- package/dist/runtime/outbox.js +128 -0
- package/dist/sdk/handlers.d.ts +28 -0
- package/dist/worker.d.ts +1 -0
- package/dist/worker.js +89 -1
- package/package.json +1 -1
- package/src/durable-object.ts +96 -4
- package/src/index.ts +1 -1
- package/src/pramen.ts +10 -3
- package/src/runtime/dispatch.ts +25 -3
- package/src/runtime/outbox.ts +204 -0
- package/src/sdk/handlers.ts +27 -0
- package/src/worker.ts +91 -1
package/dist/worker.js
CHANGED
|
@@ -4,9 +4,11 @@
|
|
|
4
4
|
// endpoints (/tenants, /admin/recover, /admin/schema). createPramen() pairs the
|
|
5
5
|
// returned fetch with the matching DO class; a consumer just re-exports both.
|
|
6
6
|
import { authorizeTenant, HmacStrategy, isAdmin, JwksStrategy, resolveIdentity } from "./auth";
|
|
7
|
-
import { dispatch } from "./runtime/dispatch";
|
|
7
|
+
import { dispatch, tasksFacade, bindTasks } from "./runtime/dispatch";
|
|
8
|
+
import { ensureOutbox, drainOutbox, listTasks } from "./runtime/outbox";
|
|
8
9
|
import { migrate } from "./runtime/migrate";
|
|
9
10
|
import { compileAcl } from "./runtime/acl";
|
|
11
|
+
import { Db } from "./runtime/db";
|
|
10
12
|
import { D1Driver } from "./runtime/driver";
|
|
11
13
|
import { toResponse } from "./runtime/errors";
|
|
12
14
|
import { Kv } from "./runtime/kv";
|
|
@@ -92,6 +94,7 @@ export function makeWorker(app) {
|
|
|
92
94
|
const ensureD1Migrated = (driver, allowDestructive) => {
|
|
93
95
|
if (!d1Ready) {
|
|
94
96
|
d1Ready = migrate(driver, app.schema, { allowDestructive })
|
|
97
|
+
.then(() => ensureOutbox(driver)) // the deferred-tasks table also lives in D1
|
|
95
98
|
.then(() => undefined)
|
|
96
99
|
.catch((e) => {
|
|
97
100
|
d1Ready = undefined;
|
|
@@ -100,6 +103,31 @@ export function makeWorker(app) {
|
|
|
100
103
|
}
|
|
101
104
|
return d1Ready;
|
|
102
105
|
};
|
|
106
|
+
// A privileged, system-scoped context for running task handlers on the D1 (Worker)
|
|
107
|
+
// path — mirrors the DO's taskCtx. No live socket, so no DO; drained by a Cron / the
|
|
108
|
+
// /admin/tasks/drain route, never a DO alarm.
|
|
109
|
+
const d1TaskCtx = (driver, env) => {
|
|
110
|
+
const identity = { roles: ["admin"] };
|
|
111
|
+
const files = createFiles({ tenant: "main", secret: filesSecret(env), adapter: new R2Adapter(env.FILES) });
|
|
112
|
+
const db = new Db(driver, { acl: d1Acl, identity, system: true, schema: app.schema }, app.schema);
|
|
113
|
+
return { db, kv: new Kv(env.KV), files, env: env, identity, tasks: tasksFacade(driver) };
|
|
114
|
+
};
|
|
115
|
+
/** Drain the D1 outbox in the Worker (no DO/alarm on this path) — called by the
|
|
116
|
+
* /admin/tasks/drain route with `x-pramen-store: d1`, and by `scheduled()` (Cron). */
|
|
117
|
+
const drainD1 = async (env) => {
|
|
118
|
+
if (!env.DB)
|
|
119
|
+
throw new Error("D1 store is not configured");
|
|
120
|
+
const driver = new D1Driver(env.DB);
|
|
121
|
+
await ensureD1Migrated(driver, env.PRAMEN_ALLOW_DESTRUCTIVE === "true");
|
|
122
|
+
return drainOutbox(driver, bindTasks(app.tasks, d1TaskCtx(driver, env)), Date.now());
|
|
123
|
+
};
|
|
124
|
+
const listD1Tasks = async (env, status, limit) => {
|
|
125
|
+
if (!env.DB)
|
|
126
|
+
throw new Error("D1 store is not configured");
|
|
127
|
+
const driver = new D1Driver(env.DB);
|
|
128
|
+
await ensureD1Migrated(driver, env.PRAMEN_ALLOW_DESTRUCTIVE === "true");
|
|
129
|
+
return listTasks(driver, { status, limit });
|
|
130
|
+
};
|
|
103
131
|
return {
|
|
104
132
|
async fetch(request, env) {
|
|
105
133
|
const url = new URL(request.url);
|
|
@@ -196,6 +224,60 @@ export function makeWorker(app) {
|
|
|
196
224
|
}));
|
|
197
225
|
return withCors(res, cors);
|
|
198
226
|
}
|
|
227
|
+
// --- admin: drain the deferred-task outbox now (the DO also self-drains via an
|
|
228
|
+
// alarm; this is the manual / Cron entry, and the ONLY drain for the D1 path).
|
|
229
|
+
// `x-pramen-store: d1` drains the D1 outbox in the Worker; else the tenant's DO. ---
|
|
230
|
+
if (url.pathname === "/admin/tasks/drain" && request.method === "POST") {
|
|
231
|
+
if (!isAdmin(identity))
|
|
232
|
+
return forbidden("tasks");
|
|
233
|
+
if (request.headers.get("x-pramen-store") === "d1") {
|
|
234
|
+
try {
|
|
235
|
+
return withCors(json({ ok: true, result: await drainD1(env) }), cors);
|
|
236
|
+
}
|
|
237
|
+
catch (err) {
|
|
238
|
+
const { status, body } = toResponse(err);
|
|
239
|
+
return withCors(json(body, status), cors);
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
const body = (await request.json().catch(() => ({})));
|
|
243
|
+
const tenant = typeof body.tenant === "string" && body.tenant ? body.tenant : "main";
|
|
244
|
+
const partition = typeof body.partition === "string" && body.partition ? body.partition : DEFAULT_PARTITION;
|
|
245
|
+
const stub = partitionStubFor(env, tenant, partition);
|
|
246
|
+
const res = await stub.fetch(new Request("https://do/__admin/tasks/drain", {
|
|
247
|
+
method: "POST",
|
|
248
|
+
headers: { "content-type": "application/json", "x-pramen-tenant": tenant, "x-pramen-partition": partition },
|
|
249
|
+
}));
|
|
250
|
+
return withCors(res, cors);
|
|
251
|
+
}
|
|
252
|
+
// --- admin: list outbox tasks (inspect dead-letters etc.). ?status=&limit=,
|
|
253
|
+
// ?tenant=&partition= (DO store) or x-pramen-store: d1 (Worker outbox). ---
|
|
254
|
+
if (url.pathname === "/admin/tasks/list") {
|
|
255
|
+
if (!isAdmin(identity))
|
|
256
|
+
return withCors(forbidden("tasks"), cors);
|
|
257
|
+
const status = url.searchParams.get("status") ?? undefined;
|
|
258
|
+
const limit = Number(url.searchParams.get("limit")) || undefined;
|
|
259
|
+
if (request.headers.get("x-pramen-store") === "d1") {
|
|
260
|
+
try {
|
|
261
|
+
return withCors(json({ ok: true, result: await listD1Tasks(env, status, limit) }), cors);
|
|
262
|
+
}
|
|
263
|
+
catch (err) {
|
|
264
|
+
const { status: s, body } = toResponse(err);
|
|
265
|
+
return withCors(json(body, s), cors);
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
const tenant = url.searchParams.get("tenant") ?? "main";
|
|
269
|
+
const partition = url.searchParams.get("partition") || DEFAULT_PARTITION;
|
|
270
|
+
const q = new URLSearchParams();
|
|
271
|
+
if (status)
|
|
272
|
+
q.set("status", status);
|
|
273
|
+
if (limit)
|
|
274
|
+
q.set("limit", String(limit));
|
|
275
|
+
const stub = partitionStubFor(env, tenant, partition);
|
|
276
|
+
const res = await stub.fetch(new Request(`https://do/__admin/tasks/list?${q}`, {
|
|
277
|
+
headers: { "x-pramen-tenant": tenant, "x-pramen-partition": partition },
|
|
278
|
+
}));
|
|
279
|
+
return withCors(res, cors);
|
|
280
|
+
}
|
|
199
281
|
const isRpc = url.pathname.startsWith("/rpc/");
|
|
200
282
|
const isLive = url.pathname === "/live";
|
|
201
283
|
if (!isRpc && !(isLive && isWs)) {
|
|
@@ -259,5 +341,11 @@ export function makeWorker(app) {
|
|
|
259
341
|
const res = await stub.fetch(new Request(req, { headers }));
|
|
260
342
|
return isWs ? res : withCors(res, cors);
|
|
261
343
|
},
|
|
344
|
+
// Cron Trigger entry: drains the D1 outbox (the DO path self-drains via an alarm,
|
|
345
|
+
// so it needs no cron). Wire a `[triggers] crons` in wrangler/oblaka to call this.
|
|
346
|
+
async scheduled(_event, env) {
|
|
347
|
+
if (env.DB)
|
|
348
|
+
await drainD1(env);
|
|
349
|
+
},
|
|
262
350
|
};
|
|
263
351
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@pramen/server",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.10",
|
|
4
4
|
"description": "pramen server runtime — schema, ACL, ORM, live queries, files, and the createPramen(app) factory for Cloudflare Workers + Durable Objects.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
package/src/durable-object.ts
CHANGED
|
@@ -17,7 +17,8 @@
|
|
|
17
17
|
|
|
18
18
|
import { DurableObject } from "cloudflare:workers";
|
|
19
19
|
import { migrate } from "./runtime/migrate";
|
|
20
|
-
import { dispatch } from "./runtime/dispatch";
|
|
20
|
+
import { dispatch, tasksFacade, bindTasks } from "./runtime/dispatch";
|
|
21
|
+
import { ensureOutbox, drainOutbox, listTasks } from "./runtime/outbox";
|
|
21
22
|
import { Db } from "./runtime/db";
|
|
22
23
|
import { digest } from "./runtime/digest";
|
|
23
24
|
import { compileAcl, type AclContext, type CompiledAcl } from "./runtime/acl";
|
|
@@ -28,6 +29,7 @@ import { registryKey } from "./runtime/registry";
|
|
|
28
29
|
import { DEFAULT_PARTITION } from "./sdk/schema";
|
|
29
30
|
import { createFiles, R2Adapter, type Files } from "./runtime/storage";
|
|
30
31
|
import type { Identity } from "./sdk/acl";
|
|
32
|
+
import type { HandlerContext } from "./sdk/handlers";
|
|
31
33
|
import type { PramenApp } from "./pramen";
|
|
32
34
|
import type { ClientMsg, ServerMsg, Subscription } from "./runtime/protocol";
|
|
33
35
|
|
|
@@ -75,6 +77,11 @@ export class PramenDOBase extends DurableObject<DoEnv> {
|
|
|
75
77
|
* constructor — see `ensureMigrated`. Guards against re-running. */
|
|
76
78
|
private migrated = false;
|
|
77
79
|
private files?: Files;
|
|
80
|
+
/** Have we persisted (tenant, partition) to DO storage this instance? They're
|
|
81
|
+
* persisted so a COLD alarm wake (no request header) can build a correctly-scoped
|
|
82
|
+
* task context — a DO can't introspect its own idFromName. */
|
|
83
|
+
private identityPersisted = false;
|
|
84
|
+
private identityLoaded = false;
|
|
78
85
|
|
|
79
86
|
constructor(ctx: DurableObjectState, env: DoEnv, app: PramenApp) {
|
|
80
87
|
super(ctx, env);
|
|
@@ -112,6 +119,7 @@ export class PramenDOBase extends DurableObject<DoEnv> {
|
|
|
112
119
|
await this.driver.transaction(() =>
|
|
113
120
|
migrate(this.driver, this.app.schema, { allowDestructive, partition }).then(() => {}),
|
|
114
121
|
);
|
|
122
|
+
await ensureOutbox(this.driver); // the deferred-tasks table (internal, all partitions)
|
|
115
123
|
this.migrated = true;
|
|
116
124
|
});
|
|
117
125
|
}
|
|
@@ -126,11 +134,25 @@ export class PramenDOBase extends DurableObject<DoEnv> {
|
|
|
126
134
|
|
|
127
135
|
await this.ensureMigrated();
|
|
128
136
|
await this.ensureRegistered(request);
|
|
137
|
+
// Persist (tenant, partition) once per instance so a cold alarm can rebuild the
|
|
138
|
+
// right task context (it has no request header to learn them from). Stored in the
|
|
139
|
+
// SQL store (_pramen_meta), NOT ctx.storage.put — mixing the KV-style storage API
|
|
140
|
+
// with raw `PRAGMA` trips workerd's DO SQLite authorizer (SQLITE_AUTH).
|
|
141
|
+
if (!this.identityPersisted) {
|
|
142
|
+
await this.driver.exec(
|
|
143
|
+
`INSERT OR REPLACE INTO _pramen_meta (key, value) VALUES (?, ?), (?, ?)`,
|
|
144
|
+
["_pramen_tenant", this.tenant, "_pramen_partition", this.partition].map((v) => this.driver.dialect.encode(v)),
|
|
145
|
+
);
|
|
146
|
+
this.identityPersisted = true;
|
|
147
|
+
this.identityLoaded = true;
|
|
148
|
+
}
|
|
129
149
|
|
|
130
150
|
const path = new URL(request.url).pathname;
|
|
131
151
|
if (path === "/__recover") return this.handleRecover(request);
|
|
132
152
|
if (path === "/__schema") return this.handleSchema();
|
|
133
153
|
if (path === "/__admin/data") return this.handleAdminData(request);
|
|
154
|
+
if (path === "/__admin/tasks/drain") return this.handleDrain();
|
|
155
|
+
if (path === "/__admin/tasks/list") return this.handleTasksList(request);
|
|
134
156
|
|
|
135
157
|
const identity = this.identityOf(request);
|
|
136
158
|
|
|
@@ -148,7 +170,7 @@ export class PramenDOBase extends DurableObject<DoEnv> {
|
|
|
148
170
|
}
|
|
149
171
|
|
|
150
172
|
try {
|
|
151
|
-
const { result, kind, touched } = await dispatch(
|
|
173
|
+
const { result, kind, touched, enqueued } = await dispatch(
|
|
152
174
|
this.app.handlers,
|
|
153
175
|
this.app.schema,
|
|
154
176
|
this.driver,
|
|
@@ -160,6 +182,7 @@ export class PramenDOBase extends DurableObject<DoEnv> {
|
|
|
160
182
|
input,
|
|
161
183
|
);
|
|
162
184
|
if (kind === "mutation" && touched.length > 0) await this.broadcast(touched);
|
|
185
|
+
if (enqueued > 0) await this.armDrain();
|
|
163
186
|
return Response.json({ ok: true, result });
|
|
164
187
|
} catch (err) {
|
|
165
188
|
const { status, body } = toResponse(err);
|
|
@@ -167,6 +190,72 @@ export class PramenDOBase extends DurableObject<DoEnv> {
|
|
|
167
190
|
}
|
|
168
191
|
}
|
|
169
192
|
|
|
193
|
+
/** Arm the drain alarm soon after a mutation enqueued task(s). setAlarm replaces any
|
|
194
|
+
* pending alarm; a near-future time batches a burst of enqueues into one drain. */
|
|
195
|
+
private async armDrain(): Promise<void> {
|
|
196
|
+
await this.ctx.storage.setAlarm(Date.now() + 50);
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
/** A privileged, system-scoped context for running task handlers (outside a request).
|
|
200
|
+
* Task handlers get full db access + env (e.g. ctx.env.EMAIL) + their own ctx.tasks. */
|
|
201
|
+
private taskCtx(): HandlerContext {
|
|
202
|
+
const identity: Identity = { roles: ["admin"] };
|
|
203
|
+
const db = new Db(
|
|
204
|
+
this.driver,
|
|
205
|
+
{ acl: this.acl, identity, system: true, schema: this.app.schema, partition: this.partition },
|
|
206
|
+
this.app.schema,
|
|
207
|
+
);
|
|
208
|
+
return { db, kv: this.kv, files: this.filesFor(this.tenant), env: this.envBag, identity, tasks: tasksFacade(this.driver) };
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
/** Restore (tenant, partition) on a cold wake (e.g. an alarm with no request) so the
|
|
212
|
+
* task context is scoped correctly. No-op once loaded/persisted this instance. */
|
|
213
|
+
private async loadIdentity(): Promise<void> {
|
|
214
|
+
if (this.identityLoaded) return;
|
|
215
|
+
// _pramen_meta exists: an alarm only fires after a fetch armed it, and that fetch
|
|
216
|
+
// ran the boot migration which creates the table.
|
|
217
|
+
const rows = (await this.driver.exec(
|
|
218
|
+
`SELECT key, value FROM _pramen_meta WHERE key IN ('_pramen_tenant', '_pramen_partition')`,
|
|
219
|
+
[],
|
|
220
|
+
)) as { key: string; value: string }[];
|
|
221
|
+
for (const r of rows) {
|
|
222
|
+
if (r.key === "_pramen_tenant" && r.value) this.tenant = r.value;
|
|
223
|
+
if (r.key === "_pramen_partition" && r.value) this.partition = r.value;
|
|
224
|
+
}
|
|
225
|
+
this.identityLoaded = true;
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
/** Drain due tasks. Called by the alarm (DO path) and the /__admin/tasks/drain route
|
|
229
|
+
* (manual / cron). Returns the drain stats incl. `nextRunAt` for rescheduling. */
|
|
230
|
+
private async drainTasks(): Promise<Awaited<ReturnType<typeof drainOutbox>>> {
|
|
231
|
+
await ensureOutbox(this.driver); // idempotent — the table may predate this instance (cold alarm)
|
|
232
|
+
return drainOutbox(this.driver, bindTasks(this.app.tasks, this.taskCtx()), Date.now());
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
override async alarm(): Promise<void> {
|
|
236
|
+
await this.loadIdentity(); // cold wake: restore tenant/partition before building taskCtx
|
|
237
|
+
const { nextRunAt } = await this.drainTasks();
|
|
238
|
+
// Reschedule to the NEXT task's due time (a backed-off retry, or the next batch if
|
|
239
|
+
// the drain hit its limit) so a failed task can't stall waiting for a new enqueue.
|
|
240
|
+
if (nextRunAt != null) await this.ctx.storage.setAlarm(Math.max(nextRunAt, Date.now() + 250));
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
private async handleDrain(): Promise<Response> {
|
|
244
|
+
await this.loadIdentity();
|
|
245
|
+
const result = await this.drainTasks();
|
|
246
|
+
// Keep the alarm honest even when drained manually: ensure a backed-off retry wakes.
|
|
247
|
+
if (result.nextRunAt != null) await this.ctx.storage.setAlarm(Math.max(result.nextRunAt, Date.now() + 250));
|
|
248
|
+
return Response.json({ ok: true, result });
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
private async handleTasksList(request: Request): Promise<Response> {
|
|
252
|
+
await ensureOutbox(this.driver);
|
|
253
|
+
const url = new URL(request.url);
|
|
254
|
+
const status = url.searchParams.get("status") ?? undefined;
|
|
255
|
+
const limit = Number(url.searchParams.get("limit")) || undefined;
|
|
256
|
+
return Response.json({ ok: true, result: await listTasks(this.driver, { status, limit }) });
|
|
257
|
+
}
|
|
258
|
+
|
|
170
259
|
// --- Hibernatable WebSocket handlers ---
|
|
171
260
|
|
|
172
261
|
override async webSocketMessage(ws: WebSocket, raw: string | ArrayBuffer): Promise<void> {
|
|
@@ -240,9 +329,10 @@ export class PramenDOBase extends DurableObject<DoEnv> {
|
|
|
240
329
|
private async onCall(ws: WebSocket, id: string, name: string, input: unknown): Promise<void> {
|
|
241
330
|
const state = this.getState(ws);
|
|
242
331
|
try {
|
|
243
|
-
const { result, kind, touched } = await dispatch(this.app.handlers, this.app.schema, this.driver, this.kv, this.filesFor(state.tenant), this.envBag, this.ctxFor(state.identity, state.partition), name, input);
|
|
332
|
+
const { result, kind, touched, enqueued } = await dispatch(this.app.handlers, this.app.schema, this.driver, this.kv, this.filesFor(state.tenant), this.envBag, this.ctxFor(state.identity, state.partition), name, input);
|
|
244
333
|
this.send(ws, { type: "result", id, result });
|
|
245
334
|
if (kind === "mutation" && touched.length > 0) await this.broadcast(touched);
|
|
335
|
+
if (enqueued > 0) await this.armDrain();
|
|
246
336
|
} catch (err) {
|
|
247
337
|
this.send(ws, toWsError(id, err));
|
|
248
338
|
}
|
|
@@ -338,11 +428,13 @@ export class PramenDOBase extends DurableObject<DoEnv> {
|
|
|
338
428
|
value: string;
|
|
339
429
|
}[];
|
|
340
430
|
const tableRows = (await this.driver.exec(
|
|
341
|
-
`SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'
|
|
431
|
+
`SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'`,
|
|
342
432
|
[],
|
|
343
433
|
)) as { name: string }[];
|
|
344
434
|
const tables: Record<string, string[]> = {};
|
|
345
435
|
for (const { name } of tableRows) {
|
|
436
|
+
// Skip pramen's internal bookkeeping tables (_pramen_meta, _pramen_outbox, …).
|
|
437
|
+
if (name.toLowerCase().startsWith("_pramen") || name.toLowerCase().startsWith("__pramen")) continue;
|
|
346
438
|
tables[name] = ((await this.driver.exec(`PRAGMA table_info(${name})`, [])) as { name: string }[]).map((r) => r.name);
|
|
347
439
|
}
|
|
348
440
|
return Response.json({ ok: true, result: { hash: hashRow[0]?.value ?? null, tables } });
|
package/src/index.ts
CHANGED
|
@@ -26,7 +26,7 @@ export type {
|
|
|
26
26
|
// --- app + handlers ---
|
|
27
27
|
export { createApp } from "./sdk/app";
|
|
28
28
|
export { query, mutation } from "./sdk/handlers";
|
|
29
|
-
export type { Handler, HandlerContext, HandlerKind, HandlerMap, HandlerOpts } from "./sdk/handlers";
|
|
29
|
+
export type { Handler, HandlerContext, HandlerKind, HandlerMap, HandlerOpts, Tasks, TaskHandler, AppTaskMap } from "./sdk/handlers";
|
|
30
30
|
|
|
31
31
|
// --- ACL ---
|
|
32
32
|
export { $identity, $input, allow, deny, policy, resolve, role, isAllow, isDeny, isResolver, isIdentityMarker, isInputMarker } from "./sdk/acl";
|
package/src/pramen.ts
CHANGED
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
import { makeWorker, type Env } from "./worker";
|
|
16
16
|
import { pramenDO, type DoEnv } from "./durable-object";
|
|
17
17
|
import type { SchemaDef } from "./sdk/schema";
|
|
18
|
-
import type { HandlerMap } from "./sdk/handlers";
|
|
18
|
+
import type { AppTaskMap, HandlerMap } from "./sdk/handlers";
|
|
19
19
|
import type { Role } from "./sdk/acl";
|
|
20
20
|
|
|
21
21
|
/** Injected into a public route's handler — forward a privileged mutation into the
|
|
@@ -45,14 +45,21 @@ export interface PramenApp {
|
|
|
45
45
|
handlers: HandlerMap;
|
|
46
46
|
acl?: Role[];
|
|
47
47
|
routes?: PublicRoute[];
|
|
48
|
+
/** Deferred side-effect handlers keyed by `kind` — drained from the outbox after a
|
|
49
|
+
* mutation enqueues via `ctx.tasks.enqueue`. For notification email, webhooks, etc. */
|
|
50
|
+
tasks?: AppTaskMap;
|
|
48
51
|
}
|
|
49
52
|
|
|
50
53
|
export type { Env, DoEnv };
|
|
51
54
|
|
|
52
|
-
/** Build the deployable pair for an app.
|
|
55
|
+
/** Build the deployable pair for an app. `scheduled` is a Cron Trigger entry that
|
|
56
|
+
* drains the D1 outbox (the DO path self-drains via an alarm) — wire it only if you
|
|
57
|
+
* use the D1 store with deferred tasks. */
|
|
53
58
|
export function createPramen(app: PramenApp): {
|
|
54
59
|
fetch: (request: Request, env: Env) => Promise<Response>;
|
|
60
|
+
scheduled: (event: unknown, env: Env) => Promise<void>;
|
|
55
61
|
PramenDO: ReturnType<typeof pramenDO>;
|
|
56
62
|
} {
|
|
57
|
-
|
|
63
|
+
const worker = makeWorker(app);
|
|
64
|
+
return { fetch: worker.fetch, scheduled: worker.scheduled, PramenDO: pramenDO(app) };
|
|
58
65
|
}
|
package/src/runtime/dispatch.ts
CHANGED
|
@@ -11,17 +11,38 @@
|
|
|
11
11
|
import { Db } from "./db";
|
|
12
12
|
import { warmup, type AclContext } from "./acl";
|
|
13
13
|
import { BadRequest } from "./errors";
|
|
14
|
+
import { enqueueTask, type TaskMap } from "./outbox";
|
|
14
15
|
import type { Driver } from "./driver";
|
|
15
16
|
import type { Kv } from "./kv";
|
|
16
17
|
import type { Files } from "../sdk/files";
|
|
17
18
|
import type { ResolverDb } from "../sdk/acl";
|
|
18
19
|
import type { SchemaDef } from "../sdk/schema";
|
|
19
|
-
import type { HandlerContext, HandlerKind, HandlerMap } from "../sdk/handlers";
|
|
20
|
+
import type { AppTaskMap, HandlerContext, HandlerKind, HandlerMap, Tasks } from "../sdk/handlers";
|
|
20
21
|
|
|
21
22
|
export interface DispatchResult {
|
|
22
23
|
readonly result: unknown;
|
|
23
24
|
readonly kind: HandlerKind;
|
|
24
25
|
readonly touched: string[];
|
|
26
|
+
/** Number of tasks the handler enqueued — the DO uses this to arm its drain alarm. */
|
|
27
|
+
readonly enqueued: number;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/** The `ctx.tasks` facade over the outbox. `onEnqueue` lets the caller count enqueues
|
|
31
|
+
* so it can wake the drainer. */
|
|
32
|
+
export function tasksFacade(driver: Driver, onEnqueue?: () => void): Tasks {
|
|
33
|
+
return {
|
|
34
|
+
enqueue: async (opts) => {
|
|
35
|
+
await enqueueTask(driver, Date.now(), opts);
|
|
36
|
+
onEnqueue?.();
|
|
37
|
+
},
|
|
38
|
+
};
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/** Bind `app.tasks` (which take a ctx) into the ctx-free `TaskMap` the drainer calls. */
|
|
42
|
+
export function bindTasks(appTasks: AppTaskMap | undefined, ctx: HandlerContext): TaskMap {
|
|
43
|
+
const out: TaskMap = {};
|
|
44
|
+
for (const [kind, handler] of Object.entries(appTasks ?? {})) out[kind] = (payload, meta) => handler(ctx, payload, meta);
|
|
45
|
+
return out;
|
|
25
46
|
}
|
|
26
47
|
|
|
27
48
|
export async function dispatch(
|
|
@@ -54,12 +75,13 @@ export async function dispatch(
|
|
|
54
75
|
const resolved = await warmup(acl.acl, acl.identity, systemDb as unknown as ResolverDb);
|
|
55
76
|
|
|
56
77
|
const db = new Db(driver, { acl: acl.acl, identity: acl.identity, input: parsed, resolved, schema, partition: acl.partition }, schema);
|
|
57
|
-
|
|
78
|
+
let enqueued = 0;
|
|
79
|
+
const ctx: HandlerContext = { db, kv, files, env, identity: acl.identity, tasks: tasksFacade(driver, () => enqueued++) };
|
|
58
80
|
|
|
59
81
|
const result =
|
|
60
82
|
handler.kind === "query"
|
|
61
83
|
? await handler.run(ctx, parsed)
|
|
62
84
|
: await driver.transaction(async () => handler.run(ctx, parsed));
|
|
63
85
|
|
|
64
|
-
return { result, kind: handler.kind, touched: [...db.touched] };
|
|
86
|
+
return { result, kind: handler.kind, touched: [...db.touched], enqueued };
|
|
65
87
|
}
|
|
@@ -0,0 +1,204 @@
|
|
|
1
|
+
// Transactional outbox — the substrate-agnostic core of deferred side-effects
|
|
2
|
+
// ("tasks"), e.g. sending a notification email off the write path.
|
|
3
|
+
//
|
|
4
|
+
// A handler calls `ctx.tasks.enqueue({ kind, payload })`, which INSERTs a row into
|
|
5
|
+
// `_pramen_outbox` through the SAME Driver (and, for a mutation, the SAME transaction)
|
|
6
|
+
// as the data write — so the task and the data commit or roll back together (no
|
|
7
|
+
// dual-write window). A drainer later runs the app's task handler for that `kind`,
|
|
8
|
+
// with retry/backoff and a dead-letter terminal state.
|
|
9
|
+
//
|
|
10
|
+
// Everything here is written against the `Driver`/`Dialect` seam, so it runs
|
|
11
|
+
// identically on the DO's in-process SQLite AND on D1 (the Worker path). What differs
|
|
12
|
+
// is only the WAKE-UP: the DO self-drains via an alarm scheduled at the next due time;
|
|
13
|
+
// the D1/Worker path drains via a Cron Trigger or POST /admin/tasks/drain — same
|
|
14
|
+
// drainOutbox(), both paths.
|
|
15
|
+
//
|
|
16
|
+
// Delivery is at-least-once. The drain CLAIMS a batch atomically (status
|
|
17
|
+
// pending→processing) so concurrent drainers (the D1/Cron path) never process the same
|
|
18
|
+
// row twice; a crashed drainer's claim is reclaimed after STALE_MS. Handlers get the
|
|
19
|
+
// task `id` as an idempotency key so they can dedupe across the rare retry.
|
|
20
|
+
|
|
21
|
+
import type { Driver } from "./driver";
|
|
22
|
+
|
|
23
|
+
export const OUTBOX_TABLE = "_pramen_outbox";
|
|
24
|
+
|
|
25
|
+
const MAX_ATTEMPTS = 5;
|
|
26
|
+
/** A claimed ('processing') row whose claim is older than this is presumed abandoned
|
|
27
|
+
* (the drainer crashed) and is reclaimed. Must exceed the slowest task. */
|
|
28
|
+
const STALE_MS = 60_000;
|
|
29
|
+
/** Keep 'done' rows this long (a dedup window + debugging), then prune. */
|
|
30
|
+
const DONE_RETENTION_MS = 3_600_000;
|
|
31
|
+
|
|
32
|
+
/** Exponential backoff (ms) before the next attempt of a failed task. */
|
|
33
|
+
function backoffMs(attempts: number): number {
|
|
34
|
+
return Math.min(2 ** attempts * 1000, 5 * 60_000); // 2s, 4s, 8s, … capped at 5min
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
const enc = (driver: Driver, params: unknown[]): unknown[] => params.map((p) => driver.dialect.encode(p));
|
|
38
|
+
|
|
39
|
+
/** Create the outbox table if absent. Idempotent — run on DO boot (and lazily on the
|
|
40
|
+
* D1 path). Internal table (`_pramen_` prefix), never part of the user schema. */
|
|
41
|
+
export async function ensureOutbox(driver: Driver): Promise<void> {
|
|
42
|
+
const t = driver.dialect.id(OUTBOX_TABLE);
|
|
43
|
+
await driver.exec(
|
|
44
|
+
`CREATE TABLE IF NOT EXISTS ${t} (` +
|
|
45
|
+
`id TEXT PRIMARY KEY, kind TEXT NOT NULL, payload TEXT NOT NULL, ` +
|
|
46
|
+
`status TEXT NOT NULL DEFAULT 'pending', attempts INTEGER NOT NULL DEFAULT 0, ` +
|
|
47
|
+
`runAt INTEGER NOT NULL, createdAt INTEGER NOT NULL, claimedAt INTEGER, lastError TEXT)`,
|
|
48
|
+
[],
|
|
49
|
+
);
|
|
50
|
+
// Drain queries filter on (status, runAt); index keeps claim/scan cheap as it grows.
|
|
51
|
+
await driver.exec(`CREATE INDEX IF NOT EXISTS _pramen_outbox_due ON ${OUTBOX_TABLE} (status, runAt)`, []);
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
export interface EnqueueOpts {
|
|
55
|
+
kind: string;
|
|
56
|
+
payload?: unknown;
|
|
57
|
+
/** Delay before the task becomes due (ms from now). Default 0 (drain ASAP). */
|
|
58
|
+
delayMs?: number;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/** Insert one task row. Uses the driver directly, so inside a mutation it joins that
|
|
62
|
+
* mutation's transaction (atomic with the data write). `now` is stamped by the caller. */
|
|
63
|
+
export async function enqueueTask(driver: Driver, now: number, opts: EnqueueOpts): Promise<void> {
|
|
64
|
+
if (!opts || typeof opts.kind !== "string" || opts.kind.length === 0) {
|
|
65
|
+
throw new Error("ctx.tasks.enqueue: `kind` is required");
|
|
66
|
+
}
|
|
67
|
+
const d = driver.dialect;
|
|
68
|
+
const ph = (i: number) => d.placeholder(i);
|
|
69
|
+
await driver.exec(
|
|
70
|
+
`INSERT INTO ${d.id(OUTBOX_TABLE)} (id, kind, payload, status, attempts, runAt, createdAt) ` +
|
|
71
|
+
`VALUES (${ph(1)}, ${ph(2)}, ${ph(3)}, ${ph(4)}, ${ph(5)}, ${ph(6)}, ${ph(7)})`,
|
|
72
|
+
enc(driver, [
|
|
73
|
+
crypto.randomUUID(),
|
|
74
|
+
opts.kind,
|
|
75
|
+
JSON.stringify(opts.payload ?? null),
|
|
76
|
+
"pending",
|
|
77
|
+
0,
|
|
78
|
+
now + Math.max(0, Math.trunc(opts.delayMs ?? 0)),
|
|
79
|
+
now,
|
|
80
|
+
]),
|
|
81
|
+
);
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/** Idempotency metadata handed to a task handler. `id` is stable across retries — a
|
|
85
|
+
* handler can record it and skip a duplicate delivery (at-least-once). */
|
|
86
|
+
export interface TaskMeta {
|
|
87
|
+
id: string;
|
|
88
|
+
/** 1-based attempt number for this delivery. */
|
|
89
|
+
attempts: number;
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/** A task handler: runs the side effect for one `kind`. Throwing schedules a retry. */
|
|
93
|
+
export type TaskHandler = (payload: unknown, meta: TaskMeta) => void | Promise<void>;
|
|
94
|
+
export type TaskMap = Record<string, TaskHandler>;
|
|
95
|
+
|
|
96
|
+
export interface DrainResult {
|
|
97
|
+
processed: number;
|
|
98
|
+
succeeded: number;
|
|
99
|
+
failed: number;
|
|
100
|
+
/** Tasks still pending+due after this pass (a caller may loop to clear a backlog). */
|
|
101
|
+
remaining: number;
|
|
102
|
+
/** Epoch-ms of the earliest not-yet-run task (due or backed-off), or null if none —
|
|
103
|
+
* the DO schedules its next alarm here so a backed-off retry can't stall. */
|
|
104
|
+
nextRunAt: number | null;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/** Run every due task once (claimed batch, up to `limit`). On success mark it done; on
|
|
108
|
+
* throw, bump attempts and back off, or dead-letter ('failed') past MAX_ATTEMPTS.
|
|
109
|
+
*
|
|
110
|
+
* Substrate-agnostic and concurrency-safe: the claim UPDATE (pending→processing) is
|
|
111
|
+
* atomic, so two drainers (the D1/Cron path) get disjoint batches; a crashed drainer's
|
|
112
|
+
* claim is reclaimed after STALE_MS. The DO path is single-writer so claims never
|
|
113
|
+
* contend, but the same code runs there too. */
|
|
114
|
+
export async function drainOutbox(driver: Driver, tasks: TaskMap, now: number, limit = 50): Promise<DrainResult> {
|
|
115
|
+
const d = driver.dialect;
|
|
116
|
+
const ph = (i: number) => d.placeholder(i);
|
|
117
|
+
|
|
118
|
+
// Prune long-since-delivered rows so the table stays bounded.
|
|
119
|
+
await driver.exec(
|
|
120
|
+
`DELETE FROM ${d.id(OUTBOX_TABLE)} WHERE status = ${ph(1)} AND createdAt < ${ph(2)}`,
|
|
121
|
+
enc(driver, ["done", now - DONE_RETENTION_MS]),
|
|
122
|
+
);
|
|
123
|
+
|
|
124
|
+
// Atomically claim a due batch: fresh 'pending', plus 'processing' rows whose claim
|
|
125
|
+
// is stale (the drainer crashed). RETURNING gives us exactly our claimed rows, so a
|
|
126
|
+
// concurrent drainer (writes serialize) claims a disjoint set.
|
|
127
|
+
const staleBefore = now - STALE_MS;
|
|
128
|
+
const claimed = await driver.exec(
|
|
129
|
+
`UPDATE ${d.id(OUTBOX_TABLE)} SET status = ${ph(1)}, claimedAt = ${ph(2)} WHERE id IN (` +
|
|
130
|
+
`SELECT id FROM ${d.id(OUTBOX_TABLE)} ` +
|
|
131
|
+
`WHERE (status = ${ph(3)} OR (status = ${ph(4)} AND claimedAt <= ${ph(5)})) AND runAt <= ${ph(6)} ` +
|
|
132
|
+
`ORDER BY createdAt LIMIT ${Math.max(1, Math.trunc(limit))}) ` +
|
|
133
|
+
`RETURNING id, kind, payload, attempts`,
|
|
134
|
+
enc(driver, ["processing", now, "pending", "processing", staleBefore, now]),
|
|
135
|
+
);
|
|
136
|
+
|
|
137
|
+
let succeeded = 0;
|
|
138
|
+
let failed = 0;
|
|
139
|
+
for (const row of claimed) {
|
|
140
|
+
const id = String(row.id);
|
|
141
|
+
const kind = String(row.kind);
|
|
142
|
+
const attempts = Number(row.attempts) + 1;
|
|
143
|
+
const handler = tasks[kind];
|
|
144
|
+
try {
|
|
145
|
+
if (!handler) throw new Error(`no task handler registered for kind ${JSON.stringify(kind)}`);
|
|
146
|
+
await handler(JSON.parse(String(row.payload)), { id, attempts });
|
|
147
|
+
await driver.exec(
|
|
148
|
+
`UPDATE ${d.id(OUTBOX_TABLE)} SET status = ${ph(1)}, attempts = ${ph(2)}, claimedAt = NULL WHERE id = ${ph(3)}`,
|
|
149
|
+
enc(driver, ["done", attempts, id]),
|
|
150
|
+
);
|
|
151
|
+
succeeded++;
|
|
152
|
+
} catch (e) {
|
|
153
|
+
const dead = attempts >= MAX_ATTEMPTS;
|
|
154
|
+
const msg = e instanceof Error ? e.message : String(e);
|
|
155
|
+
await driver.exec(
|
|
156
|
+
`UPDATE ${d.id(OUTBOX_TABLE)} SET status = ${ph(1)}, attempts = ${ph(2)}, runAt = ${ph(3)}, claimedAt = NULL, lastError = ${ph(4)} WHERE id = ${ph(5)}`,
|
|
157
|
+
enc(driver, [dead ? "failed" : "pending", attempts, now + backoffMs(attempts), msg.slice(0, 500), id]),
|
|
158
|
+
);
|
|
159
|
+
failed++;
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
// remaining = pending AND due now; nextRunAt = the earliest pending runAt (any), so
|
|
164
|
+
// the DO can schedule its alarm exactly when the next task — including a backed-off
|
|
165
|
+
// retry — becomes due.
|
|
166
|
+
const stats = await driver.exec(
|
|
167
|
+
`SELECT ` +
|
|
168
|
+
`(SELECT COUNT(*) FROM ${d.id(OUTBOX_TABLE)} WHERE status = ${ph(1)} AND runAt <= ${ph(2)}) AS due, ` +
|
|
169
|
+
`(SELECT MIN(runAt) FROM ${d.id(OUTBOX_TABLE)} WHERE status = ${ph(3)}) AS nextRunAt`,
|
|
170
|
+
enc(driver, ["pending", now, "pending"]),
|
|
171
|
+
);
|
|
172
|
+
const nextRaw = stats[0]?.nextRunAt;
|
|
173
|
+
return {
|
|
174
|
+
processed: claimed.length,
|
|
175
|
+
succeeded,
|
|
176
|
+
failed,
|
|
177
|
+
remaining: Number(stats[0]?.due ?? 0),
|
|
178
|
+
nextRunAt: nextRaw == null ? null : Number(nextRaw),
|
|
179
|
+
};
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
export interface TaskRow {
|
|
183
|
+
id: string;
|
|
184
|
+
kind: string;
|
|
185
|
+
status: string;
|
|
186
|
+
attempts: number;
|
|
187
|
+
runAt: number;
|
|
188
|
+
createdAt: number;
|
|
189
|
+
lastError: string | null;
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
/** List outbox rows for admin visibility (e.g. inspect dead-lettered tasks). Newest
|
|
193
|
+
* first; optionally filter by `status` (e.g. "failed"). Excludes the payload. */
|
|
194
|
+
export async function listTasks(driver: Driver, opts: { status?: string; limit?: number } = {}): Promise<TaskRow[]> {
|
|
195
|
+
const d = driver.dialect;
|
|
196
|
+
const limit = Math.min(Math.max(Math.trunc(opts.limit ?? 100) || 100, 1), 500);
|
|
197
|
+
const where = opts.status ? `WHERE status = ${d.placeholder(1)} ` : "";
|
|
198
|
+
const rows = await driver.exec(
|
|
199
|
+
`SELECT id, kind, status, attempts, runAt, createdAt, lastError FROM ${d.id(OUTBOX_TABLE)} ` +
|
|
200
|
+
`${where}ORDER BY createdAt DESC LIMIT ${limit}`,
|
|
201
|
+
opts.status ? enc(driver, [opts.status]) : [],
|
|
202
|
+
);
|
|
203
|
+
return rows as unknown as TaskRow[];
|
|
204
|
+
}
|
package/src/sdk/handlers.ts
CHANGED
|
@@ -25,8 +25,35 @@ export interface HandlerContext<S extends SchemaDef = SchemaDef> {
|
|
|
25
25
|
readonly env: Readonly<Record<string, unknown>>;
|
|
26
26
|
/** Resolved identity for this request (null = anonymous). */
|
|
27
27
|
readonly identity: Identity | null;
|
|
28
|
+
/** Deferred side-effects (a transactional outbox). `tasks.enqueue` persists a task
|
|
29
|
+
* row in the SAME transaction as a mutation (atomic with the data write); a drainer
|
|
30
|
+
* runs the matching `app.tasks` handler after commit, off the write path, with
|
|
31
|
+
* retry. For notification email, webhooks, etc. — see `app.tasks`. */
|
|
32
|
+
readonly tasks: Tasks;
|
|
28
33
|
}
|
|
29
34
|
|
|
35
|
+
/** The deferred-side-effects facade handed to handlers as `ctx.tasks`. */
|
|
36
|
+
export interface Tasks {
|
|
37
|
+
/** Enqueue a task to run after commit. `kind` selects the `app.tasks` handler;
|
|
38
|
+
* `payload` is JSON-serialized. `delayMs` defers when it becomes due. */
|
|
39
|
+
enqueue(opts: { kind: string; payload?: unknown; delayMs?: number }): Promise<void>;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/** Idempotency metadata for a task delivery. `id` is stable across retries — record it
|
|
43
|
+
* to dedupe the rare duplicate (delivery is at-least-once). `attempts` is 1-based. */
|
|
44
|
+
export interface TaskMeta {
|
|
45
|
+
id: string;
|
|
46
|
+
attempts: number;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/** An app task handler — runs a deferred side effect for one `kind` (e.g. send an
|
|
50
|
+
* email via `ctx.env.EMAIL`). Throwing schedules a retry (capped, then dead-lettered).
|
|
51
|
+
* Receives a privileged, system-scoped context plus the task's idempotency `meta`.
|
|
52
|
+
* Register handlers in `app.tasks`. */
|
|
53
|
+
export type TaskHandler = (ctx: HandlerContext, payload: unknown, meta: TaskMeta) => void | Promise<void>;
|
|
54
|
+
/** Map of `kind` → handler. Set as `app.tasks`; drained by the DO alarm / a Cron. */
|
|
55
|
+
export type AppTaskMap = Record<string, TaskHandler>;
|
|
56
|
+
|
|
30
57
|
export type HandlerKind = "query" | "mutation";
|
|
31
58
|
|
|
32
59
|
export interface Handler<I = unknown, O = unknown> {
|