@pramen/server 0.0.61 → 0.0.63
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/durable-object.js +1 -1
- package/dist/pramen.d.ts +1 -1
- package/dist/runtime/boot.d.ts +37 -0
- package/dist/runtime/boot.js +157 -0
- package/dist/runtime/data-migrations.js +1 -1
- package/dist/runtime/mail.d.ts +3 -5
- package/dist/runtime/mail.js +6 -9
- package/dist/sdk/handlers.d.ts +6 -0
- package/dist/worker.d.ts +8 -1
- package/dist/worker.js +53 -31
- package/package.json +1 -1
- package/src/durable-object.ts +1 -1
- package/src/pramen.ts +1 -1
- package/src/runtime/boot.ts +184 -0
- package/src/runtime/data-migrations.ts +1 -1
- package/src/runtime/mail.ts +6 -9
- package/src/sdk/handlers.ts +6 -0
- package/src/worker.ts +62 -31
package/dist/durable-object.js
CHANGED
|
@@ -144,7 +144,7 @@ export class PramenDOBase extends DurableObject {
|
|
|
144
144
|
const db = new Db(this.driver, { acl: this.acl, identity: { roles: ["admin"] }, system: true, partition: this.partition, schema: this.app.schema, suppressTriggers: true }, this.app.schema);
|
|
145
145
|
for (const fn of fns) {
|
|
146
146
|
try {
|
|
147
|
-
await this.driver.transaction(() => Promise.resolve(fn({ db, driver: this.driver, schema: this.app.schema, partition: this.partition })));
|
|
147
|
+
await this.driver.transaction(() => Promise.resolve(fn({ db, driver: this.driver, schema: this.app.schema, partition: this.partition, env: this.envBag })));
|
|
148
148
|
}
|
|
149
149
|
catch (e) {
|
|
150
150
|
console.error(`[pramen] bootstrap failed (partition=${this.partition}):`, e);
|
package/dist/pramen.d.ts
CHANGED
|
@@ -60,7 +60,7 @@ export type { Env, DoEnv };
|
|
|
60
60
|
* use the D1 store with deferred tasks. */
|
|
61
61
|
export declare function createPramen(app: PramenApp): {
|
|
62
62
|
fetch: (request: Request, env: Env, ctx: ExecutionContext) => Promise<Response>;
|
|
63
|
-
scheduled: (event: unknown, env: Env) => Promise<void>;
|
|
63
|
+
scheduled: (event: unknown, env: Env, ctx: ExecutionContext) => Promise<void>;
|
|
64
64
|
queue: (batch: QueueBatch, env: Env) => Promise<void>;
|
|
65
65
|
PramenDO: ReturnType<typeof pramenDO>;
|
|
66
66
|
};
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import type { Driver } from "./driver";
|
|
2
|
+
export interface SharedBootOpts {
|
|
3
|
+
/** How long a still-unsettled boot may go without progress before it counts as orphaned. */
|
|
4
|
+
staleMs: number;
|
|
5
|
+
/** Clock, injectable for tests. */
|
|
6
|
+
now?: () => number;
|
|
7
|
+
/** Called when a boot is judged orphaned and replaced, with how long it had been idle. */
|
|
8
|
+
onOrphaned?: (idleMs: number) => void;
|
|
9
|
+
}
|
|
10
|
+
/** The boot body. Call `progress()` whenever the boot does observable work — see
|
|
11
|
+
* `observeDriver` for the standard way to get that for free. */
|
|
12
|
+
export type BootFn = (progress: () => void) => Promise<void>;
|
|
13
|
+
export declare class SharedBoot {
|
|
14
|
+
private readonly opts;
|
|
15
|
+
private current;
|
|
16
|
+
private readonly now;
|
|
17
|
+
constructor(opts: SharedBootOpts);
|
|
18
|
+
/** Await the shared boot, starting one if none is live (or the live one looks orphaned).
|
|
19
|
+
*
|
|
20
|
+
* `run` is only invoked when THIS call has to start a boot. `keepAlive` is the calling
|
|
21
|
+
* invocation's lifetime extender (`ctx.waitUntil`); it is handed the boot's promise so
|
|
22
|
+
* the boot survives the caller disconnecting. A rejected boot rejects every awaiter and
|
|
23
|
+
* clears the memo, so the next call retries — fail closed, as before. */
|
|
24
|
+
ensure(run: BootFn, keepAlive?: (p: Promise<unknown>) => void): Promise<void>;
|
|
25
|
+
/** Whether a boot has completed in this isolate — for diagnostics only. */
|
|
26
|
+
isDone(): boolean;
|
|
27
|
+
private isStale;
|
|
28
|
+
private start;
|
|
29
|
+
/** Resolves `true` when the boot settles (rejects if it rejected), `false` once it has
|
|
30
|
+
* gone `staleMs` without progress — the timer re-arms from the latest progress instant,
|
|
31
|
+
* so a boot that keeps working is never reported stale. */
|
|
32
|
+
private settledBeforeStale;
|
|
33
|
+
}
|
|
34
|
+
/** Wrap a Driver so every statement it runs reports progress — the boot's liveness signal.
|
|
35
|
+
* `batch` is forwarded only when the wrapped driver has it, because the migrator decides
|
|
36
|
+
* between an atomic batch and sequential exec by its presence. */
|
|
37
|
+
export declare function observeDriver(driver: Driver, progress: () => void): Driver;
|
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
// SharedBoot — a once-per-isolate boot that many invocations await, made safe to share.
|
|
2
|
+
//
|
|
3
|
+
// The D1 store boots in the Worker: migrate → data migrations → outbox table → bootstrap,
|
|
4
|
+
// run by whichever request arrives first in a fresh isolate and memoized as ONE promise the
|
|
5
|
+
// rest await (running it per request would re-diff the schema on every call). That memo
|
|
6
|
+
// is the bug behind GitHub #51, and the reason is a Workers runtime rule, not pramen code:
|
|
7
|
+
// async work belongs to the INVOCATION that started it. When that invocation ends — the
|
|
8
|
+
// caller gives up and disconnects, which is exactly what a proxy with a 15 s ceiling does
|
|
9
|
+
// to the legitimately slow first request after a deploy carrying a table rebuild — its
|
|
10
|
+
// pending I/O is canceled, and a promise chained on canceled I/O never settles. Not
|
|
11
|
+
// rejected: never settles. So the `.catch` that was meant to clear the memo never runs,
|
|
12
|
+
// and every later fetch in that isolate awaits a promise that cannot resolve, for as long
|
|
13
|
+
// as its own caller allows, at 0 ms CPU. Crons stayed healthy in the report because they
|
|
14
|
+
// landed in a different isolate; the ~54 % was the share of traffic routed to the wedged
|
|
15
|
+
// one; a redeploy "fixed" it by discarding every isolate.
|
|
16
|
+
//
|
|
17
|
+
// Two defenses, and both are needed:
|
|
18
|
+
//
|
|
19
|
+
// 1. The STARTER extends its invocation over the boot with `ctx.waitUntil`, so the client
|
|
20
|
+
// disconnecting no longer cancels the boot's I/O. That closes the reported case. It is
|
|
21
|
+
// not sufficient on its own: waitUntil is capped (30 s after the response or the
|
|
22
|
+
// disconnect), and there is no rule that says a promise's owner is the only way it can
|
|
23
|
+
// be orphaned.
|
|
24
|
+
// 2. AWAITERS never trust a shared boot unconditionally. The boot reports PROGRESS (every
|
|
25
|
+
// statement the driver runs), and a boot that has made none for `staleMs` while still
|
|
26
|
+
// unsettled is treated as orphaned: the next awaiter — arriving, or one already waiting
|
|
27
|
+
// — starts a fresh boot in its own invocation and awaits that instead. An isolate can
|
|
28
|
+
// therefore be wedged for at most `staleMs`, never for its lifetime. Every step of the
|
|
29
|
+
// boot tolerates a second runner (that is the cross-isolate reality on D1 already: the
|
|
30
|
+
// migration ledger is lease-claimed, migrate is a diff, bootstrap is an upsert), so a
|
|
31
|
+
// false positive costs a redundant boot, not correctness — which is why the threshold is
|
|
32
|
+
// measured from the last STATEMENT and not from the boot's start: a boot that is slow
|
|
33
|
+
// but alive keeps making progress.
|
|
34
|
+
//
|
|
35
|
+
// Platform-agnostic on purpose (no `cloudflare:workers`): `keepAlive` is whatever the host
|
|
36
|
+
// offers to extend an invocation, and `now` is injectable for tests.
|
|
37
|
+
export class SharedBoot {
|
|
38
|
+
opts;
|
|
39
|
+
current;
|
|
40
|
+
now;
|
|
41
|
+
constructor(opts) {
|
|
42
|
+
this.opts = opts;
|
|
43
|
+
this.now = opts.now ?? Date.now;
|
|
44
|
+
}
|
|
45
|
+
/** Await the shared boot, starting one if none is live (or the live one looks orphaned).
|
|
46
|
+
*
|
|
47
|
+
* `run` is only invoked when THIS call has to start a boot. `keepAlive` is the calling
|
|
48
|
+
* invocation's lifetime extender (`ctx.waitUntil`); it is handed the boot's promise so
|
|
49
|
+
* the boot survives the caller disconnecting. A rejected boot rejects every awaiter and
|
|
50
|
+
* clears the memo, so the next call retries — fail closed, as before. */
|
|
51
|
+
async ensure(run, keepAlive) {
|
|
52
|
+
let boot = this.current;
|
|
53
|
+
// The STARTER awaits its own boot outright: the boot runs in its invocation, so if
|
|
54
|
+
// the starter is alive to watch, so is the boot — and if the invocation is canceled,
|
|
55
|
+
// this await dies with it. A starter that also watched would restart its own
|
|
56
|
+
// (dead-for-everyone-else) boot every window, endlessly.
|
|
57
|
+
if (!boot || this.isStale(boot))
|
|
58
|
+
return this.start(run, keepAlive, boot).promise;
|
|
59
|
+
for (;;) {
|
|
60
|
+
if (boot.done)
|
|
61
|
+
return;
|
|
62
|
+
// Wait, but wake when the boot WOULD count as stale, and re-check: progress since
|
|
63
|
+
// means it is alive and we keep waiting; none means it is orphaned.
|
|
64
|
+
if (await this.settledBeforeStale(boot))
|
|
65
|
+
return;
|
|
66
|
+
// Someone else may already have replaced it — join theirs rather than start a third.
|
|
67
|
+
const live = this.current;
|
|
68
|
+
if (live && live !== boot && !this.isStale(live)) {
|
|
69
|
+
boot = live;
|
|
70
|
+
continue;
|
|
71
|
+
}
|
|
72
|
+
return this.start(run, keepAlive, boot).promise;
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
/** Whether a boot has completed in this isolate — for diagnostics only. */
|
|
76
|
+
isDone() {
|
|
77
|
+
return this.current?.done === true;
|
|
78
|
+
}
|
|
79
|
+
isStale(boot) {
|
|
80
|
+
return !boot.done && this.now() - boot.lastProgressAt >= this.opts.staleMs;
|
|
81
|
+
}
|
|
82
|
+
start(run, keepAlive, replacing) {
|
|
83
|
+
if (replacing)
|
|
84
|
+
this.opts.onOrphaned?.(this.now() - replacing.lastProgressAt);
|
|
85
|
+
const record = { promise: Promise.resolve(), lastProgressAt: this.now(), done: false };
|
|
86
|
+
const progress = () => {
|
|
87
|
+
record.lastProgressAt = this.now();
|
|
88
|
+
};
|
|
89
|
+
// Through a resolved promise so a synchronous throw in `run` is a rejection, not an
|
|
90
|
+
// exception escaping `ensure` with the memo half-installed.
|
|
91
|
+
record.promise = Promise.resolve()
|
|
92
|
+
.then(() => run(progress))
|
|
93
|
+
.then(() => {
|
|
94
|
+
record.done = true;
|
|
95
|
+
}, (e) => {
|
|
96
|
+
// Only forget the memo if it is still THIS boot: a replacement started meanwhile
|
|
97
|
+
// must not be discarded because its predecessor finally failed.
|
|
98
|
+
if (this.current === record)
|
|
99
|
+
this.current = undefined;
|
|
100
|
+
throw e;
|
|
101
|
+
});
|
|
102
|
+
// Every awaiter attaches its own handler; this one just keeps the host from reporting
|
|
103
|
+
// an unhandled rejection when a boot fails with nobody awaiting the stored promise.
|
|
104
|
+
record.promise.catch(() => { });
|
|
105
|
+
keepAlive?.(record.promise.catch(() => { }));
|
|
106
|
+
this.current = record;
|
|
107
|
+
return record;
|
|
108
|
+
}
|
|
109
|
+
/** Resolves `true` when the boot settles (rejects if it rejected), `false` once it has
|
|
110
|
+
* gone `staleMs` without progress — the timer re-arms from the latest progress instant,
|
|
111
|
+
* so a boot that keeps working is never reported stale. */
|
|
112
|
+
settledBeforeStale(boot) {
|
|
113
|
+
return new Promise((resolve, reject) => {
|
|
114
|
+
let timer;
|
|
115
|
+
const arm = () => {
|
|
116
|
+
const wait = Math.max(0, boot.lastProgressAt + this.opts.staleMs - this.now());
|
|
117
|
+
timer = setTimeout(() => (this.isStale(boot) ? resolve(false) : arm()), wait);
|
|
118
|
+
};
|
|
119
|
+
const disarm = () => {
|
|
120
|
+
if (timer !== undefined)
|
|
121
|
+
clearTimeout(timer);
|
|
122
|
+
};
|
|
123
|
+
arm();
|
|
124
|
+
boot.promise.then(() => {
|
|
125
|
+
disarm();
|
|
126
|
+
resolve(true);
|
|
127
|
+
}, (e) => {
|
|
128
|
+
disarm();
|
|
129
|
+
reject(e);
|
|
130
|
+
});
|
|
131
|
+
});
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
/** Wrap a Driver so every statement it runs reports progress — the boot's liveness signal.
|
|
135
|
+
* `batch` is forwarded only when the wrapped driver has it, because the migrator decides
|
|
136
|
+
* between an atomic batch and sequential exec by its presence. */
|
|
137
|
+
export function observeDriver(driver, progress) {
|
|
138
|
+
const observed = {
|
|
139
|
+
dialect: driver.dialect,
|
|
140
|
+
exec: async (sql, params) => {
|
|
141
|
+
progress();
|
|
142
|
+
const rows = await driver.exec(sql, params);
|
|
143
|
+
progress();
|
|
144
|
+
return rows;
|
|
145
|
+
},
|
|
146
|
+
transaction: (fn) => driver.transaction(fn),
|
|
147
|
+
};
|
|
148
|
+
if (driver.batch) {
|
|
149
|
+
const batch = driver.batch.bind(driver);
|
|
150
|
+
observed.batch = async (statements) => {
|
|
151
|
+
progress();
|
|
152
|
+
await batch(statements);
|
|
153
|
+
progress();
|
|
154
|
+
};
|
|
155
|
+
}
|
|
156
|
+
return observed;
|
|
157
|
+
}
|
|
@@ -23,7 +23,7 @@
|
|
|
23
23
|
// that inserts a pending row, or steals one whose lease has expired, or does nothing — and
|
|
24
24
|
// the migration runs only if that statement won the row. On the DO this is merely bookkeeping
|
|
25
25
|
// order inside one transaction. On D1 it is what makes the once-only contract hold at all:
|
|
26
|
-
// there is no single writer and no interactive transaction,
|
|
26
|
+
// there is no single writer and no interactive transaction, the boot memo is per-isolate, so two
|
|
27
27
|
// cold isolates racing a `SET n = n * 2` backfill would each read an empty ledger, each run
|
|
28
28
|
// it, and quadruple the data. The conflicting upsert is the lock.
|
|
29
29
|
//
|
package/dist/runtime/mail.d.ts
CHANGED
|
@@ -49,11 +49,9 @@ export declare class CloudflareEmailAdapter implements MailAdapter {
|
|
|
49
49
|
}
|
|
50
50
|
/** Mailgun — an HTTP transport, for when Cloudflare Email Sending cannot be used.
|
|
51
51
|
*
|
|
52
|
-
* Worth the key for one reason: Cloudflare will only send from a domain that is a zone
|
|
53
|
-
*
|
|
54
|
-
*
|
|
55
|
-
* address"). That is workable for a handful of operators and hopeless for real users.
|
|
56
|
-
* Mailgun asks the domain be verified once, then delivers to anyone.
|
|
52
|
+
* Worth the key for one reason: Cloudflare will only send from a domain that is a zone in
|
|
53
|
+
* the same account. Mailgun asks the domain be verified once with Mailgun instead, so the
|
|
54
|
+
* sender need not be a domain this account owns.
|
|
57
55
|
*
|
|
58
56
|
* A non-2xx THROWS, deliberately: `ctx.mail.send` is normally called from a task, and a
|
|
59
57
|
* throw is what makes the outbox retry and then dead-letter visibly. Swallowing the
|
package/dist/runtime/mail.js
CHANGED
|
@@ -6,10 +6,9 @@
|
|
|
6
6
|
// await ctx.mail.send({ to: "u@x.com", subject: "Welcome", text: "…" });
|
|
7
7
|
//
|
|
8
8
|
// Two real transports. Cloudflare Email Sending (the `send_email`/`EMAIL` binding) needs
|
|
9
|
-
// no API keys, but it can only send FROM a domain that is a zone in the same account
|
|
10
|
-
//
|
|
11
|
-
//
|
|
12
|
-
// API, any recipient, at the cost of a key. Configure it and it wins.
|
|
9
|
+
// no API keys, but it can only send FROM a domain that is a zone in the same account.
|
|
10
|
+
// Mailgun is the way out of that: an HTTP API, a domain verified once with Mailgun
|
|
11
|
+
// rather than owned by the account, at the cost of a key. Configure it and it wins.
|
|
13
12
|
//
|
|
14
13
|
// With neither configured (local/dev), mail is captured instead of sent — to KV (so an
|
|
15
14
|
// e2e/dashboard can read the "inbox") or in-memory — so handlers work unchanged
|
|
@@ -56,11 +55,9 @@ export class CloudflareEmailAdapter {
|
|
|
56
55
|
}
|
|
57
56
|
/** Mailgun — an HTTP transport, for when Cloudflare Email Sending cannot be used.
|
|
58
57
|
*
|
|
59
|
-
* Worth the key for one reason: Cloudflare will only send from a domain that is a zone
|
|
60
|
-
*
|
|
61
|
-
*
|
|
62
|
-
* address"). That is workable for a handful of operators and hopeless for real users.
|
|
63
|
-
* Mailgun asks the domain be verified once, then delivers to anyone.
|
|
58
|
+
* Worth the key for one reason: Cloudflare will only send from a domain that is a zone in
|
|
59
|
+
* the same account. Mailgun asks the domain be verified once with Mailgun instead, so the
|
|
60
|
+
* sender need not be a domain this account owns.
|
|
64
61
|
*
|
|
65
62
|
* A non-2xx THROWS, deliberately: `ctx.mail.send` is normally called from a task, and a
|
|
66
63
|
* throw is what makes the outbox retry and then dead-letter visibly. Swallowing the
|
package/dist/sdk/handlers.d.ts
CHANGED
|
@@ -67,6 +67,12 @@ export interface BootstrapContext<S extends SchemaDef = SchemaDef> {
|
|
|
67
67
|
/** The partition being booted. On the DO path bootstrap runs ONLY for the default
|
|
68
68
|
* partition (reference data lives there); on the D1 path it is always the default. */
|
|
69
69
|
readonly partition: string;
|
|
70
|
+
/** The Worker/DO environment — bindings, vars and secrets, as a handler's `ctx.env` sees
|
|
71
|
+
* it. Reference data is not always unconditional: a seed that belongs in local dev and
|
|
72
|
+
* nowhere else (a demo account with a known password) has to be able to READ the flag that
|
|
73
|
+
* says which it is, and a boot-time reconciler has no request to carry one. Without this
|
|
74
|
+
* the only gates available were "always" and "never". */
|
|
75
|
+
readonly env: EnvBag;
|
|
70
76
|
}
|
|
71
77
|
/** An idempotent reconcile run once after `migrate()` on each boot (a DO's first fetch, or a
|
|
72
78
|
* Worker/D1 isolate init). It MUST be safe to run repeatedly — upsert by a stable key, never
|
package/dist/worker.d.ts
CHANGED
|
@@ -60,6 +60,13 @@ export declare function useD1Store(opts: {
|
|
|
60
60
|
isLive: boolean;
|
|
61
61
|
defaultStore: string | undefined;
|
|
62
62
|
}): boolean;
|
|
63
|
+
/** How long the D1 boot may go without completing a statement before an awaiter treats it
|
|
64
|
+
* as orphaned and starts its own (GitHub #51). 30 s is the platform's `waitUntil` cap after
|
|
65
|
+
* a response or a disconnect — the point past which a boot's starter can no longer be
|
|
66
|
+
* keeping it alive — and comfortably above any single D1 statement a bounded migration
|
|
67
|
+
* should issue. Measured from the last STATEMENT, not from the boot's start, so a slow but
|
|
68
|
+
* live boot (a table rebuild followed by a backfill) is never mistaken for a dead one. */
|
|
69
|
+
export declare const D1_BOOT_STALE_MS = 30000;
|
|
63
70
|
/** Should we warn that no Cron trigger seems to be wired?
|
|
64
71
|
*
|
|
65
72
|
* The DO store self-drains via an alarm; the D1 store has none, so a DELAYED task — a
|
|
@@ -91,6 +98,6 @@ export declare function callPrivileged(env: Env, opts: {
|
|
|
91
98
|
* compiled-ACL + one-time migration) is per-app, held in this closure. */
|
|
92
99
|
export declare function makeWorker(app: PramenApp): {
|
|
93
100
|
fetch(request: Request, env: Env, ctx: ExecutionContext): Promise<Response>;
|
|
94
|
-
scheduled(_event: unknown, env: Env): Promise<void>;
|
|
101
|
+
scheduled(_event: unknown, env: Env, ctx: ExecutionContext): Promise<void>;
|
|
95
102
|
queue(batch: QueueBatch, env: Env): Promise<void>;
|
|
96
103
|
};
|
package/dist/worker.js
CHANGED
|
@@ -11,6 +11,7 @@ import { createMail } from "./runtime/mail";
|
|
|
11
11
|
import { createQueue } from "./runtime/queue";
|
|
12
12
|
import { dispatchQueueBatch } from "./runtime/queue-consumer";
|
|
13
13
|
import { migrate } from "./runtime/migrate";
|
|
14
|
+
import { SharedBoot, observeDriver } from "./runtime/boot";
|
|
14
15
|
import { compileAcl } from "./runtime/acl";
|
|
15
16
|
import { Db } from "./runtime/db";
|
|
16
17
|
import { D1Driver } from "./runtime/driver";
|
|
@@ -44,6 +45,13 @@ export function useD1Store(opts) {
|
|
|
44
45
|
return false;
|
|
45
46
|
return opts.defaultStore === "d1";
|
|
46
47
|
}
|
|
48
|
+
/** How long the D1 boot may go without completing a statement before an awaiter treats it
|
|
49
|
+
* as orphaned and starts its own (GitHub #51). 30 s is the platform's `waitUntil` cap after
|
|
50
|
+
* a response or a disconnect — the point past which a boot's starter can no longer be
|
|
51
|
+
* keeping it alive — and comfortably above any single D1 statement a bounded migration
|
|
52
|
+
* should issue. Measured from the last STATEMENT, not from the boot's start, so a slow but
|
|
53
|
+
* live boot (a table rebuild followed by a backfill) is never mistaken for a dead one. */
|
|
54
|
+
export const D1_BOOT_STALE_MS = 30_000;
|
|
47
55
|
/** Should we warn that no Cron trigger seems to be wired?
|
|
48
56
|
*
|
|
49
57
|
* The DO store self-drains via an alarm; the D1 store has none, so a DELAYED task — a
|
|
@@ -156,14 +164,14 @@ export function makeWorker(app) {
|
|
|
156
164
|
// runBootstrap(), run once per isolate after migration. D1 is a single shared store (no
|
|
157
165
|
// partition split), so every reconciler runs under the default partition. SYSTEM-scoped
|
|
158
166
|
// Db (ACL bypassed), triggers suppressed; a failing reconciler is logged, never fatal.
|
|
159
|
-
const runBootstrapD1 = async (driver) => {
|
|
167
|
+
const runBootstrapD1 = async (driver, env) => {
|
|
160
168
|
const fns = app.bootstrap;
|
|
161
169
|
if (!fns?.length)
|
|
162
170
|
return;
|
|
163
171
|
const db = new Db(driver, { acl: d1Acl, identity: { roles: ["admin"] }, system: true, schema: app.schema, suppressTriggers: true }, app.schema);
|
|
164
172
|
for (const fn of fns) {
|
|
165
173
|
try {
|
|
166
|
-
await driver.transaction(() => Promise.resolve(fn({ db, driver, schema: app.schema, partition: DEFAULT_PARTITION })));
|
|
174
|
+
await driver.transaction(() => Promise.resolve(fn({ db, driver, schema: app.schema, partition: DEFAULT_PARTITION, env: envBag(env) })));
|
|
167
175
|
}
|
|
168
176
|
catch (e) {
|
|
169
177
|
console.error("[pramen] bootstrap failed (d1):", e);
|
|
@@ -177,14 +185,14 @@ export function makeWorker(app) {
|
|
|
177
185
|
// here and would otherwise be permanently unrunnable on this store. Each is still recorded
|
|
178
186
|
// under its OWN declared partition key, so a ledger read is comparable across stores.
|
|
179
187
|
//
|
|
180
|
-
// Errors are NOT swallowed (unlike runBootstrapD1):
|
|
181
|
-
// `
|
|
182
|
-
// the same fail-closed contract as the DO path.
|
|
188
|
+
// Errors are NOT swallowed (unlike runBootstrapD1): a rejected boot clears the shared
|
|
189
|
+
// memo (`SharedBoot`), so a failed migration fails this request and is retried on the
|
|
190
|
+
// next one — the same fail-closed contract as the DO path.
|
|
183
191
|
//
|
|
184
192
|
// ATOMICITY CAVEAT: D1's `transaction(fn)` is `fn()` (no interactive transactions), so
|
|
185
193
|
// here the claim and the work do NOT commit together. The runner claims the ledger row
|
|
186
194
|
// before running (which is what keeps two cold isolates from both applying the same
|
|
187
|
-
// backfill —
|
|
195
|
+
// backfill — the boot memo is per-isolate and there is no single writer) and releases it on a
|
|
188
196
|
// throw, so a failed migration leaves partial writes and re-runs. Write SQL that tolerates
|
|
189
197
|
// that (`WHERE col IS NULL`) when the D1 store is in play.
|
|
190
198
|
const runDataMigrationsD1 = async (driver) => {
|
|
@@ -206,7 +214,6 @@ export function makeWorker(app) {
|
|
|
206
214
|
}
|
|
207
215
|
return knownPartitionsCache;
|
|
208
216
|
};
|
|
209
|
-
let d1Ready;
|
|
210
217
|
/** Run one handler against the D1 store, in the Worker. The request path and the
|
|
211
218
|
* PRIVILEGED path (routes, which have no ctx.db) both come through here, so the two
|
|
212
219
|
* cannot drift on migration, bootstrap, the multi-tenant guard or the outbox drain.
|
|
@@ -224,7 +231,7 @@ export function makeWorker(app) {
|
|
|
224
231
|
}
|
|
225
232
|
const driver = new D1Driver(env.DB, { start: opts.start });
|
|
226
233
|
const files = createFiles({ tenant: opts.tenant, secret: filesSecret(env), adapter: new R2Adapter(env.FILES) });
|
|
227
|
-
await ensureD1Migrated(driver, env
|
|
234
|
+
await ensureD1Migrated(driver, env, ctx);
|
|
228
235
|
let dispatched;
|
|
229
236
|
try {
|
|
230
237
|
dispatched = await dispatch(app.handlers, app.schema, driver, new Kv(env.KV), files, envBag(env), { acl: d1Acl, identity: opts.identity, tenant: opts.tenant, store: "d1" }, opts.name, opts.input);
|
|
@@ -245,23 +252,38 @@ export function makeWorker(app) {
|
|
|
245
252
|
const { result, enqueued } = dispatched;
|
|
246
253
|
// Drain in the request tail so an enqueued task does not wait for the next Cron tick.
|
|
247
254
|
if (enqueued > 0 && ctx)
|
|
248
|
-
ctx.waitUntil(drainD1(env));
|
|
255
|
+
ctx.waitUntil(drainD1(env, "request", ctx));
|
|
249
256
|
return { driver, result: result };
|
|
250
257
|
};
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
258
|
+
// The D1 store's once-per-isolate boot, shared by every invocation that touches D1. It
|
|
259
|
+
// is a SharedBoot rather than a bare memoized promise because of GitHub #51: the first
|
|
260
|
+
// request after a deploy carrying a table rebuild legitimately ran past the caller's
|
|
261
|
+
// 15 s ceiling, the caller disconnected, the Workers runtime canceled that invocation's
|
|
262
|
+
// pending I/O — and a promise chained on canceled I/O never settles. Not rejects: never
|
|
263
|
+
// settles. The memo then wedged every later fetch in that isolate for its lifetime
|
|
264
|
+
// (0 ms CPU, no logs, "canceled" at whatever timeout the caller had), while crons in
|
|
265
|
+
// another isolate stayed healthy. See `runtime/boot.ts` for the two defenses.
|
|
266
|
+
const d1Boot = new SharedBoot({
|
|
267
|
+
staleMs: D1_BOOT_STALE_MS,
|
|
268
|
+
onOrphaned: (idleMs) => console.warn(`pramen: the D1 store's boot in this isolate made no progress for ${Math.round(idleMs / 1000)}s and is being restarted. ` +
|
|
269
|
+
`The invocation that started it was most likely canceled (its caller disconnected) before the boot finished.`),
|
|
270
|
+
});
|
|
271
|
+
/** Boot the D1 store (migrate → data migrations → outbox → bootstrap) once per isolate.
|
|
272
|
+
* `ctx` is the CALLING invocation's context: when this call is the one that starts the
|
|
273
|
+
* boot, the boot is put under its `waitUntil` so a disconnecting caller cannot cancel a
|
|
274
|
+
* boot everyone else is waiting on. Pass it from every entry that has one. */
|
|
275
|
+
// Takes `env` rather than the one flag it used to read off it: `runBootstrapD1` now hands
|
|
276
|
+
// the environment to each reconciler (see `BootstrapContext.env`), and three call sites
|
|
277
|
+
// each re-deriving `PRAMEN_ALLOW_DESTRUCTIVE === "true"` was already one place too many for
|
|
278
|
+
// a flag whose whole job is to gate data loss.
|
|
279
|
+
const ensureD1Migrated = (driver, env, ctx) => d1Boot.ensure((progress) => {
|
|
280
|
+
const d = observeDriver(driver, progress);
|
|
281
|
+
return migrate(d, app.schema, { allowDestructive: env.PRAMEN_ALLOW_DESTRUCTIVE === "true" })
|
|
282
|
+
.then(() => runDataMigrationsD1(d)) // imperative, recorded backfills (fail closed)
|
|
283
|
+
.then(() => ensureOutbox(d)) // the deferred-tasks table also lives in D1
|
|
284
|
+
.then(() => runBootstrapD1(d, env)) // converge code-defined reference data
|
|
285
|
+
.then(() => undefined);
|
|
286
|
+
}, ctx ? (p) => ctx.waitUntil(p) : undefined);
|
|
265
287
|
// A privileged, system-scoped context for running task handlers on the D1 (Worker)
|
|
266
288
|
// path — mirrors the DO's taskCtx. No live socket, so no DO; drained by a Cron / the
|
|
267
289
|
// /admin/tasks/drain route, never a DO alarm.
|
|
@@ -281,7 +303,7 @@ export function makeWorker(app) {
|
|
|
281
303
|
// already said it looks missing. See `shouldWarnMissingCron`.
|
|
282
304
|
let cronSeen = false;
|
|
283
305
|
let warnedNoCron = false;
|
|
284
|
-
const drainD1 = async (env, source = "request") => {
|
|
306
|
+
const drainD1 = async (env, source = "request", ctx) => {
|
|
285
307
|
if (!env.DB)
|
|
286
308
|
throw new Error("D1 store is not configured");
|
|
287
309
|
if (source === "cron")
|
|
@@ -289,7 +311,7 @@ export function makeWorker(app) {
|
|
|
289
311
|
// The drain reads due tasks then writes their status — pin the primary so it sees
|
|
290
312
|
// and updates current outbox state (not a lagging replica).
|
|
291
313
|
const driver = new D1Driver(env.DB, { start: "first-primary" });
|
|
292
|
-
await ensureD1Migrated(driver, env
|
|
314
|
+
await ensureD1Migrated(driver, env, ctx);
|
|
293
315
|
const result = await drainOutbox(driver, bindTasks(app.tasks, d1TaskCtx(driver, env)), Date.now());
|
|
294
316
|
if (source === "request" && shouldWarnMissingCron({ cronSeen, warned: warnedNoCron, nextRunAt: result.nextRunAt, now: Date.now() })) {
|
|
295
317
|
warnedNoCron = true;
|
|
@@ -301,12 +323,12 @@ export function makeWorker(app) {
|
|
|
301
323
|
}
|
|
302
324
|
return result;
|
|
303
325
|
};
|
|
304
|
-
const listD1Tasks = async (env, status, limit) => {
|
|
326
|
+
const listD1Tasks = async (env, ctx, status, limit) => {
|
|
305
327
|
if (!env.DB)
|
|
306
328
|
throw new Error("D1 store is not configured");
|
|
307
329
|
// Inspection listing — pin the primary so it reflects current outbox state.
|
|
308
330
|
const driver = new D1Driver(env.DB, { start: "first-primary" });
|
|
309
|
-
await ensureD1Migrated(driver, env
|
|
331
|
+
await ensureD1Migrated(driver, env, ctx);
|
|
310
332
|
return listTasks(driver, { status, limit });
|
|
311
333
|
};
|
|
312
334
|
/** Read the data-migration ledger straight from D1 in the Worker (there is no DO on this
|
|
@@ -509,7 +531,7 @@ export function makeWorker(app) {
|
|
|
509
531
|
return forbidden("tasks");
|
|
510
532
|
if (request.headers.get("x-pramen-store") === "d1") {
|
|
511
533
|
try {
|
|
512
|
-
return withCors(json({ ok: true, result: await drainD1(env, "admin") }), cors);
|
|
534
|
+
return withCors(json({ ok: true, result: await drainD1(env, "admin", ctx) }), cors);
|
|
513
535
|
}
|
|
514
536
|
catch (err) {
|
|
515
537
|
const { status, body } = toResponse(err);
|
|
@@ -535,7 +557,7 @@ export function makeWorker(app) {
|
|
|
535
557
|
const limit = Number(url.searchParams.get("limit")) || undefined;
|
|
536
558
|
if (request.headers.get("x-pramen-store") === "d1") {
|
|
537
559
|
try {
|
|
538
|
-
return withCors(json({ ok: true, result: await listD1Tasks(env, status, limit) }), cors);
|
|
560
|
+
return withCors(json({ ok: true, result: await listD1Tasks(env, ctx, status, limit) }), cors);
|
|
539
561
|
}
|
|
540
562
|
catch (err) {
|
|
541
563
|
const { status: s, body } = toResponse(err);
|
|
@@ -665,9 +687,9 @@ export function makeWorker(app) {
|
|
|
665
687
|
},
|
|
666
688
|
// Cron Trigger entry: drains the D1 outbox (the DO path self-drains via an alarm,
|
|
667
689
|
// so it needs no cron). Wire a `[triggers] crons` in wrangler/oblaka to call this.
|
|
668
|
-
async scheduled(_event, env) {
|
|
690
|
+
async scheduled(_event, env, ctx) {
|
|
669
691
|
if (env.DB)
|
|
670
|
-
await drainD1(env, "cron");
|
|
692
|
+
await drainD1(env, "cron", ctx);
|
|
671
693
|
},
|
|
672
694
|
// Cloudflare Queues consumer entry: routes a batch to the matching `app.queues`
|
|
673
695
|
// handler (ACK on success / RETRY on throw, per message). A consumer is Worker-level
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@pramen/server",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.63",
|
|
4
4
|
"description": "pramen server runtime \u2014 schema, ACL, ORM, live queries, files, and the createPramen(app) factory for Cloudflare Workers + Durable Objects.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
package/src/durable-object.ts
CHANGED
|
@@ -199,7 +199,7 @@ export class PramenDOBase extends DurableObject<DoEnv> {
|
|
|
199
199
|
);
|
|
200
200
|
for (const fn of fns) {
|
|
201
201
|
try {
|
|
202
|
-
await this.driver.transaction(() => Promise.resolve(fn({ db, driver: this.driver, schema: this.app.schema, partition: this.partition })));
|
|
202
|
+
await this.driver.transaction(() => Promise.resolve(fn({ db, driver: this.driver, schema: this.app.schema, partition: this.partition, env: this.envBag })));
|
|
203
203
|
} catch (e) {
|
|
204
204
|
console.error(`[pramen] bootstrap failed (partition=${this.partition}):`, e);
|
|
205
205
|
}
|
package/src/pramen.ts
CHANGED
|
@@ -75,7 +75,7 @@ export type { Env, DoEnv };
|
|
|
75
75
|
* use the D1 store with deferred tasks. */
|
|
76
76
|
export function createPramen(app: PramenApp): {
|
|
77
77
|
fetch: (request: Request, env: Env, ctx: ExecutionContext) => Promise<Response>;
|
|
78
|
-
scheduled: (event: unknown, env: Env) => Promise<void>;
|
|
78
|
+
scheduled: (event: unknown, env: Env, ctx: ExecutionContext) => Promise<void>;
|
|
79
79
|
queue: (batch: QueueBatch, env: Env) => Promise<void>;
|
|
80
80
|
PramenDO: ReturnType<typeof pramenDO>;
|
|
81
81
|
} {
|
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
// SharedBoot — a once-per-isolate boot that many invocations await, made safe to share.
|
|
2
|
+
//
|
|
3
|
+
// The D1 store boots in the Worker: migrate → data migrations → outbox table → bootstrap,
|
|
4
|
+
// run by whichever request arrives first in a fresh isolate and memoized as ONE promise the
|
|
5
|
+
// rest await (running it per request would re-diff the schema on every call). That memo
|
|
6
|
+
// is the bug behind GitHub #51, and the reason is a Workers runtime rule, not pramen code:
|
|
7
|
+
// async work belongs to the INVOCATION that started it. When that invocation ends — the
|
|
8
|
+
// caller gives up and disconnects, which is exactly what a proxy with a 15 s ceiling does
|
|
9
|
+
// to the legitimately slow first request after a deploy carrying a table rebuild — its
|
|
10
|
+
// pending I/O is canceled, and a promise chained on canceled I/O never settles. Not
|
|
11
|
+
// rejected: never settles. So the `.catch` that was meant to clear the memo never runs,
|
|
12
|
+
// and every later fetch in that isolate awaits a promise that cannot resolve, for as long
|
|
13
|
+
// as its own caller allows, at 0 ms CPU. Crons stayed healthy in the report because they
|
|
14
|
+
// landed in a different isolate; the ~54 % was the share of traffic routed to the wedged
|
|
15
|
+
// one; a redeploy "fixed" it by discarding every isolate.
|
|
16
|
+
//
|
|
17
|
+
// Two defenses, and both are needed:
|
|
18
|
+
//
|
|
19
|
+
// 1. The STARTER extends its invocation over the boot with `ctx.waitUntil`, so the client
|
|
20
|
+
// disconnecting no longer cancels the boot's I/O. That closes the reported case. It is
|
|
21
|
+
// not sufficient on its own: waitUntil is capped (30 s after the response or the
|
|
22
|
+
// disconnect), and there is no rule that says a promise's owner is the only way it can
|
|
23
|
+
// be orphaned.
|
|
24
|
+
// 2. AWAITERS never trust a shared boot unconditionally. The boot reports PROGRESS (every
|
|
25
|
+
// statement the driver runs), and a boot that has made none for `staleMs` while still
|
|
26
|
+
// unsettled is treated as orphaned: the next awaiter — arriving, or one already waiting
|
|
27
|
+
// — starts a fresh boot in its own invocation and awaits that instead. An isolate can
|
|
28
|
+
// therefore be wedged for at most `staleMs`, never for its lifetime. Every step of the
|
|
29
|
+
// boot tolerates a second runner (that is the cross-isolate reality on D1 already: the
|
|
30
|
+
// migration ledger is lease-claimed, migrate is a diff, bootstrap is an upsert), so a
|
|
31
|
+
// false positive costs a redundant boot, not correctness — which is why the threshold is
|
|
32
|
+
// measured from the last STATEMENT and not from the boot's start: a boot that is slow
|
|
33
|
+
// but alive keeps making progress.
|
|
34
|
+
//
|
|
35
|
+
// Platform-agnostic on purpose (no `cloudflare:workers`): `keepAlive` is whatever the host
|
|
36
|
+
// offers to extend an invocation, and `now` is injectable for tests.
|
|
37
|
+
|
|
38
|
+
import type { Driver } from "./driver";
|
|
39
|
+
|
|
40
|
+
export interface SharedBootOpts {
|
|
41
|
+
/** How long a still-unsettled boot may go without progress before it counts as orphaned. */
|
|
42
|
+
staleMs: number;
|
|
43
|
+
/** Clock, injectable for tests. */
|
|
44
|
+
now?: () => number;
|
|
45
|
+
/** Called when a boot is judged orphaned and replaced, with how long it had been idle. */
|
|
46
|
+
onOrphaned?: (idleMs: number) => void;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
interface BootRecord {
|
|
50
|
+
promise: Promise<void>;
|
|
51
|
+
lastProgressAt: number;
|
|
52
|
+
done: boolean;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/** The boot body. Call `progress()` whenever the boot does observable work — see
|
|
56
|
+
* `observeDriver` for the standard way to get that for free. */
|
|
57
|
+
export type BootFn = (progress: () => void) => Promise<void>;
|
|
58
|
+
|
|
59
|
+
export class SharedBoot {
|
|
60
|
+
private current: BootRecord | undefined;
|
|
61
|
+
private readonly now: () => number;
|
|
62
|
+
|
|
63
|
+
constructor(private readonly opts: SharedBootOpts) {
|
|
64
|
+
this.now = opts.now ?? Date.now;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/** Await the shared boot, starting one if none is live (or the live one looks orphaned).
|
|
68
|
+
*
|
|
69
|
+
* `run` is only invoked when THIS call has to start a boot. `keepAlive` is the calling
|
|
70
|
+
* invocation's lifetime extender (`ctx.waitUntil`); it is handed the boot's promise so
|
|
71
|
+
* the boot survives the caller disconnecting. A rejected boot rejects every awaiter and
|
|
72
|
+
* clears the memo, so the next call retries — fail closed, as before. */
|
|
73
|
+
async ensure(run: BootFn, keepAlive?: (p: Promise<unknown>) => void): Promise<void> {
|
|
74
|
+
let boot = this.current;
|
|
75
|
+
// The STARTER awaits its own boot outright: the boot runs in its invocation, so if
|
|
76
|
+
// the starter is alive to watch, so is the boot — and if the invocation is canceled,
|
|
77
|
+
// this await dies with it. A starter that also watched would restart its own
|
|
78
|
+
// (dead-for-everyone-else) boot every window, endlessly.
|
|
79
|
+
if (!boot || this.isStale(boot)) return this.start(run, keepAlive, boot).promise;
|
|
80
|
+
for (;;) {
|
|
81
|
+
if (boot.done) return;
|
|
82
|
+
// Wait, but wake when the boot WOULD count as stale, and re-check: progress since
|
|
83
|
+
// means it is alive and we keep waiting; none means it is orphaned.
|
|
84
|
+
if (await this.settledBeforeStale(boot)) return;
|
|
85
|
+
// Someone else may already have replaced it — join theirs rather than start a third.
|
|
86
|
+
const live = this.current;
|
|
87
|
+
if (live && live !== boot && !this.isStale(live)) {
|
|
88
|
+
boot = live;
|
|
89
|
+
continue;
|
|
90
|
+
}
|
|
91
|
+
return this.start(run, keepAlive, boot).promise;
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/** Whether a boot has completed in this isolate — for diagnostics only. */
|
|
96
|
+
isDone(): boolean {
|
|
97
|
+
return this.current?.done === true;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
private isStale(boot: BootRecord): boolean {
|
|
101
|
+
return !boot.done && this.now() - boot.lastProgressAt >= this.opts.staleMs;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
private start(run: BootFn, keepAlive: ((p: Promise<unknown>) => void) | undefined, replacing: BootRecord | undefined): BootRecord {
|
|
105
|
+
if (replacing) this.opts.onOrphaned?.(this.now() - replacing.lastProgressAt);
|
|
106
|
+
const record: BootRecord = { promise: Promise.resolve(), lastProgressAt: this.now(), done: false };
|
|
107
|
+
const progress = (): void => {
|
|
108
|
+
record.lastProgressAt = this.now();
|
|
109
|
+
};
|
|
110
|
+
// Through a resolved promise so a synchronous throw in `run` is a rejection, not an
|
|
111
|
+
// exception escaping `ensure` with the memo half-installed.
|
|
112
|
+
record.promise = Promise.resolve()
|
|
113
|
+
.then(() => run(progress))
|
|
114
|
+
.then(
|
|
115
|
+
() => {
|
|
116
|
+
record.done = true;
|
|
117
|
+
},
|
|
118
|
+
(e: unknown) => {
|
|
119
|
+
// Only forget the memo if it is still THIS boot: a replacement started meanwhile
|
|
120
|
+
// must not be discarded because its predecessor finally failed.
|
|
121
|
+
if (this.current === record) this.current = undefined;
|
|
122
|
+
throw e;
|
|
123
|
+
},
|
|
124
|
+
);
|
|
125
|
+
// Every awaiter attaches its own handler; this one just keeps the host from reporting
|
|
126
|
+
// an unhandled rejection when a boot fails with nobody awaiting the stored promise.
|
|
127
|
+
record.promise.catch(() => {});
|
|
128
|
+
keepAlive?.(record.promise.catch(() => {}));
|
|
129
|
+
this.current = record;
|
|
130
|
+
return record;
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
/** Resolves `true` when the boot settles (rejects if it rejected), `false` once it has
|
|
134
|
+
* gone `staleMs` without progress — the timer re-arms from the latest progress instant,
|
|
135
|
+
* so a boot that keeps working is never reported stale. */
|
|
136
|
+
private settledBeforeStale(boot: BootRecord): Promise<boolean> {
|
|
137
|
+
return new Promise<boolean>((resolve, reject) => {
|
|
138
|
+
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
139
|
+
const arm = (): void => {
|
|
140
|
+
const wait = Math.max(0, boot.lastProgressAt + this.opts.staleMs - this.now());
|
|
141
|
+
timer = setTimeout(() => (this.isStale(boot) ? resolve(false) : arm()), wait);
|
|
142
|
+
};
|
|
143
|
+
const disarm = (): void => {
|
|
144
|
+
if (timer !== undefined) clearTimeout(timer);
|
|
145
|
+
};
|
|
146
|
+
arm();
|
|
147
|
+
boot.promise.then(
|
|
148
|
+
() => {
|
|
149
|
+
disarm();
|
|
150
|
+
resolve(true);
|
|
151
|
+
},
|
|
152
|
+
(e: unknown) => {
|
|
153
|
+
disarm();
|
|
154
|
+
reject(e);
|
|
155
|
+
},
|
|
156
|
+
);
|
|
157
|
+
});
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
/** Wrap a Driver so every statement it runs reports progress — the boot's liveness signal.
|
|
162
|
+
* `batch` is forwarded only when the wrapped driver has it, because the migrator decides
|
|
163
|
+
* between an atomic batch and sequential exec by its presence. */
|
|
164
|
+
export function observeDriver(driver: Driver, progress: () => void): Driver {
|
|
165
|
+
const observed: Driver = {
|
|
166
|
+
dialect: driver.dialect,
|
|
167
|
+
exec: async (sql, params) => {
|
|
168
|
+
progress();
|
|
169
|
+
const rows = await driver.exec(sql, params);
|
|
170
|
+
progress();
|
|
171
|
+
return rows;
|
|
172
|
+
},
|
|
173
|
+
transaction: (fn) => driver.transaction(fn),
|
|
174
|
+
};
|
|
175
|
+
if (driver.batch) {
|
|
176
|
+
const batch = driver.batch.bind(driver);
|
|
177
|
+
observed.batch = async (statements) => {
|
|
178
|
+
progress();
|
|
179
|
+
await batch(statements);
|
|
180
|
+
progress();
|
|
181
|
+
};
|
|
182
|
+
}
|
|
183
|
+
return observed;
|
|
184
|
+
}
|
|
@@ -23,7 +23,7 @@
|
|
|
23
23
|
// that inserts a pending row, or steals one whose lease has expired, or does nothing — and
|
|
24
24
|
// the migration runs only if that statement won the row. On the DO this is merely bookkeeping
|
|
25
25
|
// order inside one transaction. On D1 it is what makes the once-only contract hold at all:
|
|
26
|
-
// there is no single writer and no interactive transaction,
|
|
26
|
+
// there is no single writer and no interactive transaction, the boot memo is per-isolate, so two
|
|
27
27
|
// cold isolates racing a `SET n = n * 2` backfill would each read an empty ledger, each run
|
|
28
28
|
// it, and quadruple the data. The conflicting upsert is the lock.
|
|
29
29
|
//
|
package/src/runtime/mail.ts
CHANGED
|
@@ -6,10 +6,9 @@
|
|
|
6
6
|
// await ctx.mail.send({ to: "u@x.com", subject: "Welcome", text: "…" });
|
|
7
7
|
//
|
|
8
8
|
// Two real transports. Cloudflare Email Sending (the `send_email`/`EMAIL` binding) needs
|
|
9
|
-
// no API keys, but it can only send FROM a domain that is a zone in the same account
|
|
10
|
-
//
|
|
11
|
-
//
|
|
12
|
-
// API, any recipient, at the cost of a key. Configure it and it wins.
|
|
9
|
+
// no API keys, but it can only send FROM a domain that is a zone in the same account.
|
|
10
|
+
// Mailgun is the way out of that: an HTTP API, a domain verified once with Mailgun
|
|
11
|
+
// rather than owned by the account, at the cost of a key. Configure it and it wins.
|
|
13
12
|
//
|
|
14
13
|
// With neither configured (local/dev), mail is captured instead of sent — to KV (so an
|
|
15
14
|
// e2e/dashboard can read the "inbox") or in-memory — so handlers work unchanged
|
|
@@ -90,11 +89,9 @@ export class CloudflareEmailAdapter implements MailAdapter {
|
|
|
90
89
|
|
|
91
90
|
/** Mailgun — an HTTP transport, for when Cloudflare Email Sending cannot be used.
|
|
92
91
|
*
|
|
93
|
-
* Worth the key for one reason: Cloudflare will only send from a domain that is a zone
|
|
94
|
-
*
|
|
95
|
-
*
|
|
96
|
-
* address"). That is workable for a handful of operators and hopeless for real users.
|
|
97
|
-
* Mailgun asks the domain be verified once, then delivers to anyone.
|
|
92
|
+
* Worth the key for one reason: Cloudflare will only send from a domain that is a zone in
|
|
93
|
+
* the same account. Mailgun asks the domain be verified once with Mailgun instead, so the
|
|
94
|
+
* sender need not be a domain this account owns.
|
|
98
95
|
*
|
|
99
96
|
* A non-2xx THROWS, deliberately: `ctx.mail.send` is normally called from a task, and a
|
|
100
97
|
* throw is what makes the outbox retry and then dead-letter visibly. Swallowing the
|
package/src/sdk/handlers.ts
CHANGED
|
@@ -74,6 +74,12 @@ export interface BootstrapContext<S extends SchemaDef = SchemaDef> {
|
|
|
74
74
|
/** The partition being booted. On the DO path bootstrap runs ONLY for the default
|
|
75
75
|
* partition (reference data lives there); on the D1 path it is always the default. */
|
|
76
76
|
readonly partition: string;
|
|
77
|
+
/** The Worker/DO environment — bindings, vars and secrets, as a handler's `ctx.env` sees
|
|
78
|
+
* it. Reference data is not always unconditional: a seed that belongs in local dev and
|
|
79
|
+
* nowhere else (a demo account with a known password) has to be able to READ the flag that
|
|
80
|
+
* says which it is, and a boot-time reconciler has no request to carry one. Without this
|
|
81
|
+
* the only gates available were "always" and "never". */
|
|
82
|
+
readonly env: EnvBag;
|
|
77
83
|
}
|
|
78
84
|
|
|
79
85
|
/** An idempotent reconcile run once after `migrate()` on each boot (a DO's first fetch, or a
|
package/src/worker.ts
CHANGED
|
@@ -14,6 +14,7 @@ import { createMail } from "./runtime/mail";
|
|
|
14
14
|
import { createQueue, type QueueProducerBinding } from "./runtime/queue";
|
|
15
15
|
import { dispatchQueueBatch, type QueueBatch, type QueueContext } from "./runtime/queue-consumer";
|
|
16
16
|
import { migrate } from "./runtime/migrate";
|
|
17
|
+
import { SharedBoot, observeDriver } from "./runtime/boot";
|
|
17
18
|
import { compileAcl } from "./runtime/acl";
|
|
18
19
|
import { Db } from "./runtime/db";
|
|
19
20
|
import { D1Driver, type D1SessionStart, type Driver } from "./runtime/driver";
|
|
@@ -101,6 +102,14 @@ export function useD1Store(opts: { storeHeader: string | null; isLive: boolean;
|
|
|
101
102
|
return opts.defaultStore === "d1";
|
|
102
103
|
}
|
|
103
104
|
|
|
105
|
+
/** How long the D1 boot may go without completing a statement before an awaiter treats it
|
|
106
|
+
* as orphaned and starts its own (GitHub #51). 30 s is the platform's `waitUntil` cap after
|
|
107
|
+
* a response or a disconnect — the point past which a boot's starter can no longer be
|
|
108
|
+
* keeping it alive — and comfortably above any single D1 statement a bounded migration
|
|
109
|
+
* should issue. Measured from the last STATEMENT, not from the boot's start, so a slow but
|
|
110
|
+
* live boot (a table rebuild followed by a backfill) is never mistaken for a dead one. */
|
|
111
|
+
export const D1_BOOT_STALE_MS = 30_000;
|
|
112
|
+
|
|
104
113
|
/** Should we warn that no Cron trigger seems to be wired?
|
|
105
114
|
*
|
|
106
115
|
* The DO store self-drains via an alarm; the D1 store has none, so a DELAYED task — a
|
|
@@ -222,13 +231,13 @@ export function makeWorker(app: PramenApp) {
|
|
|
222
231
|
// runBootstrap(), run once per isolate after migration. D1 is a single shared store (no
|
|
223
232
|
// partition split), so every reconciler runs under the default partition. SYSTEM-scoped
|
|
224
233
|
// Db (ACL bypassed), triggers suppressed; a failing reconciler is logged, never fatal.
|
|
225
|
-
const runBootstrapD1 = async (driver: Driver): Promise<void> => {
|
|
234
|
+
const runBootstrapD1 = async (driver: Driver, env: Env): Promise<void> => {
|
|
226
235
|
const fns = app.bootstrap;
|
|
227
236
|
if (!fns?.length) return;
|
|
228
237
|
const db = new Db(driver, { acl: d1Acl, identity: { roles: ["admin"] }, system: true, schema: app.schema, suppressTriggers: true }, app.schema);
|
|
229
238
|
for (const fn of fns) {
|
|
230
239
|
try {
|
|
231
|
-
await driver.transaction(() => Promise.resolve(fn({ db, driver, schema: app.schema, partition: DEFAULT_PARTITION })));
|
|
240
|
+
await driver.transaction(() => Promise.resolve(fn({ db, driver, schema: app.schema, partition: DEFAULT_PARTITION, env: envBag(env) })));
|
|
232
241
|
} catch (e) {
|
|
233
242
|
console.error("[pramen] bootstrap failed (d1):", e);
|
|
234
243
|
}
|
|
@@ -242,14 +251,14 @@ export function makeWorker(app: PramenApp) {
|
|
|
242
251
|
// here and would otherwise be permanently unrunnable on this store. Each is still recorded
|
|
243
252
|
// under its OWN declared partition key, so a ledger read is comparable across stores.
|
|
244
253
|
//
|
|
245
|
-
// Errors are NOT swallowed (unlike runBootstrapD1):
|
|
246
|
-
// `
|
|
247
|
-
// the same fail-closed contract as the DO path.
|
|
254
|
+
// Errors are NOT swallowed (unlike runBootstrapD1): a rejected boot clears the shared
|
|
255
|
+
// memo (`SharedBoot`), so a failed migration fails this request and is retried on the
|
|
256
|
+
// next one — the same fail-closed contract as the DO path.
|
|
248
257
|
//
|
|
249
258
|
// ATOMICITY CAVEAT: D1's `transaction(fn)` is `fn()` (no interactive transactions), so
|
|
250
259
|
// here the claim and the work do NOT commit together. The runner claims the ledger row
|
|
251
260
|
// before running (which is what keeps two cold isolates from both applying the same
|
|
252
|
-
// backfill —
|
|
261
|
+
// backfill — the boot memo is per-isolate and there is no single writer) and releases it on a
|
|
253
262
|
// throw, so a failed migration leaves partial writes and re-runs. Write SQL that tolerates
|
|
254
263
|
// that (`WHERE col IS NULL`) when the D1 store is in play.
|
|
255
264
|
const runDataMigrationsD1 = async (driver: Driver): Promise<void> => {
|
|
@@ -272,7 +281,6 @@ export function makeWorker(app: PramenApp) {
|
|
|
272
281
|
return knownPartitionsCache;
|
|
273
282
|
};
|
|
274
283
|
|
|
275
|
-
let d1Ready: Promise<void> | undefined;
|
|
276
284
|
/** Run one handler against the D1 store, in the Worker. The request path and the
|
|
277
285
|
* PRIVILEGED path (routes, which have no ctx.db) both come through here, so the two
|
|
278
286
|
* cannot drift on migration, bootstrap, the multi-tenant guard or the outbox drain.
|
|
@@ -293,7 +301,7 @@ export function makeWorker(app: PramenApp) {
|
|
|
293
301
|
}
|
|
294
302
|
const driver = new D1Driver(env.DB, { start: opts.start });
|
|
295
303
|
const files = createFiles({ tenant: opts.tenant, secret: filesSecret(env), adapter: new R2Adapter(env.FILES) });
|
|
296
|
-
await ensureD1Migrated(driver, env
|
|
304
|
+
await ensureD1Migrated(driver, env, ctx);
|
|
297
305
|
let dispatched;
|
|
298
306
|
try {
|
|
299
307
|
dispatched = await dispatch(
|
|
@@ -324,24 +332,47 @@ export function makeWorker(app: PramenApp) {
|
|
|
324
332
|
}
|
|
325
333
|
const { result, enqueued } = dispatched;
|
|
326
334
|
// Drain in the request tail so an enqueued task does not wait for the next Cron tick.
|
|
327
|
-
if (enqueued > 0 && ctx) ctx.waitUntil(drainD1(env));
|
|
335
|
+
if (enqueued > 0 && ctx) ctx.waitUntil(drainD1(env, "request", ctx));
|
|
328
336
|
return { driver, result: result as JsonValue };
|
|
329
337
|
};
|
|
330
338
|
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
339
|
+
// The D1 store's once-per-isolate boot, shared by every invocation that touches D1. It
|
|
340
|
+
// is a SharedBoot rather than a bare memoized promise because of GitHub #51: the first
|
|
341
|
+
// request after a deploy carrying a table rebuild legitimately ran past the caller's
|
|
342
|
+
// 15 s ceiling, the caller disconnected, the Workers runtime canceled that invocation's
|
|
343
|
+
// pending I/O — and a promise chained on canceled I/O never settles. Not rejects: never
|
|
344
|
+
// settles. The memo then wedged every later fetch in that isolate for its lifetime
|
|
345
|
+
// (0 ms CPU, no logs, "canceled" at whatever timeout the caller had), while crons in
|
|
346
|
+
// another isolate stayed healthy. See `runtime/boot.ts` for the two defenses.
|
|
347
|
+
const d1Boot = new SharedBoot({
|
|
348
|
+
staleMs: D1_BOOT_STALE_MS,
|
|
349
|
+
onOrphaned: (idleMs) =>
|
|
350
|
+
console.warn(
|
|
351
|
+
`pramen: the D1 store's boot in this isolate made no progress for ${Math.round(idleMs / 1000)}s and is being restarted. ` +
|
|
352
|
+
`The invocation that started it was most likely canceled (its caller disconnected) before the boot finished.`,
|
|
353
|
+
),
|
|
354
|
+
});
|
|
355
|
+
|
|
356
|
+
/** Boot the D1 store (migrate → data migrations → outbox → bootstrap) once per isolate.
|
|
357
|
+
* `ctx` is the CALLING invocation's context: when this call is the one that starts the
|
|
358
|
+
* boot, the boot is put under its `waitUntil` so a disconnecting caller cannot cancel a
|
|
359
|
+
* boot everyone else is waiting on. Pass it from every entry that has one. */
|
|
360
|
+
// Takes `env` rather than the one flag it used to read off it: `runBootstrapD1` now hands
|
|
361
|
+
// the environment to each reconciler (see `BootstrapContext.env`), and three call sites
|
|
362
|
+
// each re-deriving `PRAMEN_ALLOW_DESTRUCTIVE === "true"` was already one place too many for
|
|
363
|
+
// a flag whose whole job is to gate data loss.
|
|
364
|
+
const ensureD1Migrated = (driver: Driver, env: Env, ctx?: ExecutionContext): Promise<void> =>
|
|
365
|
+
d1Boot.ensure(
|
|
366
|
+
(progress) => {
|
|
367
|
+
const d = observeDriver(driver, progress);
|
|
368
|
+
return migrate(d, app.schema, { allowDestructive: env.PRAMEN_ALLOW_DESTRUCTIVE === "true" })
|
|
369
|
+
.then(() => runDataMigrationsD1(d)) // imperative, recorded backfills (fail closed)
|
|
370
|
+
.then(() => ensureOutbox(d)) // the deferred-tasks table also lives in D1
|
|
371
|
+
.then(() => runBootstrapD1(d, env)) // converge code-defined reference data
|
|
372
|
+
.then(() => undefined);
|
|
373
|
+
},
|
|
374
|
+
ctx ? (p) => ctx.waitUntil(p) : undefined,
|
|
375
|
+
);
|
|
345
376
|
|
|
346
377
|
// A privileged, system-scoped context for running task handlers on the D1 (Worker)
|
|
347
378
|
// path — mirrors the DO's taskCtx. No live socket, so no DO; drained by a Cron / the
|
|
@@ -364,13 +395,13 @@ export function makeWorker(app: PramenApp) {
|
|
|
364
395
|
let cronSeen = false;
|
|
365
396
|
let warnedNoCron = false;
|
|
366
397
|
|
|
367
|
-
const drainD1 = async (env: Env, source: "request" | "cron" | "admin" = "request"): Promise<unknown> => {
|
|
398
|
+
const drainD1 = async (env: Env, source: "request" | "cron" | "admin" = "request", ctx?: ExecutionContext): Promise<unknown> => {
|
|
368
399
|
if (!env.DB) throw new Error("D1 store is not configured");
|
|
369
400
|
if (source === "cron") cronSeen = true;
|
|
370
401
|
// The drain reads due tasks then writes their status — pin the primary so it sees
|
|
371
402
|
// and updates current outbox state (not a lagging replica).
|
|
372
403
|
const driver = new D1Driver(env.DB, { start: "first-primary" });
|
|
373
|
-
await ensureD1Migrated(driver, env
|
|
404
|
+
await ensureD1Migrated(driver, env, ctx);
|
|
374
405
|
const result = await drainOutbox(driver, bindTasks(app.tasks, d1TaskCtx(driver, env)), Date.now());
|
|
375
406
|
if (source === "request" && shouldWarnMissingCron({ cronSeen, warned: warnedNoCron, nextRunAt: result.nextRunAt, now: Date.now() })) {
|
|
376
407
|
warnedNoCron = true;
|
|
@@ -385,11 +416,11 @@ export function makeWorker(app: PramenApp) {
|
|
|
385
416
|
return result;
|
|
386
417
|
};
|
|
387
418
|
|
|
388
|
-
const listD1Tasks = async (env: Env, status?: string, limit?: number): Promise<unknown> => {
|
|
419
|
+
const listD1Tasks = async (env: Env, ctx: ExecutionContext, status?: string, limit?: number): Promise<unknown> => {
|
|
389
420
|
if (!env.DB) throw new Error("D1 store is not configured");
|
|
390
421
|
// Inspection listing — pin the primary so it reflects current outbox state.
|
|
391
422
|
const driver = new D1Driver(env.DB, { start: "first-primary" });
|
|
392
|
-
await ensureD1Migrated(driver, env
|
|
423
|
+
await ensureD1Migrated(driver, env, ctx);
|
|
393
424
|
return listTasks(driver, { status, limit });
|
|
394
425
|
};
|
|
395
426
|
|
|
@@ -596,7 +627,7 @@ export function makeWorker(app: PramenApp) {
|
|
|
596
627
|
if (!isAdmin(identity)) return forbidden("tasks");
|
|
597
628
|
if (request.headers.get("x-pramen-store") === "d1") {
|
|
598
629
|
try {
|
|
599
|
-
return withCors(json({ ok: true, result: await drainD1(env, "admin") }), cors);
|
|
630
|
+
return withCors(json({ ok: true, result: await drainD1(env, "admin", ctx) }), cors);
|
|
600
631
|
} catch (err) {
|
|
601
632
|
const { status, body } = toResponse(err);
|
|
602
633
|
return withCors(json(body, status), cors);
|
|
@@ -623,7 +654,7 @@ export function makeWorker(app: PramenApp) {
|
|
|
623
654
|
const limit = Number(url.searchParams.get("limit")) || undefined;
|
|
624
655
|
if (request.headers.get("x-pramen-store") === "d1") {
|
|
625
656
|
try {
|
|
626
|
-
return withCors(json({ ok: true, result: await listD1Tasks(env, status, limit) }), cors);
|
|
657
|
+
return withCors(json({ ok: true, result: await listD1Tasks(env, ctx, status, limit) }), cors);
|
|
627
658
|
} catch (err) {
|
|
628
659
|
const { status: s, body } = toResponse(err);
|
|
629
660
|
return withCors(json(body, s), cors);
|
|
@@ -763,8 +794,8 @@ export function makeWorker(app: PramenApp) {
|
|
|
763
794
|
|
|
764
795
|
// Cron Trigger entry: drains the D1 outbox (the DO path self-drains via an alarm,
|
|
765
796
|
// so it needs no cron). Wire a `[triggers] crons` in wrangler/oblaka to call this.
|
|
766
|
-
async scheduled(_event: unknown, env: Env): Promise<void> {
|
|
767
|
-
if (env.DB) await drainD1(env, "cron");
|
|
797
|
+
async scheduled(_event: unknown, env: Env, ctx: ExecutionContext): Promise<void> {
|
|
798
|
+
if (env.DB) await drainD1(env, "cron", ctx);
|
|
768
799
|
},
|
|
769
800
|
|
|
770
801
|
// Cloudflare Queues consumer entry: routes a batch to the matching `app.queues`
|