@pithy-sh/email 0.1.7 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/send/errorMapping.d.ts +27 -3
- package/dist/send/errorMapping.d.ts.map +1 -1
- package/dist/send/errorMapping.js +71 -9
- package/dist/send/runSend.d.ts.map +1 -1
- package/dist/send/runSend.js +3 -3
- package/dist/version.generated.d.ts +1 -1
- package/dist/version.generated.js +1 -1
- package/dist/workflows/hostEnv.js +1 -1
- package/dist/workflows/scheduler.d.ts +47 -4
- package/dist/workflows/scheduler.d.ts.map +1 -1
- package/dist/workflows/scheduler.js +69 -10
- package/dist/workflows/worker.d.ts +23 -2
- package/dist/workflows/worker.d.ts.map +1 -1
- package/dist/workflows/worker.js +35 -6
- package/package.json +10 -8
- package/src/send/errorMapping.ts +92 -11
- package/src/send/runSend.ts +7 -3
- package/src/version.generated.ts +1 -1
- package/src/workflows/hostEnv.ts +1 -1
- package/src/workflows/scheduler.ts +110 -22
- package/src/workflows/worker.ts +36 -9
- package/src/workflows/wrangler.jsonc +3 -2
|
@@ -10,8 +10,23 @@ import type { PithyError } from "@pithy-sh/core/src/error/pithyError";
|
|
|
10
10
|
*/
|
|
11
11
|
/** How a given send error is handled: which Pithy error to raise, and whether the send Workflow retries. */
|
|
12
12
|
export interface ClassifiedSendError {
|
|
13
|
-
/** The `E_*` code reported by the binding, or `E_UNKNOWN`
|
|
13
|
+
/** The `E_*` code reported by the binding, a numeric provider code as a string, or `E_UNKNOWN` for none. */
|
|
14
14
|
code: string;
|
|
15
|
+
/**
|
|
16
|
+
* The cause, as one operator-facing line: the code and whatever the provider said with it.
|
|
17
|
+
*
|
|
18
|
+
* **Exposed because the job row needs it and only the error object had it (pithy-sh/pithy#555).**
|
|
19
|
+
* `runSend` persisted the bare `code` into `pithy_email_jobs.error`, so a magic link that failed five
|
|
20
|
+
* times against a token for the wrong account recorded `E_UNKNOWN` five times and nothing about why —
|
|
21
|
+
* while this exact string, naming `10000 Authentication error`, was being built two lines down and
|
|
22
|
+
* thrown away with the error it decorated. Same family as #534: a failure that discards the cause it
|
|
23
|
+
* was handed.
|
|
24
|
+
*
|
|
25
|
+
* Identical to `error.payload.detail`, deliberately — one composition, two readers, so the row and the
|
|
26
|
+
* log cannot drift. `detail` is stripped at the client boundary, which is where a provider's own words
|
|
27
|
+
* belong.
|
|
28
|
+
*/
|
|
29
|
+
detail: string;
|
|
15
30
|
/** Whether the send should be retried with backoff. */
|
|
16
31
|
retryable: boolean;
|
|
17
32
|
/** Whether the failure means the recipient is suppressed (so the address should be locally suppressed). */
|
|
@@ -22,8 +37,17 @@ export interface ClassifiedSendError {
|
|
|
22
37
|
/**
|
|
23
38
|
* Classify a send failure. `E_RECIPIENT_SUPPRESSED` is terminal and means the address must be
|
|
24
39
|
* suppressed locally too; rate/quota and transient delivery codes are retryable; everything else
|
|
25
|
-
* (validation, sender, content, headers) is terminal.
|
|
26
|
-
*
|
|
40
|
+
* (validation, sender, content, headers) is terminal.
|
|
41
|
+
*
|
|
42
|
+
* **An unrecognized code is retryable, and that is deliberately unchanged.** A failure this module has
|
|
43
|
+
* no code at all keeps its bounded retry, and so does a *numeric* provider code nobody here has listed —
|
|
44
|
+
* see {@link isProviderNumericCode} for why those two are alike and an unlisted `E_*` is not.
|
|
45
|
+
*
|
|
46
|
+
* What changed for #555 is that the **authentication** class is now named and terminal: `10000
|
|
47
|
+
* Authentication error` is a token that cannot send as this domain, and spending five attempts on it in
|
|
48
|
+
* silence is what hid the whole incident. See {@link UNAUTHORIZED} for why that is a list rather than
|
|
49
|
+
* "numeric means permanent" — the latter was this fix's first shape, and it made a momentary numeric
|
|
50
|
+
* failure permanent as a side effect.
|
|
27
51
|
*/
|
|
28
52
|
export declare function classifySendError(err: unknown): ClassifiedSendError;
|
|
29
53
|
//# sourceMappingURL=errorMapping.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"errorMapping.d.ts","sourceRoot":"","sources":["../../src/send/errorMapping.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,qCAAqC,CAAC;AAGtE;;;;;;GAMG;AAEH,4GAA4G;AAC5G,MAAM,WAAW,mBAAmB;IAClC,
|
|
1
|
+
{"version":3,"file":"errorMapping.d.ts","sourceRoot":"","sources":["../../src/send/errorMapping.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,qCAAqC,CAAC;AAGtE;;;;;;GAMG;AAEH,4GAA4G;AAC5G,MAAM,WAAW,mBAAmB;IAClC,4GAA4G;IAC5G,IAAI,EAAE,MAAM,CAAC;IACb;;;;;;;;;;;;;OAaG;IACH,MAAM,EAAE,MAAM,CAAC;IACf,uDAAuD;IACvD,SAAS,EAAE,OAAO,CAAC;IACnB,2GAA2G;IAC3G,UAAU,EAAE,OAAO,CAAC;IACpB,qFAAqF;IACrF,KAAK,EAAE,UAAU,CAAC;CACnB;AAsED;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,iBAAiB,CAAC,GAAG,EAAE,OAAO,GAAG,mBAAmB,CA6BnE"}
|
|
@@ -11,41 +11,103 @@ const RETRYABLE = /* @__PURE__ */ new Set([
|
|
|
11
11
|
]);
|
|
12
12
|
/** Rate/quota codes map to the dedicated 429 error so the cause reads clearly in logs. */
|
|
13
13
|
const RATE_LIMITED = /* @__PURE__ */ new Set(["E_RATE_LIMIT_EXCEEDED", "E_DAILY_LIMIT_EXCEEDED"]);
|
|
14
|
-
/**
|
|
14
|
+
/**
|
|
15
|
+
* Codes that mean **this Worker could not authenticate** — terminal, because no retry clears a credential.
|
|
16
|
+
*
|
|
17
|
+
* `10000` is Cloudflare's own: a token that is alive but not scoped for Email Sending, or scoped for an
|
|
18
|
+
* account that does not own the sending domain, is answered `{ code: 10000, message: "Authentication
|
|
19
|
+
* error" }` (#534 captured the body; `liveSend.integration.test.ts` documents the same rejection for a
|
|
20
|
+
* general account token). That is the exact failure #555 spent five attempts on in silence.
|
|
21
|
+
*
|
|
22
|
+
* **Named rather than inferred from the code being numeric.** Treating every numeric code as terminal was
|
|
23
|
+
* the first shape of this fix and it bought the auth case by making a *momentary* numeric failure
|
|
24
|
+
* permanent too — one attempt, `status: "failed"`, where the retry budget used to apply. A code nobody
|
|
25
|
+
* here recognizes is still unknown, whichever alphabet it is written in, and unknown has earned its
|
|
26
|
+
* bounded retry since the module was written. So the auth class is listed, and the rule below is
|
|
27
|
+
* unchanged for everything else.
|
|
28
|
+
*/
|
|
29
|
+
const UNAUTHORIZED = /* @__PURE__ */ new Set([
|
|
30
|
+
"10000",
|
|
31
|
+
"E_UNAUTHORIZED",
|
|
32
|
+
"E_FORBIDDEN"
|
|
33
|
+
]);
|
|
34
|
+
/**
|
|
35
|
+
* Whether a code is one of Cloudflare's numeric ones rather than the binding's `E_*` spelling.
|
|
36
|
+
*
|
|
37
|
+
* The distinction decides the retry, so it is a predicate rather than a guess at the call site. An `E_*`
|
|
38
|
+
* code this module does not list stays **terminal**, exactly as it always has — the binding's documented
|
|
39
|
+
* unknowns are validation, sender, content and header faults, and retrying one cannot help. A *numeric*
|
|
40
|
+
* code this module does not list is retryable, because it arrives from the account/REST layer where a
|
|
41
|
+
* momentary fault is ordinary and nothing here has an opinion yet.
|
|
42
|
+
*/
|
|
43
|
+
function isProviderNumericCode(code) {
|
|
44
|
+
return /^[0-9]+$/.test(code);
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* Read a thrown value's `.code`/`.message` without assuming its shape.
|
|
48
|
+
*
|
|
49
|
+
* **A numeric code counts, and that is not a detail.** The Email Service binding surfaces `E_*` strings,
|
|
50
|
+
* but the REST path and the account layer behind it answer with Cloudflare's own numeric codes — an
|
|
51
|
+
* unauthorized send is `{ code: 10000, message: "Authentication error" }` (#534 captured the body). Read
|
|
52
|
+
* as "no code" that became `E_UNKNOWN`, which is both silent *and* retryable, so a credential fault
|
|
53
|
+
* consumed every attempt without ever naming itself (#555).
|
|
54
|
+
*
|
|
55
|
+
* **And a value with neither field still leaves a trace.** `String(err)` rather than `""`: an opaque
|
|
56
|
+
* throw is the case where the operator has least to go on, so an empty string is the one answer that
|
|
57
|
+
* cannot help. `E_UNKNOWN` remains the code — it means *nothing named itself*, which is the fact that
|
|
58
|
+
* earns the bounded retry.
|
|
59
|
+
*/
|
|
15
60
|
function readError(err) {
|
|
16
|
-
if (err && typeof err === "object")
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
61
|
+
if (err && typeof err === "object") {
|
|
62
|
+
const raw = "code" in err ? err.code : void 0;
|
|
63
|
+
const code = typeof raw === "string" && raw !== "" ? raw : typeof raw === "number" ? String(raw) : "E_UNKNOWN";
|
|
64
|
+
const named = "message" in err && typeof err.message === "string" ? err.message : "";
|
|
65
|
+
return {
|
|
66
|
+
code,
|
|
67
|
+
message: named !== "" ? named : code === "E_UNKNOWN" ? String(err) : ""
|
|
68
|
+
};
|
|
69
|
+
}
|
|
20
70
|
return {
|
|
21
71
|
code: "E_UNKNOWN",
|
|
22
|
-
message: typeof err === "string" ? err :
|
|
72
|
+
message: typeof err === "string" ? err : String(err)
|
|
23
73
|
};
|
|
24
74
|
}
|
|
25
75
|
/**
|
|
26
76
|
* Classify a send failure. `E_RECIPIENT_SUPPRESSED` is terminal and means the address must be
|
|
27
77
|
* suppressed locally too; rate/quota and transient delivery codes are retryable; everything else
|
|
28
|
-
* (validation, sender, content, headers) is terminal.
|
|
29
|
-
*
|
|
78
|
+
* (validation, sender, content, headers) is terminal.
|
|
79
|
+
*
|
|
80
|
+
* **An unrecognized code is retryable, and that is deliberately unchanged.** A failure this module has
|
|
81
|
+
* no code at all keeps its bounded retry, and so does a *numeric* provider code nobody here has listed —
|
|
82
|
+
* see {@link isProviderNumericCode} for why those two are alike and an unlisted `E_*` is not.
|
|
83
|
+
*
|
|
84
|
+
* What changed for #555 is that the **authentication** class is now named and terminal: `10000
|
|
85
|
+
* Authentication error` is a token that cannot send as this domain, and spending five attempts on it in
|
|
86
|
+
* silence is what hid the whole incident. See {@link UNAUTHORIZED} for why that is a list rather than
|
|
87
|
+
* "numeric means permanent" — the latter was this fix's first shape, and it made a momentary numeric
|
|
88
|
+
* failure permanent as a side effect.
|
|
30
89
|
*/
|
|
31
90
|
function classifySendError(err) {
|
|
32
91
|
const { code, message } = readError(err);
|
|
33
92
|
const detail = `email send failed: ${code}${message ? ` — ${message}` : ""}`;
|
|
34
93
|
if (code === "E_RECIPIENT_SUPPRESSED") return {
|
|
35
94
|
code,
|
|
95
|
+
detail,
|
|
36
96
|
retryable: false,
|
|
37
97
|
suppressed: true,
|
|
38
98
|
error: new EmailSuppressedError({ detail }, { cause: err })
|
|
39
99
|
};
|
|
40
100
|
if (RATE_LIMITED.has(code)) return {
|
|
41
101
|
code,
|
|
102
|
+
detail,
|
|
42
103
|
retryable: true,
|
|
43
104
|
suppressed: false,
|
|
44
105
|
error: new EmailRateLimitedError({ detail }, { cause: err })
|
|
45
106
|
};
|
|
46
107
|
return {
|
|
47
108
|
code,
|
|
48
|
-
|
|
109
|
+
detail,
|
|
110
|
+
retryable: RETRYABLE.has(code) || code === "E_UNKNOWN" || isProviderNumericCode(code) && !UNAUTHORIZED.has(code),
|
|
49
111
|
suppressed: false,
|
|
50
112
|
error: new EmailSendFailedError({ detail }, { cause: err })
|
|
51
113
|
};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"runSend.d.ts","sourceRoot":"","sources":["../../src/send/runSend.ts"],"names":[],"mappings":"AASA,OAAO,KAAK,EAAE,cAAc,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AACvE,OAAO,KAAK,EAAE,aAAa,EAAE,wBAAwB,EAAe,MAAM,gBAAgB,CAAC;AAE3F,OAAO,EAAE,KAAK,kBAAkB,EAAmC,MAAM,uBAAuB,CAAC;AACjG,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAGrD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AAG5C;;;;;;;;;;;;;;;;;;GAkBG;AAEH,kGAAkG;AAClG,MAAM,WAAW,WAAW;IAC1B,qCAAqC;IACrC,GAAG,EAAE,MAAM,CAAC;IACZ,oEAAoE;IACpE,GAAG,EAAE,MAAM,CAAC;CACb;AAED,yDAAyD;AACzD,MAAM,WAAW,QAAQ;IACvB,gDAAgD;IAChD,EAAE,EAAE,aAAa,CAAC;IAClB,sGAAsG;IACtG,aAAa,EAAE,wBAAwB,CAAC;IACxC,MAAM,EAAE,WAAW,CAAC;IACpB,KAAK,EAAE,UAAU,CAAC;IAClB;;;;;;;OAOG;IACH,SAAS,CAAC,EAAE,kBAAkB,CAAC;IAC/B,8CAA8C;IAC9C,OAAO,EAAE,MAAM,CAAC;IAChB,6GAA6G;IAC7G,OAAO,CAAC,EAAE,WAAW,CAAC;IACtB,2CAA2C;IAC3C,WAAW,EAAE,MAAM,CAAC;IACpB,qEAAqE;IACrE,WAAW,EAAE,MAAM,CAAC;IACpB,iHAAiH;IACjH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;;;;;;;;;;OAYG;IACH,aAAa,EAAE,IAAI,CAAC;IACpB;;;;;;;;;;;;;OAaG;IACH,WAAW,EAAE,MAAM,IAAI,CAAC;CACzB;AAED,qCAAqC;AACrC,MAAM,WAAW,WAAW;IAC1B,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,cAAc,CAAC;IACvB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB;;;;;;;OAOG;IACH,iBAAiB,CAAC,EAAE,iBAAiB,CAAC;CACvC;AAuBD,wBAAsB,OAAO,CAAC,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,
|
|
1
|
+
{"version":3,"file":"runSend.d.ts","sourceRoot":"","sources":["../../src/send/runSend.ts"],"names":[],"mappings":"AASA,OAAO,KAAK,EAAE,cAAc,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AACvE,OAAO,KAAK,EAAE,aAAa,EAAE,wBAAwB,EAAe,MAAM,gBAAgB,CAAC;AAE3F,OAAO,EAAE,KAAK,kBAAkB,EAAmC,MAAM,uBAAuB,CAAC;AACjG,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAGrD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AAG5C;;;;;;;;;;;;;;;;;;GAkBG;AAEH,kGAAkG;AAClG,MAAM,WAAW,WAAW;IAC1B,qCAAqC;IACrC,GAAG,EAAE,MAAM,CAAC;IACZ,oEAAoE;IACpE,GAAG,EAAE,MAAM,CAAC;CACb;AAED,yDAAyD;AACzD,MAAM,WAAW,QAAQ;IACvB,gDAAgD;IAChD,EAAE,EAAE,aAAa,CAAC;IAClB,sGAAsG;IACtG,aAAa,EAAE,wBAAwB,CAAC;IACxC,MAAM,EAAE,WAAW,CAAC;IACpB,KAAK,EAAE,UAAU,CAAC;IAClB;;;;;;;OAOG;IACH,SAAS,CAAC,EAAE,kBAAkB,CAAC;IAC/B,8CAA8C;IAC9C,OAAO,EAAE,MAAM,CAAC;IAChB,6GAA6G;IAC7G,OAAO,CAAC,EAAE,WAAW,CAAC;IACtB,2CAA2C;IAC3C,WAAW,EAAE,MAAM,CAAC;IACpB,qEAAqE;IACrE,WAAW,EAAE,MAAM,CAAC;IACpB,iHAAiH;IACjH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;;;;;;;;;;OAYG;IACH,aAAa,EAAE,IAAI,CAAC;IACpB;;;;;;;;;;;;;OAaG;IACH,WAAW,EAAE,MAAM,IAAI,CAAC;CACzB;AAED,qCAAqC;AACrC,MAAM,WAAW,WAAW;IAC1B,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,cAAc,CAAC;IACvB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB;;;;;;;OAOG;IACH,iBAAiB,CAAC,EAAE,iBAAiB,CAAC;CACvC;AAuBD,wBAAsB,OAAO,CAAC,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,CAoLjF"}
|
package/dist/send/runSend.js
CHANGED
|
@@ -197,18 +197,18 @@ async function runSend(deps, jobId) {
|
|
|
197
197
|
};
|
|
198
198
|
}
|
|
199
199
|
if (classified.retryable && attempts < deps.maxAttempts) {
|
|
200
|
-
await patchJob(deps, jobId, { error: classified.
|
|
200
|
+
await patchJob(deps, jobId, { error: classified.detail });
|
|
201
201
|
throw classified.error;
|
|
202
202
|
}
|
|
203
203
|
await patchJob(deps, jobId, {
|
|
204
204
|
status: "failed",
|
|
205
|
-
error: classified.
|
|
205
|
+
error: classified.detail
|
|
206
206
|
});
|
|
207
207
|
await recordEvent(deps.db, {
|
|
208
208
|
jobId,
|
|
209
209
|
recipient,
|
|
210
210
|
type: "failed",
|
|
211
|
-
detail: classified.
|
|
211
|
+
detail: classified.detail
|
|
212
212
|
}, deps.passStartedAt);
|
|
213
213
|
return {
|
|
214
214
|
jobId,
|
|
@@ -3,5 +3,5 @@
|
|
|
3
3
|
/** This package's npm name — the join key against a release feed. */
|
|
4
4
|
export declare const PACKAGE_NAME = "@pithy-sh/email";
|
|
5
5
|
/** This package's version, stamped from its own package.json at generation time. */
|
|
6
|
-
export declare const PACKAGE_VERSION = "0.
|
|
6
|
+
export declare const PACKAGE_VERSION = "0.3.0";
|
|
7
7
|
//# sourceMappingURL=version.generated.d.ts.map
|
|
@@ -4,6 +4,6 @@
|
|
|
4
4
|
/** This package's npm name — the join key against a release feed. */
|
|
5
5
|
const PACKAGE_NAME = "@pithy-sh/email";
|
|
6
6
|
/** This package's version, stamped from its own package.json at generation time. */
|
|
7
|
-
const PACKAGE_VERSION = "0.
|
|
7
|
+
const PACKAGE_VERSION = "0.3.0";
|
|
8
8
|
//#endregion
|
|
9
9
|
export { PACKAGE_NAME, PACKAGE_VERSION };
|
|
@@ -97,7 +97,7 @@ const EmailHostEnv = z.object({
|
|
|
97
97
|
ENVIRONMENT: z.string().min(1).optional().describe("The environment this host was deployed for. Stamped on every send as `X-Pithy-Env`, which is how the single inbound bounce worker attributes a bounce back to the environment that sent it."),
|
|
98
98
|
LINK_TTL_DAYS: tunedNumber(90).describe("How long a signed tracking or unsubscribe link stays valid, in days. Long by design: the link is minted into mail nobody can recall, and an unsubscribe that has expired is a complaint."),
|
|
99
99
|
MAX_ATTEMPTS: tunedNumber(5).describe("How many times one job may be attempted before it is failed terminally. Spent attempts are what separates a provider hiccup from an address that will never accept mail."),
|
|
100
|
-
SCHEDULER_ENABLED: z.enum(["true", "false"]).describe("The var as wrangler binds it — a string, and only these two spellings of it.").default("true").transform((value) => value === "true").describe("Whether the every-minute scheduler Workflow runs. `false` stops due jobs being claimed at all — the cron still fires and does
|
|
100
|
+
SCHEDULER_ENABLED: z.enum(["true", "false"]).describe("The var as wrangler binds it — a string, and only these two spellings of it.").default("true").transform((value) => value === "true").describe("Whether the every-minute scheduler Workflow runs. `false` stops due jobs being claimed at all — the cron still fires, and does not even look — so it is a maintenance switch, not a tuning knob."),
|
|
101
101
|
SCHEDULER_BATCH_SIZE: tunedNumber(50).describe("How many jobs one send Workflow instance takes. Each job is a durable step, so this is the unit of retry and of the instance's report."),
|
|
102
102
|
SCHEDULER_MAX_JOBS: tunedNumber(500).describe("The most rows one scheduler tick will claim. A ceiling on the work a single minute can start, so a backlog drains steadily rather than in one instance-creating burst."),
|
|
103
103
|
SCHEDULER_GRACE_MS: tunedNumber(12e4).describe("How long an immediate job is left alone before the safety net treats it as stranded, in milliseconds. The window a freshly-enqueued job's own dispatch has to land in; too short re-drives sends that were already on their way."),
|
|
@@ -6,8 +6,9 @@ import type { EmailDatabase } from "../data/tables";
|
|
|
6
6
|
* arrived, plus the safety net of `pending` and `undispatched` immediate jobs that never dispatched and
|
|
7
7
|
* `sending` jobs left stale by a dead dispatch — claims them (status → `sending`, so the next tick won't
|
|
8
8
|
* repick them), and fans them out into sender batches. **The fan-out scales with volume:** the more rows are due, the
|
|
9
|
-
* more batches are dispatched, each its own durable send Workflow.
|
|
10
|
-
*
|
|
9
|
+
* more batches are dispatched, each its own durable send Workflow. The cron probes for a due row
|
|
10
|
+
* first ({@link hasDueJobs}) and creates no instance at all when there is none, so an idle minute
|
|
11
|
+
* costs one read rather than a Workflow and a billed step (pithy-sh/pithy#538).
|
|
11
12
|
*
|
|
12
13
|
* Jobs are claimed before dispatch, so a dispatch failure strands a row in `sending` rather than
|
|
13
14
|
* double-sending; the `sending` re-drive recovers it on a later tick, and `runSend`'s idempotency makes
|
|
@@ -19,9 +20,16 @@ import type { EmailDatabase } from "../data/tables";
|
|
|
19
20
|
* the re-drive decision (pithy-sh/pithy#342), and it costs the claim one column and the tick one question
|
|
20
21
|
* per batch.
|
|
21
22
|
*/
|
|
22
|
-
/**
|
|
23
|
-
|
|
23
|
+
/**
|
|
24
|
+
* What it takes to ask whether anything is due — the database and the three numbers that define
|
|
25
|
+
* "due", and nothing else.
|
|
26
|
+
*
|
|
27
|
+
* It is the front half of {@link SchedulerDeps} rather than a type beside it, so the cron's probe and
|
|
28
|
+
* the tick cannot be pointed at different databases or measured against different windows.
|
|
29
|
+
*/
|
|
30
|
+
export interface DueProbeDeps {
|
|
24
31
|
db: EmailDatabase;
|
|
32
|
+
/** The instant this tick measures every cutoff from. */
|
|
25
33
|
now: Date;
|
|
26
34
|
/** How stale (ms) a `pending`/`undispatched` immediate job must be before the safety net re-drives it. */
|
|
27
35
|
graceMs: number;
|
|
@@ -38,6 +46,9 @@ export interface SchedulerDeps {
|
|
|
38
46
|
* slower horse, and the queue gets longer.
|
|
39
47
|
*/
|
|
40
48
|
stuckMs: number;
|
|
49
|
+
}
|
|
50
|
+
/** Inputs the scheduler needs. `dispatch` creates one send Workflow per batch. */
|
|
51
|
+
export interface SchedulerDeps extends DueProbeDeps {
|
|
41
52
|
/**
|
|
42
53
|
* Jobs per dispatched batch — one send Workflow each. From `SCHEDULER_BATCH_SIZE`.
|
|
43
54
|
*
|
|
@@ -79,6 +90,38 @@ export interface SchedulerResult {
|
|
|
79
90
|
/** How many stale-looking jobs were left alone because the batch holding them is still alive. */
|
|
80
91
|
held: number;
|
|
81
92
|
}
|
|
93
|
+
/**
|
|
94
|
+
* Is anything due? The question the every-minute cron asks **before** it creates a Workflow instance
|
|
95
|
+
* (pithy-sh/pithy#538).
|
|
96
|
+
*
|
|
97
|
+
* It used to be asked from inside the instance, so an idle minute cost a Workflow creation and a
|
|
98
|
+
* billed step to discover there was nothing to do — 1,440 of each per day per environment, against a
|
|
99
|
+
* Workers Free allowance of 3,000 steps a day for the whole account. Now an idle minute costs one
|
|
100
|
+
* indexed read and creates nothing.
|
|
101
|
+
*
|
|
102
|
+
* **It is deliberately the wider question.** It reads rows, not batches, so a stale row whose batch is
|
|
103
|
+
* still alive wakes the scheduler and the tick then declines to re-drive it — one instance, spent
|
|
104
|
+
* being right. The opposite asymmetry is the one that loses mail, which is why both sides share
|
|
105
|
+
* {@link isDue} rather than agreeing by inspection.
|
|
106
|
+
*/
|
|
107
|
+
export declare function hasDueJobs(deps: DueProbeDeps): Promise<boolean>;
|
|
108
|
+
/**
|
|
109
|
+
* The whole of what the cron asks before it creates anything: **is this scheduler configured, and is
|
|
110
|
+
* anything due** — in that order (pithy-sh/pithy#538).
|
|
111
|
+
*
|
|
112
|
+
* The order is the point. {@link assertBatchSize} was written to run before the tick does any work, so
|
|
113
|
+
* that a misconfigured worker says so on its first tick rather than on its first busy one; the probe
|
|
114
|
+
* that now stands in front of the tick would have postponed it to the first minute a job came due. An
|
|
115
|
+
* idle cron that quietly accepts a broken batch size is the shape of the bug this file has already
|
|
116
|
+
* been bitten by twice, and "quietly" is what the probe would have restored.
|
|
117
|
+
*
|
|
118
|
+
* It takes the tick's own {@link SchedulerDeps} rather than the probe's {@link DueProbeDeps}, because
|
|
119
|
+
* the question is about the configuration the tick would run under. The email host validates the same
|
|
120
|
+
* number a second way — `SCHEDULER_BATCH_SIZE` is coerced and refused by `hostEnv.ts` before this is
|
|
121
|
+
* reached — so on that path this is a second lock on one door. It is the module's own invariant, and
|
|
122
|
+
* it holds for any caller that assembles deps itself.
|
|
123
|
+
*/
|
|
124
|
+
export declare function schedulerHasWork(deps: SchedulerDeps): Promise<boolean>;
|
|
82
125
|
/** Run one scheduler tick: find due rows, claim them, fan out batches. */
|
|
83
126
|
export declare function runScheduler(deps: SchedulerDeps): Promise<SchedulerResult>;
|
|
84
127
|
//# sourceMappingURL=scheduler.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"scheduler.d.ts","sourceRoot":"","sources":["../../src/workflows/scheduler.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"scheduler.d.ts","sourceRoot":"","sources":["../../src/workflows/scheduler.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAE,aAAa,EAAe,MAAM,gBAAgB,CAAC;AAEjE;;;;;;;;;;;;;;;;;;GAkBG;AAEH;;;;;;GAMG;AACH,MAAM,WAAW,YAAY;IAC3B,EAAE,EAAE,aAAa,CAAC;IAClB,wDAAwD;IACxD,GAAG,EAAE,IAAI,CAAC;IACV,0GAA0G;IAC1G,OAAO,EAAE,MAAM,CAAC;IAChB;;;;;;;;;;;OAWG;IACH,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,kFAAkF;AAClF,MAAM,WAAW,aAAc,SAAQ,YAAY;IACjD;;;;;;OAMG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB,0CAA0C;IAC1C,OAAO,EAAE,MAAM,CAAC;IAChB;;;OAGG;IACH,UAAU,EAAE,MAAM,MAAM,CAAC;IACzB;;;;;;;;;;;;OAYG;IACH,YAAY,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;IACpD,8FAA8F;IAC9F,QAAQ,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CAChE;AAED,mCAAmC;AACnC,MAAM,WAAW,eAAe;IAC9B,sCAAsC;IACtC,GAAG,EAAE,MAAM,CAAC;IACZ,6DAA6D;IAC7D,OAAO,EAAE,MAAM,CAAC;IAChB,iGAAiG;IACjG,IAAI,EAAE,MAAM,CAAC;CACd;AAuHD;;;;;;;;;;;;;GAaG;AACH,wBAAsB,UAAU,CAAC,IAAI,EAAE,YAAY,GAAG,OAAO,CAAC,OAAO,CAAC,CAQrE;AAED;;;;;;;;;;;;;;;GAeG;AACH,wBAAsB,gBAAgB,CAAC,IAAI,EAAE,aAAa,GAAG,OAAO,CAAC,OAAO,CAAC,CAG5E;AAED,0EAA0E;AAC1E,wBAAsB,YAAY,CAAC,IAAI,EAAE,aAAa,GAAG,OAAO,CAAC,eAAe,CAAC,CA0ChF"}
|
|
@@ -68,17 +68,76 @@ async function strandedIds(deps, rows) {
|
|
|
68
68
|
}
|
|
69
69
|
return stranded;
|
|
70
70
|
}
|
|
71
|
+
/** Derive this tick's cutoffs. The tick claims against the same `nowMs` it selected against. */
|
|
72
|
+
function dueCutoffs(deps) {
|
|
73
|
+
const nowMs = deps.now.getTime();
|
|
74
|
+
return {
|
|
75
|
+
nowMs,
|
|
76
|
+
graceCutoff: nowMs - deps.graceMs,
|
|
77
|
+
stuckCutoff: nowMs - deps.stuckMs
|
|
78
|
+
};
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* **"Due" — the one definition.** Three branches over four statuses: a `scheduled` job whose `sendAt`
|
|
82
|
+
* has arrived, a `pending` or `undispatched` immediate job past its grace, and a `sending` job past
|
|
83
|
+
* `stuckMs`.
|
|
84
|
+
*
|
|
85
|
+
* It is a function rather than a clause typed into a query because two callers ask it now: the tick
|
|
86
|
+
* below, and the cron's probe ({@link hasDueJobs}) that decides whether to create the tick at all
|
|
87
|
+
* (pithy-sh/pithy#538). A second copy would be a second definition, and the drift is silent in the
|
|
88
|
+
* worst way — a probe that stops matching a branch never wakes the scheduler for those rows, and they
|
|
89
|
+
* sit there with nothing logged. This file has been bitten twice by that shape already: the `NaN`
|
|
90
|
+
* batch size that produced one empty batch a minute, and the `undispatched` dead end (#410).
|
|
91
|
+
*/
|
|
92
|
+
function isDue(cutoffs) {
|
|
93
|
+
return (eb) => eb.or([
|
|
94
|
+
eb.and([eb("status", "=", "scheduled"), eb("sendAt", "<=", cutoffs.nowMs)]),
|
|
95
|
+
eb.and([eb("status", "in", ["pending", "undispatched"]), eb("createdAt", "<=", cutoffs.graceCutoff)]),
|
|
96
|
+
eb.and([eb("status", "=", "sending"), eb("updatedAt", "<=", cutoffs.stuckCutoff)])
|
|
97
|
+
]);
|
|
98
|
+
}
|
|
99
|
+
/**
|
|
100
|
+
* Is anything due? The question the every-minute cron asks **before** it creates a Workflow instance
|
|
101
|
+
* (pithy-sh/pithy#538).
|
|
102
|
+
*
|
|
103
|
+
* It used to be asked from inside the instance, so an idle minute cost a Workflow creation and a
|
|
104
|
+
* billed step to discover there was nothing to do — 1,440 of each per day per environment, against a
|
|
105
|
+
* Workers Free allowance of 3,000 steps a day for the whole account. Now an idle minute costs one
|
|
106
|
+
* indexed read and creates nothing.
|
|
107
|
+
*
|
|
108
|
+
* **It is deliberately the wider question.** It reads rows, not batches, so a stale row whose batch is
|
|
109
|
+
* still alive wakes the scheduler and the tick then declines to re-drive it — one instance, spent
|
|
110
|
+
* being right. The opposite asymmetry is the one that loses mail, which is why both sides share
|
|
111
|
+
* {@link isDue} rather than agreeing by inspection.
|
|
112
|
+
*/
|
|
113
|
+
async function hasDueJobs(deps) {
|
|
114
|
+
return await deps.db.selectFrom("pithyEmailJobs").select("id").where(isDue(dueCutoffs(deps))).limit(1).executeTakeFirst() !== void 0;
|
|
115
|
+
}
|
|
116
|
+
/**
|
|
117
|
+
* The whole of what the cron asks before it creates anything: **is this scheduler configured, and is
|
|
118
|
+
* anything due** — in that order (pithy-sh/pithy#538).
|
|
119
|
+
*
|
|
120
|
+
* The order is the point. {@link assertBatchSize} was written to run before the tick does any work, so
|
|
121
|
+
* that a misconfigured worker says so on its first tick rather than on its first busy one; the probe
|
|
122
|
+
* that now stands in front of the tick would have postponed it to the first minute a job came due. An
|
|
123
|
+
* idle cron that quietly accepts a broken batch size is the shape of the bug this file has already
|
|
124
|
+
* been bitten by twice, and "quietly" is what the probe would have restored.
|
|
125
|
+
*
|
|
126
|
+
* It takes the tick's own {@link SchedulerDeps} rather than the probe's {@link DueProbeDeps}, because
|
|
127
|
+
* the question is about the configuration the tick would run under. The email host validates the same
|
|
128
|
+
* number a second way — `SCHEDULER_BATCH_SIZE` is coerced and refused by `hostEnv.ts` before this is
|
|
129
|
+
* reached — so on that path this is a second lock on one door. It is the module's own invariant, and
|
|
130
|
+
* it holds for any caller that assembles deps itself.
|
|
131
|
+
*/
|
|
132
|
+
async function schedulerHasWork(deps) {
|
|
133
|
+
assertBatchSize(deps.batchSize);
|
|
134
|
+
return await hasDueJobs(deps);
|
|
135
|
+
}
|
|
71
136
|
/** Run one scheduler tick: find due rows, claim them, fan out batches. */
|
|
72
137
|
async function runScheduler(deps) {
|
|
73
138
|
assertBatchSize(deps.batchSize);
|
|
74
|
-
const
|
|
75
|
-
const
|
|
76
|
-
const stuckCutoff = nowMs - deps.stuckMs;
|
|
77
|
-
const rows = await deps.db.selectFrom("pithyEmailJobs").select(["id", "batchId"]).where((eb) => eb.or([
|
|
78
|
-
eb.and([eb("status", "=", "scheduled"), eb("sendAt", "<=", nowMs)]),
|
|
79
|
-
eb.and([eb("status", "in", ["pending", "undispatched"]), eb("createdAt", "<=", graceCutoff)]),
|
|
80
|
-
eb.and([eb("status", "=", "sending"), eb("updatedAt", "<=", stuckCutoff)])
|
|
81
|
-
])).orderBy("sendAt", "asc").limit(deps.maxJobs).execute();
|
|
139
|
+
const cutoffs = dueCutoffs(deps);
|
|
140
|
+
const rows = await deps.db.selectFrom("pithyEmailJobs").select(["id", "batchId"]).where(isDue(cutoffs)).orderBy("sendAt", "asc").limit(deps.maxJobs).execute();
|
|
82
141
|
if (rows.length === 0) return {
|
|
83
142
|
due: 0,
|
|
84
143
|
batches: 0,
|
|
@@ -97,7 +156,7 @@ async function runScheduler(deps) {
|
|
|
97
156
|
const batchId = deps.newBatchId();
|
|
98
157
|
for (const claim of chunkByBoundParameters(batch, CLAIM_FIXED_PARAMETERS)) await deps.db.updateTable("pithyEmailJobs").set({
|
|
99
158
|
status: "sending",
|
|
100
|
-
updatedAt: nowMs,
|
|
159
|
+
updatedAt: cutoffs.nowMs,
|
|
101
160
|
batchId
|
|
102
161
|
}).where("id", "in", claim).execute();
|
|
103
162
|
await deps.dispatch(batchId, batch);
|
|
@@ -110,4 +169,4 @@ async function runScheduler(deps) {
|
|
|
110
169
|
};
|
|
111
170
|
}
|
|
112
171
|
//#endregion
|
|
113
|
-
export { runScheduler };
|
|
172
|
+
export { hasDueJobs, runScheduler, schedulerHasWork };
|
|
@@ -14,7 +14,7 @@ import { type BatchSendReport } from "./sendBatch";
|
|
|
14
14
|
* - `EmailSendWorkflow` — sends a batch of jobs durably (dispatch target for immediate sends and
|
|
15
15
|
* the scheduler's fan-out).
|
|
16
16
|
* - `EmailSchedulerWorkflow` — finds due jobs and fans them out into send batches.
|
|
17
|
-
* - `scheduled()` — the every-minute cron that fires the scheduler Workflow.
|
|
17
|
+
* - `scheduled()` — the every-minute cron that fires the scheduler Workflow when a row is due.
|
|
18
18
|
* - `fetch()` — the loopback dispatch door, served only in `dev` (see {@link createEmailHostApp}).
|
|
19
19
|
*
|
|
20
20
|
* The bodies (`runSendBatch`, `runScheduler`, `runSend`) are tested against Miniflare; these classes are
|
|
@@ -85,7 +85,28 @@ export declare class EmailSchedulerWorkflow extends WorkflowEntrypoint<EmailWork
|
|
|
85
85
|
run(_event: WorkflowEvent<unknown>, step: WorkflowStep): Promise<void>;
|
|
86
86
|
}
|
|
87
87
|
declare const _default: {
|
|
88
|
-
/**
|
|
88
|
+
/**
|
|
89
|
+
* Cron entry: fire the scheduler Workflow every minute — when there is something to fire it for.
|
|
90
|
+
*
|
|
91
|
+
* The instance used to be created unconditionally, and the "is anything due" question was asked
|
|
92
|
+
* inside it. At `* * * * *` that spends 1,440 instance creations and 1,440 billed steps a day, per
|
|
93
|
+
* environment, discovering nothing to do; Workers Free allows 3,000 steps a day for the whole
|
|
94
|
+
* account, so a default project's staging and prod burn 96% of it sending no mail, and the sends
|
|
95
|
+
* that do happen compete for the rest (pithy-sh/pithy#538).
|
|
96
|
+
*
|
|
97
|
+
* So the question moved out in front: one indexed read on `pithy_email_jobs`, asked with the tick's
|
|
98
|
+
* own predicate rather than a second copy of it, and an instance only when it finds a row. Nothing
|
|
99
|
+
* is lost with the tick that never starts: this is not the path an immediate send takes (`enqueue`
|
|
100
|
+
* starts that Workflow itself), the work is a pure re-derivation from D1, and a lost minute is
|
|
101
|
+
* recovered by the next one exactly as a throwing `create()` has always been.
|
|
102
|
+
*
|
|
103
|
+
* What it does give up is the per-minute instance in the dashboard that proved the scheduler was
|
|
104
|
+
* alive. An idle minute now leaves no trace, by design — the trace was costing a billed step.
|
|
105
|
+
*
|
|
106
|
+
* **Through {@link schedulerHasWork}, so the tick's configuration is checked before the probe.** The
|
|
107
|
+
* batch-size check inside `runScheduler` exists to complain on the first tick rather than the first
|
|
108
|
+
* busy one, and a probe in front of the instance is in front of that check too.
|
|
109
|
+
*/
|
|
89
110
|
scheduled(_controller: unknown, env: EmailWorkerEnv): Promise<void>;
|
|
90
111
|
/**
|
|
91
112
|
* The loopback dispatch door — how a sibling worker under `pithy dev` starts a send batch on this
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"worker.d.ts","sourceRoot":"","sources":["../../src/workflows/worker.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,kBAAkB,EAAE,KAAK,aAAa,EAAE,KAAK,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAE/F,OAAO,KAAK,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,2BAA2B,CAAC;AAG9E,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,oCAAoC,CAAC;AAK1E,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AAE3D,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAKlD,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AAEzD,OAAO,EAAE,KAAK,eAAe,EAAoC,MAAM,aAAa,CAAC;AAErF;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH;;;;;;GAMG;AACH,MAAM,WAAW,cAAe,SAAQ,eAAe;IACrD,uEAAuE;IACvE,EAAE,EAAE,UAAU,CAAC;IACf,gDAAgD;IAChD,kBAAkB,EAAE,UAAU,CAAC;IAC/B,iDAAiD;IACjD,KAAK,EAAE,WAAW,CAAC;IACnB,gGAAgG;IAChG,YAAY,EAAE,mBAAmB,GAAG,qBAAqB,CAAC;IAC1D,yDAAyD;IACzD,eAAe,EAAE;QAAE,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,CAAA;KAAE,CAAC;IAChD,+GAA+G;IAC/G,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;OAGG;IACH,CAAC,UAAU,EAAE,kBAAkB,MAAM,EAAE,GAAG,OAAO,CAAC;IAClD,QAAQ,EAAE,MAAM,CAAC;IAEjB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,kBAAkB,CAAC,EAAE,MAAM,CAAC;CAC7B;
|
|
1
|
+
{"version":3,"file":"worker.d.ts","sourceRoot":"","sources":["../../src/workflows/worker.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,kBAAkB,EAAE,KAAK,aAAa,EAAE,KAAK,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAE/F,OAAO,KAAK,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,2BAA2B,CAAC;AAG9E,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,oCAAoC,CAAC;AAK1E,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AAE3D,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAKlD,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AAEzD,OAAO,EAAE,KAAK,eAAe,EAAoC,MAAM,aAAa,CAAC;AAErF;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH;;;;;;GAMG;AACH,MAAM,WAAW,cAAe,SAAQ,eAAe;IACrD,uEAAuE;IACvE,EAAE,EAAE,UAAU,CAAC;IACf,gDAAgD;IAChD,kBAAkB,EAAE,UAAU,CAAC;IAC/B,iDAAiD;IACjD,KAAK,EAAE,WAAW,CAAC;IACnB,gGAAgG;IAChG,YAAY,EAAE,mBAAmB,GAAG,qBAAqB,CAAC;IAC1D,yDAAyD;IACzD,eAAe,EAAE;QAAE,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,CAAA;KAAE,CAAC;IAChD,+GAA+G;IAC/G,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;OAGG;IACH,CAAC,UAAU,EAAE,kBAAkB,MAAM,EAAE,GAAG,OAAO,CAAC;IAClD,QAAQ,EAAE,MAAM,CAAC;IAEjB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,kBAAkB,CAAC,EAAE,MAAM,CAAC;CAC7B;AA2ED;;;;;;;GAOG;AACH,qBAAa,iBAAkB,SAAQ,kBAAkB,CAAC,cAAc,EAAE;IAAE,MAAM,EAAE,MAAM,EAAE,CAAA;CAAE,CAAC;IAI9E,GAAG,CAAC,KAAK,EAAE,aAAa,CAAC;QAAE,MAAM,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC,EAAE,IAAI,EAAE,YAAY,GAAG,OAAO,CAAC,eAAe,CAAC,CAM3G;CACF;AAED,0FAA0F;AAC1F,qBAAa,sBAAuB,SAAQ,kBAAkB,CAAC,cAAc,EAAE,OAAO,CAAC;IACtE,GAAG,CAAC,MAAM,EAAE,aAAa,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAIpF;CACF;;IAGC;;;;;;;;;;;;;;;;;;;;;OAqBG;IACG,SAAS,cAAc,OAAO,OAAO,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC;IAOzE;;;;;;;;OAQG;IACG,KAAK,UAAU,OAAO,OAAO,cAAc,OAAO,gBAAgB,GAAG,OAAO,CAAC,QAAQ,CAAC"}
|
package/dist/workflows/worker.js
CHANGED
|
@@ -8,7 +8,7 @@ import { emailHostEnv } from "./hostEnv.js";
|
|
|
8
8
|
import { emailWorkflowRetry } from "../send/retryPolicy.js";
|
|
9
9
|
import { createEmailHostApp } from "./hostApp.js";
|
|
10
10
|
import { isLiveInstanceStatus } from "./instanceLiveness.js";
|
|
11
|
-
import { runScheduler } from "./scheduler.js";
|
|
11
|
+
import { runScheduler, schedulerHasWork } from "./scheduler.js";
|
|
12
12
|
import { runSendBatch } from "./sendBatch.js";
|
|
13
13
|
import { configureSharedSecrets } from "@pithy-sh/secrets/src/sharedSecretsStore";
|
|
14
14
|
import { requireHostEnv } from "@pithy-sh/core/src/workflow/hostEnv";
|
|
@@ -48,9 +48,14 @@ async function buildSendDeps(env) {
|
|
|
48
48
|
heartbeatAt: () => /* @__PURE__ */ new Date()
|
|
49
49
|
};
|
|
50
50
|
}
|
|
51
|
-
/**
|
|
52
|
-
|
|
53
|
-
|
|
51
|
+
/**
|
|
52
|
+
* Assemble the scheduler dependencies, dispatching each batch as a send Workflow.
|
|
53
|
+
*
|
|
54
|
+
* `config` is a parameter so the cron can validate the env once and hand the answer on: it reads
|
|
55
|
+
* `SCHEDULER_ENABLED` before it probes, and re-parsing the same env object a second line later would
|
|
56
|
+
* be one Zod pass per minute for nothing.
|
|
57
|
+
*/
|
|
58
|
+
function buildSchedulerDeps(env, config = hostConfig(env)) {
|
|
54
59
|
return {
|
|
55
60
|
db: emailDatabase(env.DB),
|
|
56
61
|
now: /* @__PURE__ */ new Date(),
|
|
@@ -97,9 +102,33 @@ var EmailSchedulerWorkflow = class extends WorkflowEntrypoint {
|
|
|
97
102
|
}
|
|
98
103
|
};
|
|
99
104
|
var worker_default = {
|
|
100
|
-
/**
|
|
105
|
+
/**
|
|
106
|
+
* Cron entry: fire the scheduler Workflow every minute — when there is something to fire it for.
|
|
107
|
+
*
|
|
108
|
+
* The instance used to be created unconditionally, and the "is anything due" question was asked
|
|
109
|
+
* inside it. At `* * * * *` that spends 1,440 instance creations and 1,440 billed steps a day, per
|
|
110
|
+
* environment, discovering nothing to do; Workers Free allows 3,000 steps a day for the whole
|
|
111
|
+
* account, so a default project's staging and prod burn 96% of it sending no mail, and the sends
|
|
112
|
+
* that do happen compete for the rest (pithy-sh/pithy#538).
|
|
113
|
+
*
|
|
114
|
+
* So the question moved out in front: one indexed read on `pithy_email_jobs`, asked with the tick's
|
|
115
|
+
* own predicate rather than a second copy of it, and an instance only when it finds a row. Nothing
|
|
116
|
+
* is lost with the tick that never starts: this is not the path an immediate send takes (`enqueue`
|
|
117
|
+
* starts that Workflow itself), the work is a pure re-derivation from D1, and a lost minute is
|
|
118
|
+
* recovered by the next one exactly as a throwing `create()` has always been.
|
|
119
|
+
*
|
|
120
|
+
* What it does give up is the per-minute instance in the dashboard that proved the scheduler was
|
|
121
|
+
* alive. An idle minute now leaves no trace, by design — the trace was costing a billed step.
|
|
122
|
+
*
|
|
123
|
+
* **Through {@link schedulerHasWork}, so the tick's configuration is checked before the probe.** The
|
|
124
|
+
* batch-size check inside `runScheduler` exists to complain on the first tick rather than the first
|
|
125
|
+
* busy one, and a probe in front of the instance is in front of that check too.
|
|
126
|
+
*/
|
|
101
127
|
async scheduled(_controller, env) {
|
|
102
|
-
|
|
128
|
+
const config = hostConfig(env);
|
|
129
|
+
if (!config.SCHEDULER_ENABLED) return;
|
|
130
|
+
if (!await schedulerHasWork(buildSchedulerDeps(env, config))) return;
|
|
131
|
+
await env.EMAIL_SCHEDULER.create();
|
|
103
132
|
},
|
|
104
133
|
/**
|
|
105
134
|
* The loopback dispatch door — how a sibling worker under `pithy dev` starts a send batch on this
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@pithy-sh/email",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.3.0",
|
|
4
4
|
"license": "MIT",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -38,14 +38,21 @@
|
|
|
38
38
|
"dependencies": {
|
|
39
39
|
"@cloudflare/workers-types": "^5.20260729.1",
|
|
40
40
|
"@hono/zod-validator": "^0.9.0",
|
|
41
|
-
"@pithy-sh/core": "^0.3.0",
|
|
42
|
-
"@pithy-sh/secrets": "^0.1.6",
|
|
43
41
|
"handlebars": "^4.7.9",
|
|
44
42
|
"postal-mime": "^3.0.0"
|
|
45
43
|
},
|
|
44
|
+
"peerDependencies": {
|
|
45
|
+
"@pithy-sh/core": "^0.4.0",
|
|
46
|
+
"@pithy-sh/secrets": "^0.2.0",
|
|
47
|
+
"hono": "^4.13.2",
|
|
48
|
+
"kysely": "^0.29.0",
|
|
49
|
+
"zod": "^4.4.0"
|
|
50
|
+
},
|
|
46
51
|
"devDependencies": {
|
|
47
52
|
"@cloudflare/vitest-plugin": "^1.0.0",
|
|
48
53
|
"@pithy-sh/cloudflare": "workspace:*",
|
|
54
|
+
"@pithy-sh/core": "workspace:*",
|
|
55
|
+
"@pithy-sh/secrets": "workspace:*",
|
|
49
56
|
"@pithy-sh/tsconfig": "workspace:*",
|
|
50
57
|
"@types/node": "^22.15.0",
|
|
51
58
|
"@vitest/coverage-v8": "^4.1.0",
|
|
@@ -57,10 +64,5 @@
|
|
|
57
64
|
"vitest": "^4.1.0",
|
|
58
65
|
"wrangler": "^4.115.0",
|
|
59
66
|
"zod": "^4.4.0"
|
|
60
|
-
},
|
|
61
|
-
"peerDependencies": {
|
|
62
|
-
"hono": "^4.13.2",
|
|
63
|
-
"kysely": "^0.29.0",
|
|
64
|
-
"zod": "^4.4.0"
|
|
65
67
|
}
|
|
66
68
|
}
|
package/src/send/errorMapping.ts
CHANGED
|
@@ -14,8 +14,23 @@ import { EmailRateLimitedError, EmailSendFailedError, EmailSuppressedError } fro
|
|
|
14
14
|
|
|
15
15
|
/** How a given send error is handled: which Pithy error to raise, and whether the send Workflow retries. */
|
|
16
16
|
export interface ClassifiedSendError {
|
|
17
|
-
/** The `E_*` code reported by the binding, or `E_UNKNOWN`
|
|
17
|
+
/** The `E_*` code reported by the binding, a numeric provider code as a string, or `E_UNKNOWN` for none. */
|
|
18
18
|
code: string;
|
|
19
|
+
/**
|
|
20
|
+
* The cause, as one operator-facing line: the code and whatever the provider said with it.
|
|
21
|
+
*
|
|
22
|
+
* **Exposed because the job row needs it and only the error object had it (pithy-sh/pithy#555).**
|
|
23
|
+
* `runSend` persisted the bare `code` into `pithy_email_jobs.error`, so a magic link that failed five
|
|
24
|
+
* times against a token for the wrong account recorded `E_UNKNOWN` five times and nothing about why —
|
|
25
|
+
* while this exact string, naming `10000 Authentication error`, was being built two lines down and
|
|
26
|
+
* thrown away with the error it decorated. Same family as #534: a failure that discards the cause it
|
|
27
|
+
* was handed.
|
|
28
|
+
*
|
|
29
|
+
* Identical to `error.payload.detail`, deliberately — one composition, two readers, so the row and the
|
|
30
|
+
* log cannot drift. `detail` is stripped at the client boundary, which is where a provider's own words
|
|
31
|
+
* belong.
|
|
32
|
+
*/
|
|
33
|
+
detail: string;
|
|
19
34
|
/** Whether the send should be retried with backoff. */
|
|
20
35
|
retryable: boolean;
|
|
21
36
|
/** Whether the failure means the recipient is suppressed (so the address should be locally suppressed). */
|
|
@@ -35,21 +50,77 @@ const RETRYABLE = new Set([
|
|
|
35
50
|
/** Rate/quota codes map to the dedicated 429 error so the cause reads clearly in logs. */
|
|
36
51
|
const RATE_LIMITED = new Set(["E_RATE_LIMIT_EXCEEDED", "E_DAILY_LIMIT_EXCEEDED"]);
|
|
37
52
|
|
|
38
|
-
/**
|
|
53
|
+
/**
|
|
54
|
+
* Codes that mean **this Worker could not authenticate** — terminal, because no retry clears a credential.
|
|
55
|
+
*
|
|
56
|
+
* `10000` is Cloudflare's own: a token that is alive but not scoped for Email Sending, or scoped for an
|
|
57
|
+
* account that does not own the sending domain, is answered `{ code: 10000, message: "Authentication
|
|
58
|
+
* error" }` (#534 captured the body; `liveSend.integration.test.ts` documents the same rejection for a
|
|
59
|
+
* general account token). That is the exact failure #555 spent five attempts on in silence.
|
|
60
|
+
*
|
|
61
|
+
* **Named rather than inferred from the code being numeric.** Treating every numeric code as terminal was
|
|
62
|
+
* the first shape of this fix and it bought the auth case by making a *momentary* numeric failure
|
|
63
|
+
* permanent too — one attempt, `status: "failed"`, where the retry budget used to apply. A code nobody
|
|
64
|
+
* here recognizes is still unknown, whichever alphabet it is written in, and unknown has earned its
|
|
65
|
+
* bounded retry since the module was written. So the auth class is listed, and the rule below is
|
|
66
|
+
* unchanged for everything else.
|
|
67
|
+
*/
|
|
68
|
+
const UNAUTHORIZED = new Set(["10000", "E_UNAUTHORIZED", "E_FORBIDDEN"]);
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* Whether a code is one of Cloudflare's numeric ones rather than the binding's `E_*` spelling.
|
|
72
|
+
*
|
|
73
|
+
* The distinction decides the retry, so it is a predicate rather than a guess at the call site. An `E_*`
|
|
74
|
+
* code this module does not list stays **terminal**, exactly as it always has — the binding's documented
|
|
75
|
+
* unknowns are validation, sender, content and header faults, and retrying one cannot help. A *numeric*
|
|
76
|
+
* code this module does not list is retryable, because it arrives from the account/REST layer where a
|
|
77
|
+
* momentary fault is ordinary and nothing here has an opinion yet.
|
|
78
|
+
*/
|
|
79
|
+
function isProviderNumericCode(code: string): boolean {
|
|
80
|
+
return /^[0-9]+$/.test(code);
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* Read a thrown value's `.code`/`.message` without assuming its shape.
|
|
85
|
+
*
|
|
86
|
+
* **A numeric code counts, and that is not a detail.** The Email Service binding surfaces `E_*` strings,
|
|
87
|
+
* but the REST path and the account layer behind it answer with Cloudflare's own numeric codes — an
|
|
88
|
+
* unauthorized send is `{ code: 10000, message: "Authentication error" }` (#534 captured the body). Read
|
|
89
|
+
* as "no code" that became `E_UNKNOWN`, which is both silent *and* retryable, so a credential fault
|
|
90
|
+
* consumed every attempt without ever naming itself (#555).
|
|
91
|
+
*
|
|
92
|
+
* **And a value with neither field still leaves a trace.** `String(err)` rather than `""`: an opaque
|
|
93
|
+
* throw is the case where the operator has least to go on, so an empty string is the one answer that
|
|
94
|
+
* cannot help. `E_UNKNOWN` remains the code — it means *nothing named itself*, which is the fact that
|
|
95
|
+
* earns the bounded retry.
|
|
96
|
+
*/
|
|
39
97
|
function readError(err: unknown): { code: string; message: string } {
|
|
40
98
|
if (err && typeof err === "object") {
|
|
41
|
-
const
|
|
42
|
-
const
|
|
43
|
-
|
|
99
|
+
const raw = "code" in err ? err.code : undefined;
|
|
100
|
+
const code = typeof raw === "string" && raw !== "" ? raw : typeof raw === "number" ? String(raw) : "E_UNKNOWN";
|
|
101
|
+
// `String(err)` only where nothing named itself. A coded throw carrying no message would otherwise
|
|
102
|
+
// record `E_INVALID_SENDER — [object Object]`, which is the code the operator already had plus noise —
|
|
103
|
+
// and that string is what the job row and the admin detail route now read.
|
|
104
|
+
const named = "message" in err && typeof err.message === "string" ? err.message : "";
|
|
105
|
+
return { code, message: named !== "" ? named : code === "E_UNKNOWN" ? String(err) : "" };
|
|
44
106
|
}
|
|
45
|
-
return { code: "E_UNKNOWN", message: typeof err === "string" ? err :
|
|
107
|
+
return { code: "E_UNKNOWN", message: typeof err === "string" ? err : String(err) };
|
|
46
108
|
}
|
|
47
109
|
|
|
48
110
|
/**
|
|
49
111
|
* Classify a send failure. `E_RECIPIENT_SUPPRESSED` is terminal and means the address must be
|
|
50
112
|
* suppressed locally too; rate/quota and transient delivery codes are retryable; everything else
|
|
51
|
-
* (validation, sender, content, headers) is terminal.
|
|
52
|
-
*
|
|
113
|
+
* (validation, sender, content, headers) is terminal.
|
|
114
|
+
*
|
|
115
|
+
* **An unrecognized code is retryable, and that is deliberately unchanged.** A failure this module has
|
|
116
|
+
* no code at all keeps its bounded retry, and so does a *numeric* provider code nobody here has listed —
|
|
117
|
+
* see {@link isProviderNumericCode} for why those two are alike and an unlisted `E_*` is not.
|
|
118
|
+
*
|
|
119
|
+
* What changed for #555 is that the **authentication** class is now named and terminal: `10000
|
|
120
|
+
* Authentication error` is a token that cannot send as this domain, and spending five attempts on it in
|
|
121
|
+
* silence is what hid the whole incident. See {@link UNAUTHORIZED} for why that is a list rather than
|
|
122
|
+
* "numeric means permanent" — the latter was this fix's first shape, and it made a momentary numeric
|
|
123
|
+
* failure permanent as a side effect.
|
|
53
124
|
*/
|
|
54
125
|
export function classifySendError(err: unknown): ClassifiedSendError {
|
|
55
126
|
const { code, message } = readError(err);
|
|
@@ -58,6 +129,7 @@ export function classifySendError(err: unknown): ClassifiedSendError {
|
|
|
58
129
|
if (code === "E_RECIPIENT_SUPPRESSED") {
|
|
59
130
|
return {
|
|
60
131
|
code,
|
|
132
|
+
detail,
|
|
61
133
|
retryable: false,
|
|
62
134
|
suppressed: true,
|
|
63
135
|
error: new EmailSuppressedError({ detail }, { cause: err }),
|
|
@@ -65,9 +137,18 @@ export function classifySendError(err: unknown): ClassifiedSendError {
|
|
|
65
137
|
}
|
|
66
138
|
|
|
67
139
|
if (RATE_LIMITED.has(code)) {
|
|
68
|
-
return {
|
|
140
|
+
return {
|
|
141
|
+
code,
|
|
142
|
+
detail,
|
|
143
|
+
retryable: true,
|
|
144
|
+
suppressed: false,
|
|
145
|
+
error: new EmailRateLimitedError({ detail }, { cause: err }),
|
|
146
|
+
};
|
|
69
147
|
}
|
|
70
148
|
|
|
71
|
-
|
|
72
|
-
|
|
149
|
+
// Three ways to earn a retry, and nothing else: a code documented as transient, no code at all, or a
|
|
150
|
+
// numeric provider code that is not the authentication class. An unlisted `E_*` stays terminal.
|
|
151
|
+
const retryable =
|
|
152
|
+
RETRYABLE.has(code) || code === "E_UNKNOWN" || (isProviderNumericCode(code) && !UNAUTHORIZED.has(code));
|
|
153
|
+
return { code, detail, retryable, suppressed: false, error: new EmailSendFailedError({ detail }, { cause: err }) };
|
|
73
154
|
}
|
package/src/send/runSend.ts
CHANGED
|
@@ -309,12 +309,16 @@ export async function runSend(deps: SendDeps, jobId: string): Promise<SendOutcom
|
|
|
309
309
|
|
|
310
310
|
if (classified.retryable && attempts < deps.maxAttempts) {
|
|
311
311
|
// Persist the last error, then throw so the Workflow step retries with backoff (row stays `sending`).
|
|
312
|
-
await patchJob(deps, jobId, { error: classified.
|
|
312
|
+
await patchJob(deps, jobId, { error: classified.detail });
|
|
313
313
|
throw classified.error;
|
|
314
314
|
}
|
|
315
315
|
|
|
316
|
-
|
|
317
|
-
|
|
316
|
+
// **`detail`, not `code`.** The column is described as "the last error code/message", and it held only
|
|
317
|
+
// the code — so a magic link that failed five times recorded `E_UNKNOWN` five times, and the one
|
|
318
|
+
// sentence naming the cause was discarded with the error object (pithy-sh/pithy#555, and #534's
|
|
319
|
+
// family). `detail` leads with the code, so nothing a reader had before is lost.
|
|
320
|
+
await patchJob(deps, jobId, { status: "failed", error: classified.detail });
|
|
321
|
+
await recordEvent(deps.db, { jobId, recipient, type: "failed", detail: classified.detail }, deps.passStartedAt);
|
|
318
322
|
return { jobId, status: "failed" };
|
|
319
323
|
}
|
|
320
324
|
}
|
package/src/version.generated.ts
CHANGED
package/src/workflows/hostEnv.ts
CHANGED
|
@@ -148,7 +148,7 @@ export const EmailHostEnv = z
|
|
|
148
148
|
.default("true")
|
|
149
149
|
.transform((value) => value === "true")
|
|
150
150
|
.describe(
|
|
151
|
-
"Whether the every-minute scheduler Workflow runs. `false` stops due jobs being claimed at all — the cron still fires and does
|
|
151
|
+
"Whether the every-minute scheduler Workflow runs. `false` stops due jobs being claimed at all — the cron still fires, and does not even look — so it is a maintenance switch, not a tuning knob.",
|
|
152
152
|
),
|
|
153
153
|
SCHEDULER_BATCH_SIZE: tunedNumber(50).describe(
|
|
154
154
|
"How many jobs one send Workflow instance takes. Each job is a durable step, so this is the unit of retry and of the instance's report.",
|
|
@@ -2,16 +2,19 @@
|
|
|
2
2
|
// SPDX-License-Identifier: MIT
|
|
3
3
|
|
|
4
4
|
import { chunkByBoundParameters } from "@pithy-sh/core/src/data/boundParameters";
|
|
5
|
+
import type { DatabaseSchema } from "@pithy-sh/core/src/data/db";
|
|
5
6
|
import { ValidationError } from "@pithy-sh/core/src/error/pithyError";
|
|
6
|
-
import type {
|
|
7
|
+
import type { Expression, ExpressionBuilder, SqlBool } from "kysely";
|
|
8
|
+
import type { EmailDatabase, EmailTables } from "../data/tables";
|
|
7
9
|
|
|
8
10
|
/**
|
|
9
11
|
* The every-minute scheduler. It finds due rows — `scheduled`/`timezone` jobs whose `sendAt` has
|
|
10
12
|
* arrived, plus the safety net of `pending` and `undispatched` immediate jobs that never dispatched and
|
|
11
13
|
* `sending` jobs left stale by a dead dispatch — claims them (status → `sending`, so the next tick won't
|
|
12
14
|
* repick them), and fans them out into sender batches. **The fan-out scales with volume:** the more rows are due, the
|
|
13
|
-
* more batches are dispatched, each its own durable send Workflow.
|
|
14
|
-
*
|
|
15
|
+
* more batches are dispatched, each its own durable send Workflow. The cron probes for a due row
|
|
16
|
+
* first ({@link hasDueJobs}) and creates no instance at all when there is none, so an idle minute
|
|
17
|
+
* costs one read rather than a Workflow and a billed step (pithy-sh/pithy#538).
|
|
15
18
|
*
|
|
16
19
|
* Jobs are claimed before dispatch, so a dispatch failure strands a row in `sending` rather than
|
|
17
20
|
* double-sending; the `sending` re-drive recovers it on a later tick, and `runSend`'s idempotency makes
|
|
@@ -24,9 +27,16 @@ import type { EmailDatabase } from "../data/tables";
|
|
|
24
27
|
* per batch.
|
|
25
28
|
*/
|
|
26
29
|
|
|
27
|
-
/**
|
|
28
|
-
|
|
30
|
+
/**
|
|
31
|
+
* What it takes to ask whether anything is due — the database and the three numbers that define
|
|
32
|
+
* "due", and nothing else.
|
|
33
|
+
*
|
|
34
|
+
* It is the front half of {@link SchedulerDeps} rather than a type beside it, so the cron's probe and
|
|
35
|
+
* the tick cannot be pointed at different databases or measured against different windows.
|
|
36
|
+
*/
|
|
37
|
+
export interface DueProbeDeps {
|
|
29
38
|
db: EmailDatabase;
|
|
39
|
+
/** The instant this tick measures every cutoff from. */
|
|
30
40
|
now: Date;
|
|
31
41
|
/** How stale (ms) a `pending`/`undispatched` immediate job must be before the safety net re-drives it. */
|
|
32
42
|
graceMs: number;
|
|
@@ -43,6 +53,10 @@ export interface SchedulerDeps {
|
|
|
43
53
|
* slower horse, and the queue gets longer.
|
|
44
54
|
*/
|
|
45
55
|
stuckMs: number;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/** Inputs the scheduler needs. `dispatch` creates one send Workflow per batch. */
|
|
59
|
+
export interface SchedulerDeps extends DueProbeDeps {
|
|
46
60
|
/**
|
|
47
61
|
* Jobs per dispatched batch — one send Workflow each. From `SCHEDULER_BATCH_SIZE`.
|
|
48
62
|
*
|
|
@@ -160,32 +174,106 @@ async function strandedIds(
|
|
|
160
174
|
return stranded;
|
|
161
175
|
}
|
|
162
176
|
|
|
177
|
+
/** The instants "due" is measured against, derived once from one clock. */
|
|
178
|
+
interface DueCutoffs {
|
|
179
|
+
/** Now, as ms since the epoch — what `sendAt` is compared against, and what a claim stamps. */
|
|
180
|
+
nowMs: number;
|
|
181
|
+
/** Before this, an immediate job that never dispatched is the safety net's. */
|
|
182
|
+
graceCutoff: number;
|
|
183
|
+
/** Before this, a `sending` job is old enough to ask the runtime about. */
|
|
184
|
+
stuckCutoff: number;
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
/** Derive this tick's cutoffs. The tick claims against the same `nowMs` it selected against. */
|
|
188
|
+
function dueCutoffs(deps: DueProbeDeps): DueCutoffs {
|
|
189
|
+
const nowMs = deps.now.getTime();
|
|
190
|
+
return { nowMs, graceCutoff: nowMs - deps.graceMs, stuckCutoff: nowMs - deps.stuckMs };
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
/**
|
|
194
|
+
* **"Due" — the one definition.** Three branches over four statuses: a `scheduled` job whose `sendAt`
|
|
195
|
+
* has arrived, a `pending` or `undispatched` immediate job past its grace, and a `sending` job past
|
|
196
|
+
* `stuckMs`.
|
|
197
|
+
*
|
|
198
|
+
* It is a function rather than a clause typed into a query because two callers ask it now: the tick
|
|
199
|
+
* below, and the cron's probe ({@link hasDueJobs}) that decides whether to create the tick at all
|
|
200
|
+
* (pithy-sh/pithy#538). A second copy would be a second definition, and the drift is silent in the
|
|
201
|
+
* worst way — a probe that stops matching a branch never wakes the scheduler for those rows, and they
|
|
202
|
+
* sit there with nothing logged. This file has been bitten twice by that shape already: the `NaN`
|
|
203
|
+
* batch size that produced one empty batch a minute, and the `undispatched` dead end (#410).
|
|
204
|
+
*/
|
|
205
|
+
function isDue(cutoffs: DueCutoffs) {
|
|
206
|
+
return (eb: ExpressionBuilder<DatabaseSchema<EmailTables>, "pithyEmailJobs">): Expression<SqlBool> =>
|
|
207
|
+
eb.or([
|
|
208
|
+
eb.and([eb("status", "=", "scheduled"), eb("sendAt", "<=", cutoffs.nowMs)]),
|
|
209
|
+
// `undispatched` beside `pending`, and it is the whole recovery path for one
|
|
210
|
+
// (pithy-sh/pithy#410). A row born `undispatched` was enqueued by a composition that binds no
|
|
211
|
+
// send Workflow — deployed before `pithy <capability> provision`, or a plain `wrangler dev` —
|
|
212
|
+
// and the tick reading this query is running on the host that composition was missing. So the
|
|
213
|
+
// first tick after the host exists is what drains that backlog; without this line the row is a
|
|
214
|
+
// dead end, because `retryJob` takes only `failed` and no command moves it.
|
|
215
|
+
eb.and([eb("status", "in", ["pending", "undispatched"]), eb("createdAt", "<=", cutoffs.graceCutoff)]),
|
|
216
|
+
eb.and([eb("status", "=", "sending"), eb("updatedAt", "<=", cutoffs.stuckCutoff)]),
|
|
217
|
+
]);
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
/**
|
|
221
|
+
* Is anything due? The question the every-minute cron asks **before** it creates a Workflow instance
|
|
222
|
+
* (pithy-sh/pithy#538).
|
|
223
|
+
*
|
|
224
|
+
* It used to be asked from inside the instance, so an idle minute cost a Workflow creation and a
|
|
225
|
+
* billed step to discover there was nothing to do — 1,440 of each per day per environment, against a
|
|
226
|
+
* Workers Free allowance of 3,000 steps a day for the whole account. Now an idle minute costs one
|
|
227
|
+
* indexed read and creates nothing.
|
|
228
|
+
*
|
|
229
|
+
* **It is deliberately the wider question.** It reads rows, not batches, so a stale row whose batch is
|
|
230
|
+
* still alive wakes the scheduler and the tick then declines to re-drive it — one instance, spent
|
|
231
|
+
* being right. The opposite asymmetry is the one that loses mail, which is why both sides share
|
|
232
|
+
* {@link isDue} rather than agreeing by inspection.
|
|
233
|
+
*/
|
|
234
|
+
export async function hasDueJobs(deps: DueProbeDeps): Promise<boolean> {
|
|
235
|
+
const row = await deps.db
|
|
236
|
+
.selectFrom("pithyEmailJobs")
|
|
237
|
+
.select("id")
|
|
238
|
+
.where(isDue(dueCutoffs(deps)))
|
|
239
|
+
.limit(1)
|
|
240
|
+
.executeTakeFirst();
|
|
241
|
+
return row !== undefined;
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
/**
|
|
245
|
+
* The whole of what the cron asks before it creates anything: **is this scheduler configured, and is
|
|
246
|
+
* anything due** — in that order (pithy-sh/pithy#538).
|
|
247
|
+
*
|
|
248
|
+
* The order is the point. {@link assertBatchSize} was written to run before the tick does any work, so
|
|
249
|
+
* that a misconfigured worker says so on its first tick rather than on its first busy one; the probe
|
|
250
|
+
* that now stands in front of the tick would have postponed it to the first minute a job came due. An
|
|
251
|
+
* idle cron that quietly accepts a broken batch size is the shape of the bug this file has already
|
|
252
|
+
* been bitten by twice, and "quietly" is what the probe would have restored.
|
|
253
|
+
*
|
|
254
|
+
* It takes the tick's own {@link SchedulerDeps} rather than the probe's {@link DueProbeDeps}, because
|
|
255
|
+
* the question is about the configuration the tick would run under. The email host validates the same
|
|
256
|
+
* number a second way — `SCHEDULER_BATCH_SIZE` is coerced and refused by `hostEnv.ts` before this is
|
|
257
|
+
* reached — so on that path this is a second lock on one door. It is the module's own invariant, and
|
|
258
|
+
* it holds for any caller that assembles deps itself.
|
|
259
|
+
*/
|
|
260
|
+
export async function schedulerHasWork(deps: SchedulerDeps): Promise<boolean> {
|
|
261
|
+
assertBatchSize(deps.batchSize);
|
|
262
|
+
return await hasDueJobs(deps);
|
|
263
|
+
}
|
|
264
|
+
|
|
163
265
|
/** Run one scheduler tick: find due rows, claim them, fan out batches. */
|
|
164
266
|
export async function runScheduler(deps: SchedulerDeps): Promise<SchedulerResult> {
|
|
165
267
|
// Before the query, so a misconfigured worker says so on its first tick rather than on its first
|
|
166
268
|
// busy one. An idle cron that quietly accepts a broken batch size is the shape of the bug, not a
|
|
167
269
|
// reason to postpone the complaint.
|
|
168
270
|
assertBatchSize(deps.batchSize);
|
|
169
|
-
const
|
|
170
|
-
const graceCutoff = nowMs - deps.graceMs;
|
|
171
|
-
const stuckCutoff = nowMs - deps.stuckMs;
|
|
271
|
+
const cutoffs = dueCutoffs(deps);
|
|
172
272
|
|
|
173
273
|
const rows = await deps.db
|
|
174
274
|
.selectFrom("pithyEmailJobs")
|
|
175
275
|
.select(["id", "batchId"])
|
|
176
|
-
.where((
|
|
177
|
-
eb.or([
|
|
178
|
-
eb.and([eb("status", "=", "scheduled"), eb("sendAt", "<=", nowMs)]),
|
|
179
|
-
// `undispatched` beside `pending`, and it is the whole recovery path for one
|
|
180
|
-
// (pithy-sh/pithy#410). A row born `undispatched` was enqueued by a composition that binds no
|
|
181
|
-
// send Workflow — deployed before `pithy <capability> provision`, or a plain `wrangler dev` —
|
|
182
|
-
// and the tick reading this query is running on the host that composition was missing. So the
|
|
183
|
-
// first tick after the host exists is what drains that backlog; without this line the row is a
|
|
184
|
-
// dead end, because `retryJob` takes only `failed` and no command moves it.
|
|
185
|
-
eb.and([eb("status", "in", ["pending", "undispatched"]), eb("createdAt", "<=", graceCutoff)]),
|
|
186
|
-
eb.and([eb("status", "=", "sending"), eb("updatedAt", "<=", stuckCutoff)]),
|
|
187
|
-
]),
|
|
188
|
-
)
|
|
276
|
+
.where(isDue(cutoffs))
|
|
189
277
|
.orderBy("sendAt", "asc")
|
|
190
278
|
.limit(deps.maxJobs)
|
|
191
279
|
.execute();
|
|
@@ -208,7 +296,7 @@ export async function runScheduler(deps: SchedulerDeps): Promise<SchedulerResult
|
|
|
208
296
|
for (const claim of chunkByBoundParameters(batch, CLAIM_FIXED_PARAMETERS)) {
|
|
209
297
|
await deps.db
|
|
210
298
|
.updateTable("pithyEmailJobs")
|
|
211
|
-
.set({ status: "sending", updatedAt: nowMs, batchId })
|
|
299
|
+
.set({ status: "sending", updatedAt: cutoffs.nowMs, batchId })
|
|
212
300
|
.where("id", "in", claim)
|
|
213
301
|
.execute();
|
|
214
302
|
}
|
package/src/workflows/worker.ts
CHANGED
|
@@ -19,7 +19,7 @@ import { createEmailHostApp } from "./hostApp";
|
|
|
19
19
|
import { type EmailHostEnv, emailHostEnv } from "./hostEnv";
|
|
20
20
|
import { isLiveInstanceStatus } from "./instanceLiveness";
|
|
21
21
|
import type { SendWorkflowInstances } from "./instances";
|
|
22
|
-
import { runScheduler, type SchedulerDeps } from "./scheduler";
|
|
22
|
+
import { runScheduler, type SchedulerDeps, schedulerHasWork } from "./scheduler";
|
|
23
23
|
import { type BatchSendReport, runSendBatch, type SendBatchDeps } from "./sendBatch";
|
|
24
24
|
|
|
25
25
|
/**
|
|
@@ -29,7 +29,7 @@ import { type BatchSendReport, runSendBatch, type SendBatchDeps } from "./sendBa
|
|
|
29
29
|
* - `EmailSendWorkflow` — sends a batch of jobs durably (dispatch target for immediate sends and
|
|
30
30
|
* the scheduler's fan-out).
|
|
31
31
|
* - `EmailSchedulerWorkflow` — finds due jobs and fans them out into send batches.
|
|
32
|
-
* - `scheduled()` — the every-minute cron that fires the scheduler Workflow.
|
|
32
|
+
* - `scheduled()` — the every-minute cron that fires the scheduler Workflow when a row is due.
|
|
33
33
|
* - `fetch()` — the loopback dispatch door, served only in `dev` (see {@link createEmailHostApp}).
|
|
34
34
|
*
|
|
35
35
|
* The bodies (`runSendBatch`, `runScheduler`, `runSend`) are tested against Miniflare; these classes are
|
|
@@ -119,9 +119,14 @@ async function buildSendDeps(env: EmailWorkerEnv): Promise<SendBatchDeps> {
|
|
|
119
119
|
};
|
|
120
120
|
}
|
|
121
121
|
|
|
122
|
-
/**
|
|
123
|
-
|
|
124
|
-
|
|
122
|
+
/**
|
|
123
|
+
* Assemble the scheduler dependencies, dispatching each batch as a send Workflow.
|
|
124
|
+
*
|
|
125
|
+
* `config` is a parameter so the cron can validate the env once and hand the answer on: it reads
|
|
126
|
+
* `SCHEDULER_ENABLED` before it probes, and re-parsing the same env object a second line later would
|
|
127
|
+
* be one Zod pass per minute for nothing.
|
|
128
|
+
*/
|
|
129
|
+
function buildSchedulerDeps(env: EmailWorkerEnv, config: EmailHostEnv = hostConfig(env)): SchedulerDeps {
|
|
125
130
|
return {
|
|
126
131
|
db: emailDatabase(env.DB),
|
|
127
132
|
now: new Date(),
|
|
@@ -180,11 +185,33 @@ export class EmailSchedulerWorkflow extends WorkflowEntrypoint<EmailWorkerEnv, u
|
|
|
180
185
|
}
|
|
181
186
|
|
|
182
187
|
export default {
|
|
183
|
-
/**
|
|
188
|
+
/**
|
|
189
|
+
* Cron entry: fire the scheduler Workflow every minute — when there is something to fire it for.
|
|
190
|
+
*
|
|
191
|
+
* The instance used to be created unconditionally, and the "is anything due" question was asked
|
|
192
|
+
* inside it. At `* * * * *` that spends 1,440 instance creations and 1,440 billed steps a day, per
|
|
193
|
+
* environment, discovering nothing to do; Workers Free allows 3,000 steps a day for the whole
|
|
194
|
+
* account, so a default project's staging and prod burn 96% of it sending no mail, and the sends
|
|
195
|
+
* that do happen compete for the rest (pithy-sh/pithy#538).
|
|
196
|
+
*
|
|
197
|
+
* So the question moved out in front: one indexed read on `pithy_email_jobs`, asked with the tick's
|
|
198
|
+
* own predicate rather than a second copy of it, and an instance only when it finds a row. Nothing
|
|
199
|
+
* is lost with the tick that never starts: this is not the path an immediate send takes (`enqueue`
|
|
200
|
+
* starts that Workflow itself), the work is a pure re-derivation from D1, and a lost minute is
|
|
201
|
+
* recovered by the next one exactly as a throwing `create()` has always been.
|
|
202
|
+
*
|
|
203
|
+
* What it does give up is the per-minute instance in the dashboard that proved the scheduler was
|
|
204
|
+
* alive. An idle minute now leaves no trace, by design — the trace was costing a billed step.
|
|
205
|
+
*
|
|
206
|
+
* **Through {@link schedulerHasWork}, so the tick's configuration is checked before the probe.** The
|
|
207
|
+
* batch-size check inside `runScheduler` exists to complain on the first tick rather than the first
|
|
208
|
+
* busy one, and a probe in front of the instance is in front of that check too.
|
|
209
|
+
*/
|
|
184
210
|
async scheduled(_controller: unknown, env: EmailWorkerEnv): Promise<void> {
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
211
|
+
const config = hostConfig(env);
|
|
212
|
+
if (!config.SCHEDULER_ENABLED) return;
|
|
213
|
+
if (!(await schedulerHasWork(buildSchedulerDeps(env, config)))) return;
|
|
214
|
+
await env.EMAIL_SCHEDULER.create();
|
|
188
215
|
},
|
|
189
216
|
|
|
190
217
|
/**
|
|
@@ -58,8 +58,9 @@
|
|
|
58
58
|
{ "binding": "EMAIL_SCHEDULER", "name": "pithy-email-schedule", "class_name": "EmailSchedulerWorkflow" }
|
|
59
59
|
],
|
|
60
60
|
|
|
61
|
-
// The every-minute scheduler cron. Cheap
|
|
62
|
-
//
|
|
61
|
+
// The every-minute scheduler cron. Cheap because it stays cheap when idle: scheduled() probes
|
|
62
|
+
// pithy_email_jobs for a due row and creates a Workflow instance only when it finds one, so an idle
|
|
63
|
+
// minute costs one read rather than an instance and a billed step (pithy-sh/pithy#538).
|
|
63
64
|
"triggers": { "crons": ["* * * * *"] },
|
|
64
65
|
|
|
65
66
|
"vars": {
|