@voltro/runtime 0.85.0 → 0.87.0

package/dist/scheduler.js

@@ -0,0 +1,840 @@
+import { settleHandlerBody as e } from "./handlerBody.js";
+import { recordExternalMissed as t, recordExternalTrigger as n, recordScheduleQueueRecovery as r, recordScheduleQueueWait as i, recordScheduleQueued as a, recordScheduleRun as o } from "./metrics.js";
+import { newTraceId as s } from "./traceparent.js";
+import { SYSTEM_SUBJECT as c } from "./systemSubject.js";
+import { DEFAULT_MAX_RUNTIME_MS as l, compileCron as u, cronOccurrences as d } from "./schedule.js";
+import { acceptScheduleBackfillRequest as f, executeScheduleBackfillRequest as p, interruptOwnedScheduleBackfills as ee, interruptStaleScheduleBackfills as m, readScheduleBackfillRequest as h } from "./scheduleBackfillRequests.js";
+import { registerWakeup as te } from "./wakeRegistry.js";
+import { Cron as g, Effect as ne } from "effect";
+import { and as _, eq as v, extractDbCause as y, lt as re } from "@voltro/database";
+import { randomUUID as ie } from "node:crypto";
+import { publishServerError as ae } from "@voltro/protocol";
+//#region src/scheduler.ts
+var b = c, x = "default", S = {
+	kind: "single",
+	tryClaim: async () => !0
+}, C = "_voltro_schedule_claims", oe = 6e4, w = 64, T = (e) => Math.max((e ?? 6e4) * 64, 6e4), E = (e, t, n) => {
+	let r = async (t) => (await e.query({
+		table: C,
+		order: [],
+		predicate: v("id", t),
+		projection: ["id"],
+		skip: void 0,
+		take: 1
+	})).length > 0, i = /* @__PURE__ */ new Map(), a = async (t, n) => {
+		let r = Date.now();
+		if (!(r - (i.get(t) ?? 0) < n)) {
+			i.set(t, r);
+			try {
+				await e.deleteMany(C, { where: _(v("scheduleName", t), re("claimedAt", new Date(r - n))) });
+			} catch {}
+		}
+	};
+	return {
+		kind: "advisoryLock",
+		tryClaim: async (i, o, s) => {
+			let c = o.toISOString(), l = o.getUTCMilliseconds() === 0 ? c.slice(0, 19) : c.slice(0, 23), u = `${i}@${l}`;
+			try {
+				if (await r(u)) return !1;
+			} catch {}
+			try {
+				return await e.insert(C, {
+					id: u,
+					scheduleName: i,
+					bucket: l,
+					claimedBy: t,
+					claimedAt: /* @__PURE__ */ new Date()
+				}), await a(i, T(s)), !0;
+			} catch (e) {
+				try {
+					if (await r(u)) return !1;
+				} catch {}
+				return n?.warn("schedule: advisoryLock claim failed without a winning row — claims table may be unmigrated/unreachable; schedule will NOT fire until fixed", {
+					schedule: i,
+					reason: e?.message ?? String(e)
+				}), !1;
+			}
+		}
+	};
+}, se = 3e5, ce = 3e5, le = 6e4, ue = 864e5, D = 200, de = /* @__PURE__ */ new Set([
+	"succeeded",
+	"failed",
+	"skipped",
+	"missed"
+]), O = "_voltro_schedule_runs", fe = 3e4, k = 3, pe = 1e3, me = (e, t) => Math.max(e * 2, t * k * 2), A = (e, t, n) => {
+	let r = n.heartbeatMs * k, i = e.heartbeatAt == null ? NaN : new Date(e.heartbeatAt).getTime();
+	if (Number.isFinite(i)) return t - i < r;
+	let a = e.firedAt == null ? NaN : new Date(e.firedAt).getTime();
+	return Number.isFinite(a) ? t - a < n.maxRuntimeMs + r : !1;
+}, j = async (c, x) => {
+	let { store: S, coordinator: C, buildContext: oe, replicaId: w, log: T, onRunEvent: de, interceptFire: k } = x, j = x.runHeartbeatMs !== void 0 && x.runHeartbeatMs > 0 ? x.runHeartbeatMs : fe, _e = x.queueAdoptionWindowMs !== void 0 && x.queueAdoptionWindowMs > 0 ? x.queueAdoptionWindowMs : void 0, N = x.dormancy === "sleep", ve = x.dormancyTenantId ?? "default", P = /* @__PURE__ */ new Map(), F = /* @__PURE__ */ new Set(), ye = /* @__PURE__ */ new Map(), I = !1, be = `${w}:${ie()}`, L, R, z = !1, B = () => {
+		I || z || L !== void 0 || (L = setTimeout(() => {
+			L = void 0, z = !0, R = m(S).then(() => {}, (e) => T.warn("schedule backfill recovery unavailable", { reason: M(e) })).finally(() => {
+				z = !1, I || B();
+			});
+		}, 1e4), L.unref?.());
+	}, V = new Map(c.map((e) => [e.name, e])), H = (e, t) => {
+		de?.(e, ge(t));
+	}, U = async (e, t, n, r, i, a) => {
+		try {
+			let o = `sr_${ie()}`, s = await S.insert(O, {
+				id: o,
+				name: e.name,
+				status: r,
+				trigger: i,
+				scheduledAt: t,
+				firedAt: n,
+				completedAt: null,
+				durationMs: null,
+				errorTag: null,
+				errorMessage: null,
+				coordinationOutcome: a,
+				replicaId: w,
+				heartbeatAt: r === "running" ? n : r === "queued" ? /* @__PURE__ */ new Date() : null,
+				result: null
+			}), c = s.id ?? o;
+			return H("insert", s), c;
+		} catch (t) {
+			T.warn("schedule: failed to record run start", {
+				name: e.name,
+				reason: M(t)
+			});
+			return;
+		}
+	}, xe = async (e, t) => {
+		try {
+			let n = await S.update(O, e, t);
+			if (n) {
+				H("update", n);
+				return;
+			}
+			T.warn("schedule: run row did not transition — it stays `running` and will suppress this schedule", {
+				runId: e,
+				status: t.status
+			});
+		} catch (t) {
+			T.warn("schedule: failed to record run finish", {
+				runId: e,
+				reason: M(t)
+			});
+		}
+	}, W = async (e) => {
+		try {
+			let t = await S.query({
+				table: O,
+				predicate: { and: [{
+					_tag: "leaf",
+					column: "status",
+					op: "eq",
+					value: "running"
+				}, {
+					_tag: "leaf",
+					column: "name",
+					op: "eq",
+					value: e.name
+				}] },
+				order: [{
+					column: "firedAt",
+					direction: "desc"
+				}],
+				take: 200,
+				skip: void 0,
+				projection: void 0
+			}), n = Date.now(), r = {
+				heartbeatMs: j,
+				maxRuntimeMs: e.maxRuntimeMs ?? 18e5
+			}, i = t.find((e) => A(e, n, r));
+			if (i === void 0) return;
+			let a = i.heartbeatAt ?? i.firedAt;
+			return {
+				replicaId: i.replicaId ?? null,
+				lastSeenMs: a == null ? NaN : n - new Date(a).getTime()
+			};
+		} catch (t) {
+			T.warn("schedule: cluster-running check failed", {
+				name: e.name,
+				reason: M(t)
+			});
+			return;
+		}
+	}, Se = async (e, t) => {
+		try {
+			await S.update(O, e, { heartbeatAt: /* @__PURE__ */ new Date() });
+		} catch (n) {
+			t.yes || (t.yes = !0, T.warn("schedule: heartbeat write failed — a peer may read this run as dead and fire alongside it", {
+				runId: e,
+				reason: M(n)
+			}));
+		}
+	}, Ce = (e, t) => t === void 0 ? e === "external" || C.kind !== "single" : t !== "single", G = /* @__PURE__ */ new Set(), K = 0, we = (e, t) => new Promise((n) => {
+		if (K !== t) {
+			n();
+			return;
+		}
+		let r, i = !1, a = () => {
+			i || (i = !0, G.delete(o), r !== void 0 && clearTimeout(r), n());
+		}, o = () => {
+			a();
+		};
+		G.add(o), r = setTimeout(a, e), r.unref?.();
+	}), Te = x.onChange?.((e) => {
+		if (e.table === O) {
+			K += 1;
+			for (let e of [...G]) e();
+		}
+	}), Ee = async (e, t) => await S.updateMany(O, {
+		status: "running",
+		firedAt: t,
+		heartbeatAt: t,
+		replicaId: w
+	}, { where: _(v("id", e), v("status", "queued")) }) > 0, De = (e, t) => {
+		try {
+			let n = u(e), r = g.next(n, t), i = g.next(n, r).getTime() - r.getTime();
+			return i > 0 ? i : j;
+		} catch {
+			return j;
+		}
+	}, Oe = async (e, t, n) => {
+		let r = await S.query({
+			table: O,
+			predicate: _(v("name", e.name), v("status", "queued"), re("scheduledAt", t)),
+			order: [{
+				column: "scheduledAt",
+				direction: "asc"
+			}],
+			take: 200,
+			skip: void 0,
+			projection: void 0
+		}), i = Date.now(), a = {
+			heartbeatMs: j,
+			maxRuntimeMs: e.maxRuntimeMs ?? 18e5
+		};
+		for (let t of r) {
+			if (t.id === n) continue;
+			let r = new Date(t.scheduledAt);
+			if (A(t, i, a)) return {
+				kind: "blocked",
+				behind: r
+			};
+			let o = t.heartbeatAt ?? t.firedAt ?? t.scheduledAt, s = i - new Date(o).getTime();
+			if (s <= (_e ?? me(De(e, r), j))) return {
+				kind: "adopt",
+				runId: t.id,
+				scheduledAt: r,
+				trigger: t.trigger ?? e.trigger ?? x.defaultTrigger
+			};
+			await ke(t.id, e.name, r, s);
+		}
+		return { kind: "next" };
+	}, ke = async (e, t, n, i, a = "the replica holding this firing stopped reporting for longer than the adoption window") => {
+		try {
+			await S.updateMany(O, {
+				status: "missed",
+				errorTag: "abandoned",
+				errorMessage: a,
+				completedAt: /* @__PURE__ */ new Date()
+			}, { where: _(v("id", e), v("status", "queued")) }) > 0 && (r(t, "abandoned"), T.warn("schedule: a queued firing was abandoned", {
+				name: t,
+				runId: e,
+				scheduledAt: n.toISOString(),
+				silenceMs: Math.round(i),
+				reason: a
+			}));
+		} catch (n) {
+			T.warn("schedule: writing off an abandoned queued firing failed", {
+				name: t,
+				runId: e,
+				reason: M(n)
+			});
+		}
+	}, Ae = async (e) => {
+		try {
+			let t = await S.query({
+				table: O,
+				predicate: v("id", e),
+				order: [],
+				take: 1,
+				skip: void 0,
+				projection: void 0
+			});
+			t[0] !== void 0 && H("update", t[0]);
+		} catch {}
+	}, je = async (e, t, n, a) => {
+		let o = { yes: !1 }, s = !1, c = setInterval(() => {
+			Se(n, o);
+		}, j);
+		c.unref?.();
+		try {
+			for (;;) {
+				if (I) return;
+				let o = K;
+				try {
+					let o = await W(e);
+					if (o === void 0) {
+						let o = await Oe(e, t, n);
+						if (o.kind === "next") {
+							let r = /* @__PURE__ */ new Date();
+							if (!await Ee(n, r)) {
+								T.warn("schedule: queued firing lost its row before its turn", {
+									name: e.name,
+									runId: n,
+									scheduledAt: t.toISOString()
+								});
+								return;
+							}
+							return await Ae(n), i(e.name, Date.now() - a), r;
+						}
+						if (o.kind === "adopt") {
+							let t = /* @__PURE__ */ new Date();
+							await Ee(o.runId, t) && (T.info("schedule: adopted a queued firing from a replica that stopped reporting", {
+								name: e.name,
+								runId: o.runId,
+								scheduledAt: o.scheduledAt.toISOString()
+							}), r(e.name, "adopted"), await Ae(o.runId), await q(e, o.scheduledAt, o.trigger, "wonLock", {
+								runId: o.runId,
+								firedAt: t
+							}));
+							continue;
+						}
+						s || (s = !0, T.info("schedule: queued behind an earlier firing", {
+							name: e.name,
+							scheduledAt: t.toISOString(),
+							behind: o.behind.toISOString()
+						}));
+					} else s || (s = !0, T.info("schedule: queued behind a run in flight", {
+						name: e.name,
+						scheduledAt: t.toISOString(),
+						heldBy: o.replicaId ?? "unknown"
+					}));
+				} catch (t) {
+					T.warn("schedule: queue turn check failed", {
+						name: e.name,
+						reason: M(t)
+					});
+				}
+				await we(pe, o);
+			}
+		} finally {
+			clearInterval(c);
+		}
+	}, Me = async (e, t, n, r, i) => {
+		let o = Date.now(), s = await U(e, t, null, "queued", n, r);
+		if (s === void 0) return T.warn("schedule: could not record the queued firing — running it without the fleet-wide turn, which may overlap a run elsewhere", { name: e.name }), q(e, t, n, r, void 0, i);
+		i?.(s), a(e.name, 1);
+		let c;
+		try {
+			c = await je(e, t, s, o);
+		} finally {
+			a(e.name, -1);
+		}
+		return c === void 0 ? s : q(e, t, n, r, {
+			runId: s,
+			firedAt: c
+		});
+	}, q = async (t, n, r, i, a, c) => {
+		let l = a?.firedAt ?? /* @__PURE__ */ new Date(), u = a?.runId ?? await U(t, n, l, "running", r, i);
+		c?.(u);
+		let d = Date.now(), f, p, ee = { yes: !1 }, m = new AbortController();
+		try {
+			let i = new Promise((e, n) => {
+				f = setTimeout(() => {
+					n(/* @__PURE__ */ Error(`schedule "${t.name}" exceeded maxRuntimeMs=${t.maxRuntimeMs}`)), m.abort();
+				}, t.maxRuntimeMs);
+			});
+			i.catch(() => {}), u !== void 0 && (p = setInterval(() => {
+				Se(u, ee);
+			}, j), p.unref?.());
+			let a = {
+				scheduledAt: n,
+				firedAt: l,
+				runId: u ?? "unrecorded",
+				trigger: r,
+				app: oe({
+					traceId: s(),
+					scheduleName: t.name,
+					trigger: r,
+					runId: u,
+					subject: b
+				})
+			}, c = async () => {
+				if (t.workflow !== void 0) {
+					let e = a.app.workflows;
+					if (e === void 0) throw Error(`schedule "${t.name}" targets workflow "${t.workflow.name}" but no workflows runtime is configured`);
+					let n = typeof t.workflow.payload == "function" ? await t.workflow.payload(a) : t.workflow.payload ?? {};
+					await e.start(t.workflow.name, n);
+				}
+				if (t.handler !== void 0) {
+					let n = t.handler, r = x.measureSync ? x.measureSync(() => n(a)) : n(a);
+					await e(r, (e) => x.runEffect ? x.runEffect(e, m.signal) : ne.runPromise(e, { signal: m.signal }));
+				}
+			}, h = k ? k(c, {
+				name: t.name,
+				scheduledAt: n,
+				firedAt: l,
+				runId: u ?? "unrecorded",
+				trigger: r
+			}) : c();
+			await Promise.race([h, i]), u && await xe(u, {
+				status: "succeeded",
+				completedAt: /* @__PURE__ */ new Date(),
+				durationMs: Date.now() - d
+			}), T.info("schedule: fired", {
+				name: t.name,
+				durationMs: Date.now() - d
+			}), o({
+				name: t.name,
+				status: "succeeded",
+				durationMs: Date.now() - d,
+				completedAtSec: Date.now() / 1e3
+			});
+		} catch (e) {
+			let n = M(e).includes("exceeded maxRuntimeMs"), r = y(e).dbMessage, i = typeof r == "string" && r !== "" ? `${M(e)} — ${r}` : M(e);
+			u && await xe(u, {
+				status: "failed",
+				errorTag: n ? "timeout" : he(e),
+				errorMessage: i,
+				completedAt: /* @__PURE__ */ new Date(),
+				durationMs: Date.now() - d
+			});
+			let a = {
+				name: t.name,
+				reason: M(e),
+				timeout: n,
+				...y(e)
+			};
+			T.error ? T.error("schedule: handler failed", a) : T.warn("schedule: handler failed", a), o({
+				name: t.name,
+				status: "failed",
+				durationMs: Date.now() - d
+			}), ae({
+				error: e,
+				source: "schedule",
+				name: t.name,
+				fields: {
+					timeout: n,
+					...u ? { runId: u } : {}
+				}
+			});
+		} finally {
+			f && clearTimeout(f), p && clearInterval(p);
+		}
+		return u;
+	}, J = async (e, t, n, r, i) => {
+		if (n !== "manual" && e.onOverlap === "queue" && !Ce(n, r)) {
+			let a = (ye.get(e.name) ?? Promise.resolve()).then(() => Ne(e, t, n, r, i));
+			return ye.set(e.name, a.catch(() => {})), a;
+		}
+		if (n !== "manual" && e.onOverlap === "skip") {
+			let a = F.has(e.name), o = a ? void 0 : await W(e);
+			if (a || o !== void 0) {
+				let s = await U(e, t, null, "skipped", n, r ?? "single");
+				return i?.(s), T.info("schedule: overlap skip", {
+					name: e.name,
+					scope: a ? "local" : "cluster",
+					...o === void 0 ? {} : {
+						heldBy: o.replicaId ?? "unknown",
+						lastSeenMs: Number.isFinite(o.lastSeenMs) ? Math.round(o.lastSeenMs) : null
+					}
+				}), s;
+			}
+		}
+		return Ne(e, t, n, r, i);
+	}, Ne = async (e, t, n, r, i) => {
+		let a;
+		if (r !== void 0) a = r;
+		else if (n === "manual") a = "single";
+		else if (n === "external") a = "external";
+		else {
+			if (!await C.tryClaim(e.name, t)) {
+				T.debug?.("schedule: lost coordination claim", {
+					name: e.name,
+					coordinator: C.kind
+				});
+				return;
+			}
+			a = C.kind === "single" ? "single" : "wonLock";
+		}
+		F.add(e.name);
+		try {
+			return n !== "manual" && e.onOverlap === "queue" && Ce(n, r) ? await Me(e, t, n, a, i) : await q(e, t, n, a, void 0, i);
+		} finally {
+			F.delete(e.name);
+		}
+	}, Y = async (e) => {
+		try {
+			let t = g.next(u(e), /* @__PURE__ */ new Date());
+			await te(S, {
+				tenantId: ve,
+				kind: "schedule",
+				refId: e.name,
+				wakeAt: t
+			});
+		} catch (t) {
+			T.warn("schedule: wakeup register failed", {
+				name: e.name,
+				reason: M(t)
+			});
+		}
+	}, Pe = 2147483647, Fe = (e) => {
+		if (I) return;
+		let t = u(e), n = g.next(t, /* @__PURE__ */ new Date()), r = () => {
+			if (I) return;
+			let t = n.getTime() - Date.now();
+			if (t > Pe) {
+				let t = setTimeout(r, Pe);
+				P.set(e.name, t);
+				return;
+			}
+			let i = setTimeout(() => {
+				J(e, n, e.trigger ?? x.defaultTrigger).catch((t) => T.warn("schedule: fire rejected", {
+					name: e.name,
+					reason: M(t)
+				})).finally(() => Fe(e));
+			}, Math.max(0, t));
+			P.set(e.name, i);
+		};
+		r();
+	}, Ie = async () => {
+		if (V.size !== 0) try {
+			let e = await S.query({
+				table: O,
+				predicate: v("status", "queued"),
+				order: [{
+					column: "scheduledAt",
+					direction: "asc"
+				}],
+				take: 500,
+				skip: void 0,
+				projection: void 0
+			}), t = Date.now(), n = {
+				heartbeatMs: j,
+				maxRuntimeMs: l
+			};
+			for (let r of e) {
+				if (V.has(r.name) || A(r, t, n)) continue;
+				let e = r.heartbeatAt ?? r.firedAt ?? r.scheduledAt;
+				await ke(r.id, r.name, new Date(r.scheduledAt), t - new Date(e).getTime(), "no schedule of this name is declared any more, so nothing can take this firing");
+			}
+		} catch (e) {
+			T.warn("schedule: orphaned-waiter sweep failed", { reason: M(e) });
+		}
+	}, Le = async (e) => {
+		if (e.backfill === "skip") return;
+		let t;
+		try {
+			let n = (await S.query({
+				table: O,
+				order: [{
+					column: "scheduledAt",
+					direction: "desc"
+				}],
+				predicate: v("name", e.name),
+				projection: void 0,
+				skip: void 0,
+				take: 1
+			}))[0];
+			n && (t = new Date(n.scheduledAt));
+		} catch (t) {
+			T.warn("schedule: backfill lookup failed", {
+				name: e.name,
+				reason: M(t)
+			});
+			return;
+		}
+		if (!t) return;
+		let n = u(e), r = [], i = g.next(n, t), a = Date.now();
+		for (; i.getTime() < a && r.length < 1e3;) r.push(i), i = g.next(n, i);
+		if (r.length !== 0) {
+			if (T.info("schedule: backfill", {
+				name: e.name,
+				missed: r.length,
+				policy: e.backfill
+			}), e.backfill === "latest") {
+				for (let t of r.slice(0, -1)) {
+					let n = !0;
+					try {
+						n = await C.tryClaim(e.name, t);
+					} catch {
+						n = !0;
+					}
+					n && await U(e, t, null, "missed", e.trigger ?? x.defaultTrigger, C.kind === "single" ? "single" : "wonLock");
+				}
+				let t = r[r.length - 1];
+				await J(e, t, e.trigger ?? x.defaultTrigger);
+			} else for (let t of r) await J(e, t, e.trigger ?? x.defaultTrigger);
+		}
+	}, Re = c.filter((e) => (e.trigger ?? x.defaultTrigger) === "external" && e.cron.trim().split(/\s+/).length === 6);
+	if (Re.length > 0) throw Error(`startScheduler: ${Re.map((e) => `"${e.name}" ("${e.cron}")`).join(", ")} use a six-field (seconds) cron and resolve to trigger: 'external' via scheduling.trigger — no platform scheduler can express sub-minute cadence, so nothing would ever fire them.
+Give each a five-field expression, or set trigger: 'self' on it.`);
+	let X = c.filter((e) => (e.trigger ?? x.defaultTrigger) === "self");
+	T.info("scheduler: starting", {
+		total: c.length,
+		self: X.length,
+		external: c.length - X.length,
+		coordinator: C.kind
+	}), await Ie();
+	let ze = x.externalTriggerSkewMs !== void 0 && x.externalTriggerSkewMs > 0 ? x.externalTriggerSkewMs : se, Be = C.kind === "advisoryLock" ? C : E(S, w, T), Ve = (e, t, n) => {
+		try {
+			let r = u(e), i = g.next(r, new Date(t.getTime() - n)), a, o = 0;
+			for (; i.getTime() <= t.getTime() && o++ < 1e3;) a = i, i = g.next(r, i);
+			return a;
+		} catch {
+			return;
+		}
+	}, He = (e, t) => Math.min(ze, De(e, t)), Z = (e) => e == null ? null : e instanceof Date ? e : new Date(String(e)), Ue = async (e, t) => {
+		try {
+			return (await S.query({
+				table: O,
+				order: [],
+				predicate: _(v("name", e), v("scheduledAt", t)),
+				projection: ["id"],
+				skip: void 0,
+				take: 1
+			}))[0]?.id;
+		} catch {
+			return;
+		}
+	}, We = async (e) => {
+		let t = (await S.query({
+			table: O,
+			order: [],
+			predicate: v("id", e),
+			projection: void 0,
+			skip: void 0,
+			take: 1
+		}))[0];
+		if (t !== void 0) return {
+			runId: String(t.id),
+			name: String(t.name),
+			status: String(t.status),
+			trigger: String(t.trigger),
+			scheduledAt: Z(t.scheduledAt),
+			firedAt: Z(t.firedAt),
+			completedAt: Z(t.completedAt),
+			durationMs: typeof t.durationMs == "number" ? t.durationMs : null,
+			errorTag: t.errorTag === null || t.errorTag === void 0 ? null : String(t.errorTag),
+			errorMessage: t.errorMessage === null || t.errorMessage === void 0 ? null : String(t.errorMessage)
+		};
+	}, Ge = c.filter((e) => (e.trigger ?? x.defaultTrigger) === "external"), Q = x.externalMissedPolicy ?? "record", Ke = x.externalReconcileMs !== void 0 && x.externalReconcileMs > 0 ? x.externalReconcileMs : ce, $, qe = async (e) => {
+		try {
+			let t = (await S.query({
+				table: O,
+				predicate: v("name", e),
+				order: [{
+					column: "scheduledAt",
+					direction: "desc"
+				}],
+				projection: ["scheduledAt"],
+				skip: void 0,
+				take: 1
+			}))[0]?.scheduledAt;
+			return t === void 0 ? void 0 : Z(t) ?? void 0;
+		} catch {
+			return;
+		}
+	}, Je = async () => {
+		let e = Date.now();
+		for (let n of Ge) try {
+			let r = new Date(e - He(n, new Date(e))), i = await qe(n.name) ?? /* @__PURE__ */ new Date(e - ue);
+			if (i.getTime() >= r.getTime()) continue;
+			let a = d(n, i, r, D);
+			if (a.length === 0) continue;
+			a.length > D && T.warn("schedule: more unfired occurrences than one reconciliation records", {
+				name: n.name,
+				cap: D,
+				since: i.toISOString()
+			});
+			let o = 0;
+			for (let e of a.slice(0, D)) await Be.tryClaim(n.name, e) && (Q === "record-and-run" ? await J(n, e, "external", "external") : await U(n, e, null, "missed", "external", "external"), o += 1);
+			o > 0 && (t(n.name, o), T.warn("schedule: occurrences were never triggered by the external scheduler", {
+				name: n.name,
+				count: o,
+				since: i.toISOString(),
+				policy: Q
+			}));
+		} catch (e) {
+			T.warn("schedule: external reconciliation failed", {
+				name: n.name,
+				reason: e?.message ?? String(e)
+			});
+		}
+	}, Ye = () => {
+		I || ($ = setTimeout(() => {
+			Je().finally(Ye);
+		}, Ke), $.unref?.());
+	};
+	Ge.length > 0 && Q !== "off" && Ye();
+	let Xe = async (e, t) => {
+		if (I) throw Error("scheduler stopped");
+		let n = V.get(e);
+		if (!n) return {
+			kind: "unknownSchedule",
+			name: e
+		};
+		let r = n.trigger ?? x.defaultTrigger;
+		if (r !== "external") return {
+			kind: "notExternal",
+			name: e,
+			trigger: r
+		};
+		let i = /* @__PURE__ */ new Date();
+		if (t.getTime() > i.getTime() + 6e4) return T.warn("schedule: external trigger states a future time — refused, so the real firing is not turned into a duplicate", {
+			name: e,
+			arrivalAt: t.toISOString(),
+			now: i.toISOString()
+		}), {
+			kind: "ahead",
+			arrivalAt: t,
+			now: i,
+			toleranceMs: le
+		};
+		let a = He(n, t), o = Ve(n, t, a);
+		if (o === void 0) return T.warn("schedule: external trigger matched no occurrence — the deployed cadence and the declared one disagree", {
+			name: e,
+			arrivalAt: t.toISOString(),
+			cron: n.cron,
+			timezone: n.timezone,
+			skewMs: a
+		}), {
+			kind: "noOccurrence",
+			arrivalAt: t,
+			skewMs: a,
+			cron: n.cron,
+			timezone: n.timezone
+		};
+		if (!await Be.tryClaim(n.name, o)) return T.debug?.("schedule: external trigger deduplicated", {
+			name: e,
+			scheduledAt: o.toISOString()
+		}), {
+			kind: "duplicate",
+			runId: await Ue(n.name, o),
+			scheduledAt: o
+		};
+		let s = () => {}, c = new Promise((e) => {
+			s = e;
+		}), l = !1, u = (e) => {
+			l || (l = !0, s(e));
+		};
+		return J(n, o, "external", "external", u).then((e) => u(e)).catch((t) => {
+			T.warn("schedule: external firing failed to start", {
+				name: e,
+				reason: t?.message ?? String(t)
+			}), u(void 0);
+		}), N && await Y(n), {
+			kind: "accepted",
+			runId: await c,
+			scheduledAt: o
+		};
+	}, Ze = (x.armSelf ?? !0) && !N;
+	if (N) {
+		T.info("scheduler: sleep mode — registering wakeups, not arming timers", { count: X.length });
+		for (let e of X) await Y(e);
+	} else if (Ze) {
+		for (let e of X) await Le(e);
+		for (let e of X) Fe(e);
+	}
+	return {
+		isRunning: () => !I,
+		triggerExternal: async (e, t) => {
+			let r = await Xe(e, t);
+			return n(e, r.kind), r;
+		},
+		runStatus: We,
+		startNow: async (e, t) => {
+			let n = V.get(e);
+			if (!n) throw Error(`scheduler.startNow: unknown schedule "${e}"`);
+			if (I) throw Error("scheduler stopped");
+			let r = await new Promise((e, r) => {
+				J(n, /* @__PURE__ */ new Date(), t, void 0, e).then(e, r);
+			});
+			return N && await Y(n), r;
+		},
+		fireNow: async (e, t) => {
+			let n = V.get(e);
+			if (!n) throw Error(`scheduler.fireNow: unknown schedule "${e}"`);
+			let r = await J(n, /* @__PURE__ */ new Date(), t);
+			return N && await Y(n), r;
+		},
+		fireFromCluster: async (e) => {
+			let t = V.get(e);
+			if (!t) throw Error(`scheduler.fireFromCluster: unknown schedule "${e}"`);
+			let n = await J(t, /* @__PURE__ */ new Date(), "self", "cluster");
+			return N && await Y(t), n;
+		},
+		backfillRequest: async (e) => (await m(S), B(), h(S, e)),
+		startBackfillRange: async (e, t, n, r) => {
+			if (I) throw Error("scheduler stopped");
+			let i = V.get(e);
+			if (!i) throw Error(`Unknown schedule "${e}"`);
+			if (!Number.isInteger(r.cap) || r.cap < 1 || r.cap > 1e4) throw Error("Invalid backfill cap");
+			if (!(t.getTime() < n.getTime())) throw Error("Invalid backfill range");
+			let a = d(i, t, n, r.cap);
+			if (a.length > r.cap) throw Error("Backfill occurrence limit exceeded");
+			await m(S), B();
+			let o = await f(S, {
+				id: r.requestId,
+				name: e,
+				from: t,
+				to: n,
+				occurrences: a
+			});
+			return p(S, r.requestId, be, async (e) => {
+				if (I) throw Error("Scheduler stopped during backfill");
+				let t = await J(i, e, "manual");
+				if (!t) throw Error("Backfill firing has no recorded run ID");
+				let n = await We(t);
+				if (!n || ![
+					"succeeded",
+					"failed",
+					"skipped",
+					"missed"
+				].includes(n.status)) throw Error("Backfill firing has no terminal recorded outcome");
+				switch (n.status) {
+					case "succeeded":
+					case "failed":
+					case "skipped":
+					case "missed": return {
+						runId: t,
+						status: n.status
+					};
+					default: throw Error("Backfill firing is still running");
+				}
+			}).catch((e) => T.error?.("schedule backfill execution unavailable", {
+				requestId: r.requestId,
+				reason: M(e)
+			})), o.request;
+		},
+		backfillRange: async (e, t, n, r) => {
+			let i = V.get(e);
+			if (!i) throw Error(`scheduler.backfillRange: unknown schedule "${e}"`);
+			if (!(t.getTime() < n.getTime())) throw Error(`scheduler.backfillRange("${e}"): 'from' (${t.toISOString()}) must be before 'to' (${n.toISOString()})`);
+			let a = d(i, t, n, r.cap);
+			if (a.length > r.cap) throw Error(`scheduler.backfillRange("${e}"): more than ${r.cap} occurrences in ${t.toISOString()}..${n.toISOString()} — narrow the range or raise the cap`);
+			let o = [], s = 0;
+			for (let e of a) {
+				let t = await J(i, e, "manual");
+				o.push(t ?? null), t != null && s++;
+			}
+			return {
+				occurrences: a.length,
+				fired: s,
+				runIds: o
+			};
+		},
+		shutdown: async () => {
+			I = !0;
+			let e = L !== void 0 || R !== void 0;
+			clearTimeout(L), await R, e && await ee(S, be), $ && clearTimeout($);
+			for (let e of P.values()) clearTimeout(e);
+			P.clear();
+			for (let e of [...G]) e();
+			G.clear(), Te?.(), T.info("scheduler: stopped");
+		}
+	};
+}, M = (e) => e?.message ?? String(e), he = (e) => e && typeof e == "object" && "_tag" in e ? String(e._tag) : null, ge = (e) => {
+	let t = {};
+	for (let [n, r] of Object.entries(e)) t[n] = r instanceof Date ? r.toISOString() : r;
+	return t;
+};
+//#endregion
+export { w as CLAIM_RETENTION_BUCKETS, oe as DEFAULT_CLAIM_BUCKET_MS, ce as DEFAULT_EXTERNAL_RECONCILE_MS, se as DEFAULT_EXTERNAL_TRIGGER_SKEW_MS, fe as DEFAULT_SCHEDULE_HEARTBEAT_MS, x as DORMANCY_WAKEUP_TENANT, le as EXTERNAL_TRIGGER_AHEAD_TOLERANCE_MS, b as SCHEDULE_SUBJECT, de as TERMINAL_RUN_STATUSES, T as claimGraceMs, E as makeAdvisoryLockCoordinator, A as peerRunIsAlive, S as singleCoordinator, j as startScheduler };