@gnldev/scheduler 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/README.md +110 -0
- package/dist/cron.d.ts +7 -0
- package/dist/cron.js +66 -0
- package/dist/cron.js.map +1 -0
- package/dist/index.d.ts +147 -0
- package/dist/index.js +441 -0
- package/dist/index.js.map +1 -0
- package/dist/workflow-waker.d.ts +60 -0
- package/dist/workflow-waker.js +101 -0
- package/dist/workflow-waker.js.map +1 -0
- package/package.json +60 -0
package/dist/index.js
ADDED
|
@@ -0,0 +1,441 @@
|
|
|
1
|
+
// @gnldev/scheduler — durable workflow scheduler on top of @gnldev/durable.
|
|
2
|
+
// Keeps triggers in the journal (definition immutable, state mutable). The poll loop (now=Date.now())
|
|
3
|
+
// fires due triggers exactly-once (acquireRunLock + per-fireCount runId). The workflow run carries its
|
|
4
|
+
// own durable guarantee. Time = DATA (nextRunAt in the journal) → resolve-then-freeze, replay-safe.
|
|
5
|
+
import { acquireRunLock, createPollLoop } from '@gnldev/durable';
|
|
6
|
+
import { nextCronTime } from './cron.js';
|
|
7
|
+
export { nextCronTime, parseField } from './cron.js';
|
|
8
|
+
export { createWorkflowWaker } from './workflow-waker.js';
|
|
9
|
+
const DEF = (id) => `sched:def:${id}`;
|
|
10
|
+
const STATE = (id) => `sched:state:${id}`;
|
|
11
|
+
const FAIL = (id) => `sched:fail:${id}`;
|
|
12
|
+
const BUDGET_SKIP = (id) => `sched:budget-skip:${id}`;
|
|
13
|
+
const BUSY_SKIP = (id) => `sched:busy-skip:${id}`;
|
|
14
|
+
/**
|
|
15
|
+
* THE FIRE LOCK'S KEY — deliberately NOT the runId.
|
|
16
|
+
*
|
|
17
|
+
* This lock answers "is another POLLER firing this occurrence?". The runId's own lock answers "is
|
|
18
|
+
* this RUN executing anywhere?". Two different questions, and for a long time they were written to
|
|
19
|
+
* the same place: `acquireRunLock(journal, runId, …)` produces `<runId>:lock`, so a poller holding
|
|
20
|
+
* the fire lock was holding the run's lock too.
|
|
21
|
+
*
|
|
22
|
+
* That was invisible until a run started taking its own lock. Measured, in a `preset: 'critical'`
|
|
23
|
+
* app: the poller locked `sched:saglik-5dk:0:lock`, then called `runWorkflow` with that same runId,
|
|
24
|
+
* and the critical preset's run-lock (registry.ts) found the key occupied and refused —
|
|
25
|
+
* `RunBusyError`, "already running, locked by another process". The other process was the caller.
|
|
26
|
+
* Five polls, five refusals, `status: 'failed'`, and not one step ever ran. Nothing raced; it simply
|
|
27
|
+
* could not work.
|
|
28
|
+
*
|
|
29
|
+
* The two locks now live on separate keys and are free to be held at the same time, which is the
|
|
30
|
+
* correct arrangement: they nest rather than compete. Fire exclusivity is unchanged (the key is still
|
|
31
|
+
* per-occurrence and still CAS-claimed).
|
|
32
|
+
*
|
|
33
|
+
* SUFFIXED rather than moved to a `sched:fire:…` namespace of its own, and that is not a style
|
|
34
|
+
* choice. `<runId>:fire:lock` stays UNDER the run's own key prefix, which is the prefix `purgeRun` /
|
|
35
|
+
* `sweepRuns` delete by — a sibling namespace would have left one small orphan record per fire
|
|
36
|
+
* behind forever, and a five-minute trigger fires a hundred thousand times a year. `parseJournalKey`
|
|
37
|
+
* still ignores it (it claims only `:model:`/`:tool:`), so it stays invisible to replay exactly as
|
|
38
|
+
* `<runId>:lock` always has.
|
|
39
|
+
*
|
|
40
|
+
* MIXED-VERSION NOTE, honestly: during a rolling deploy an old poller and a new one hold DIFFERENT
|
|
41
|
+
* keys for the same occurrence, so both can start the fire. That window is covered by what already
|
|
42
|
+
* covers every other lock loss here — the state write is a CAS (`commit`), so only one poller's
|
|
43
|
+
* result is recorded, and under the critical preset the run's own lock refuses the second executor
|
|
44
|
+
* outright. UNDER THE CRITICAL PRESET it degrades to documented takeover behaviour, not to a double
|
|
45
|
+
* side effect; a non-critical preset has no run lock, so two executors in this window CAN each run a
|
|
46
|
+
* side-effecting tool once — drain the pollers over an upgrade if that matters to the workflow.
|
|
47
|
+
*/
|
|
48
|
+
const FIRE_LOCK = (runId) => `${runId}:fire`;
|
|
49
|
+
/**
|
|
50
|
+
* "Somebody else is already running this exact run" — the durable engine's refusal AT LOCK
|
|
51
|
+
* ACQUISITION, before the run did anything (`registry.ts` for the critical workflow path,
|
|
52
|
+
* `run.ts` for the agent paths; both stamp `atLockAcquisition`).
|
|
53
|
+
*
|
|
54
|
+
* This is NOT a failed attempt. Nothing was tried and nothing went wrong: the work is in flight
|
|
55
|
+
* somewhere else, and the only sane response is to come back later. Counting it against
|
|
56
|
+
* `maxAttempts` is how a busy minute becomes a permanently dead trigger — which is exactly what the
|
|
57
|
+
* live finding was, five deferrals spent as five failures.
|
|
58
|
+
*
|
|
59
|
+
* Deliberately narrower than durable's own `classifyRunError`: that helper also calls a compensated
|
|
60
|
+
* or cancelled run "not a failure", and it is right to, but those are TERMINAL — deferring them
|
|
61
|
+
* would retry them forever. A MID-FLIGHT `RunBusyError` (no `atLockAcquisition`: this poller got in,
|
|
62
|
+
* ran, and was fenced out by a concurrent executor) is left as an ordinary failure for the same
|
|
63
|
+
* reason: something did happen, and it is worth a retry budget.
|
|
64
|
+
*
|
|
65
|
+
* Matched by NAME, not `instanceof`: `@gnldev/durable` is a peer dependency here, and a host with
|
|
66
|
+
* two resolved copies of it would silently fail every `instanceof` check — a lock refusal is not
|
|
67
|
+
* where a version skew should get to change the behaviour.
|
|
68
|
+
*/
|
|
69
|
+
function isRunBusyAtAcquisition(e) {
|
|
70
|
+
return e?.name === 'RunBusyError'
|
|
71
|
+
&& e.atLockAcquisition === true;
|
|
72
|
+
}
|
|
73
|
+
function firstRunAt(spec, now) {
|
|
74
|
+
if (spec.at != null)
|
|
75
|
+
return spec.at;
|
|
76
|
+
if (spec.every != null)
|
|
77
|
+
return now + spec.every;
|
|
78
|
+
if (spec.cron != null)
|
|
79
|
+
return nextCronTime(spec.cron, now);
|
|
80
|
+
throw new Error('scheduleWorkflow: one of at | every | cron is required');
|
|
81
|
+
}
|
|
82
|
+
/**
|
|
83
|
+
* Computes the next fire time. `prevSlot` = the PLANNED slot that just fired (state.nextRunAt) —
|
|
84
|
+
* NOT `now` (poll time). This aligns the 'every' calculation to the planned grid rather than the actual
|
|
85
|
+
* elapsed time → poll delay doesn't accumulate drift (part b). Missed occurrences are skipped or caught
|
|
86
|
+
* up in sequence depending on policy (part a).
|
|
87
|
+
*/
|
|
88
|
+
function computeNext(def, prevSlot, now) {
|
|
89
|
+
if (def.kind === 'every') {
|
|
90
|
+
const interval = def.value;
|
|
91
|
+
if (def.misfire === 'catchup')
|
|
92
|
+
return prevSlot + interval; // next missed occurrence — may be due again immediately
|
|
93
|
+
const missed = Math.floor((now - prevSlot) / interval); // number of fully missed intervals (0 = on time)
|
|
94
|
+
return prevSlot + interval * (missed + 1); // aligned to the planned grid, the first slot right after now
|
|
95
|
+
}
|
|
96
|
+
// cron: nextCronTime already works off the absolute time grid (no drift). The policy difference is
|
|
97
|
+
// where the scan starts from: 'catchup' starts from the last planned slot (finds the next missed one,
|
|
98
|
+
// may be due immediately), 'skip' starts from the current time (skips everything missed, jumps to the
|
|
99
|
+
// next future match).
|
|
100
|
+
return def.misfire === 'catchup' ? nextCronTime(def.value, prevSlot) : nextCronTime(def.value, now);
|
|
101
|
+
}
|
|
102
|
+
function backoff(attempts) {
|
|
103
|
+
return Math.min(1000 * 2 ** (attempts - 1), 60_000);
|
|
104
|
+
}
|
|
105
|
+
/**
|
|
106
|
+
* Where a REPAIRED trigger's fire counter has to start, and why it is not zero.
|
|
107
|
+
*
|
|
108
|
+
* `fireCount` is not bookkeeping: it is part of the run's id (`sched:<id>:<fireCount>`), and a
|
|
109
|
+
* durable run is exactly-once PER ID. A repair that started the counter at 0 would point the first
|
|
110
|
+
* fire at a run that already completed — the engine would hand back the recorded answer, no step
|
|
111
|
+
* would execute, the counter would tick to 1 and do it again. The trigger would look alive on every
|
|
112
|
+
* dashboard and do nothing, which is the same silence this repair exists to end.
|
|
113
|
+
*
|
|
114
|
+
* So the journal is asked what it already knows: the highest `<n>` that ever appeared under this
|
|
115
|
+
* trigger's own key prefix, plus one. Every fire leaves something there — the run's entries, and the
|
|
116
|
+
* fire lock even when the run itself was swept — so the answer is a floor, not a guess.
|
|
117
|
+
*
|
|
118
|
+
* WITHOUT `listKeys` there is no honest answer and it returns 0, which is the historical behaviour of
|
|
119
|
+
* a fresh trigger. That is stated rather than hidden: `pollScheduler` already REQUIRES `listKeys`, so
|
|
120
|
+
* a journal that lacks it cannot run this scheduler at all, and this path is reachable only by a host
|
|
121
|
+
* that schedules through one journal and polls through another.
|
|
122
|
+
*/
|
|
123
|
+
async function fireCountAfterLoss(journal, id) {
|
|
124
|
+
if (!journal.listKeys)
|
|
125
|
+
return 0;
|
|
126
|
+
const prefix = `sched:${id}:`;
|
|
127
|
+
let highest = -1;
|
|
128
|
+
try {
|
|
129
|
+
for (const key of await journal.listKeys(prefix)) {
|
|
130
|
+
const n = /^(\d+)(?::|$)/.exec(key.slice(prefix.length));
|
|
131
|
+
if (n)
|
|
132
|
+
highest = Math.max(highest, Number(n[1]));
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
catch {
|
|
136
|
+
return 0; // a reader that cannot answer is not evidence that nothing ever fired — but see the warn
|
|
137
|
+
}
|
|
138
|
+
return highest + 1;
|
|
139
|
+
}
|
|
140
|
+
/** Schedules a workflow (at | every | cron). Idempotent: repeating with the same id = no-op. Returns the id. */
|
|
141
|
+
export async function scheduleWorkflow(journal, spec, now = Date.now()) {
|
|
142
|
+
const id = spec.id ?? spec.name;
|
|
143
|
+
if ((await journal.get(DEF(id))) !== undefined) {
|
|
144
|
+
/**
|
|
145
|
+
* THE DEFINITION EXISTS. That used to end the function — and it was reading only half the record.
|
|
146
|
+
*
|
|
147
|
+
* A trigger is two entries written together, `sched:def:<id>` and `sched:state:<id>`, and only the
|
|
148
|
+
* second one can be lost on its own: the definition is immutable and nothing rewrites it, while
|
|
149
|
+
* the state is written on every fire and is what a too-wide purge or a retention sweep takes. Once
|
|
150
|
+
* it is gone the trigger is not broken loudly, it is INVISIBLE — `pollScheduler` skips it on its
|
|
151
|
+
* `!state` branch, `listTriggers` drops it as a "partial record" so it is absent from Studio, and
|
|
152
|
+
* this function said "already scheduled" to every restart. Nothing fires and nothing complains.
|
|
153
|
+
* Measured in production: the discovery was somebody noticing that work had not happened.
|
|
154
|
+
*
|
|
155
|
+
* The repair belongs HERE because this is the only place that still holds the spec. `firstRunAt`
|
|
156
|
+
* needs `at | every | cron`, and neither the poller nor a listing has them — they have a def, but
|
|
157
|
+
* a def that has already lost its state has no honest "next time" either. Hosts already call this
|
|
158
|
+
* on every boot (that is the idempotency promise), so the trigger for the repair is free.
|
|
159
|
+
*
|
|
160
|
+
* DEF + STATE BOTH PRESENT: nothing is written, not even the definition. A running trigger's
|
|
161
|
+
* schedule is not silently redefined by a redeploy — that is today's behaviour and it stays.
|
|
162
|
+
*/
|
|
163
|
+
if ((await journal.get(STATE(id))) === undefined) {
|
|
164
|
+
const state = {
|
|
165
|
+
nextRunAt: firstRunAt(spec, now),
|
|
166
|
+
attempts: 0,
|
|
167
|
+
fireCount: await fireCountAfterLoss(journal, id),
|
|
168
|
+
status: 'pending',
|
|
169
|
+
};
|
|
170
|
+
await journal.put(STATE(id), state);
|
|
171
|
+
// LOUD, and deliberately not a debug line. A system that heals itself in silence never shows
|
|
172
|
+
// the operator the thing that keeps breaking it — and what breaks this is usually a purge or a
|
|
173
|
+
// retention rule that runs again next week. `attempts` is reset because the previous attempt
|
|
174
|
+
// history is genuinely gone; the trigger is honestly starting over.
|
|
175
|
+
console.warn(`[scheduler] orphaned trigger state repaired (trigger ${id}, workflow ${spec.name}) — the definition was in the journal but 'sched:state:${id}' was missing, so the trigger was invisible to the poller AND to listTriggers, and could not have fired again. A fresh state was written: nextRunAt=${state.nextRunAt}, attempts=0, fireCount=${state.fireCount}. Find out what deleted it (a too-wide purge or a retention sweep is the usual cause) — this will happen again otherwise.`);
|
|
176
|
+
}
|
|
177
|
+
return id;
|
|
178
|
+
}
|
|
179
|
+
// A trigger nobody has seen before: both halves are written here, and only here.
|
|
180
|
+
const kind = spec.at != null ? 'at' : spec.every != null ? 'every' : 'cron';
|
|
181
|
+
const value = spec.at ?? spec.every ?? spec.cron;
|
|
182
|
+
const def = {
|
|
183
|
+
name: spec.name,
|
|
184
|
+
input: spec.input,
|
|
185
|
+
kind,
|
|
186
|
+
value,
|
|
187
|
+
maxAttempts: spec.maxAttempts ?? 5,
|
|
188
|
+
misfire: spec.misfire ?? 'skip',
|
|
189
|
+
};
|
|
190
|
+
await journal.put(DEF(id), def);
|
|
191
|
+
const state = { nextRunAt: firstRunAt(spec, now), attempts: 0, fireCount: 0, status: 'pending' };
|
|
192
|
+
await journal.put(STATE(id), state);
|
|
193
|
+
return id;
|
|
194
|
+
}
|
|
195
|
+
/**
|
|
196
|
+
* Fires triggers that are due ('pending' && now≥nextRunAt). Double-firing is prevented via the run-lock;
|
|
197
|
+
* the real exactly-once guarantee comes from the durable workflow run (per-fireCount runId).
|
|
198
|
+
*
|
|
199
|
+
* Y2: the lock is kept alive by a heartbeat while the workflow runs (`lockTtlMs`, default 60s, renewed
|
|
200
|
+
* every ttl/3) — a long workflow no longer lets a second poller take the fire over. And EVERY state
|
|
201
|
+
* write is a CAS (`putIfMatch`) against the state read at the start of the fire, so even if a takeover
|
|
202
|
+
* does happen the late poller cannot write its stale result back.
|
|
203
|
+
*/
|
|
204
|
+
export async function pollScheduler(journal, runner, now = Date.now(), opts = {}) {
|
|
205
|
+
if (!journal.listKeys)
|
|
206
|
+
throw new Error('@gnldev/scheduler: journal.listKeys is required (trigger enumeration)');
|
|
207
|
+
const owner = opts.owner ?? `sched-${Math.random().toString(36).slice(2, 8)}`;
|
|
208
|
+
const retryMs = opts.retryMs ?? 30_000;
|
|
209
|
+
const lockTtlMs = opts.lockTtlMs ?? 60_000;
|
|
210
|
+
const out = { fired: 0, rescheduled: 0, failed: 0, skipped: 0 };
|
|
211
|
+
const defKeys = await journal.listKeys('sched:def:');
|
|
212
|
+
for (const dkey of defKeys) {
|
|
213
|
+
const id = dkey.slice('sched:def:'.length);
|
|
214
|
+
const def = await journal.get(DEF(id));
|
|
215
|
+
const state = await journal.get(STATE(id));
|
|
216
|
+
if (!def || !state || state.status !== 'pending' || now < state.nextRunAt)
|
|
217
|
+
continue;
|
|
218
|
+
const runId = `sched:${id}:${state.fireCount}`;
|
|
219
|
+
// The fire lock, on its OWN key — see FIRE_LOCK. `runId` below is the RUN's name and is passed to
|
|
220
|
+
// the runner untouched; the two must not be the same lock.
|
|
221
|
+
const lock = await acquireRunLock(journal, FIRE_LOCK(runId), owner, lockTtlMs, now);
|
|
222
|
+
if (!lock) {
|
|
223
|
+
// SESSİZ DEĞİL: bu dal `out.skipped`'a da yazmıyordu, log da basmıyordu — yani zamanlanmış bir
|
|
224
|
+
// tetik atlandığında hiçbir iz kalmıyordu. Atlama DOĞRU (başka bir poller o ateşlemeyi tutuyor),
|
|
225
|
+
// ama görünmez olması "sessiz-VE-görünmez hiçbir şey olamaz" kuralının ihlali: operatör
|
|
226
|
+
// "tetik çalışmadı mı, atlandı mı, çakıştı mı" sorusunu cevaplayamıyordu.
|
|
227
|
+
out.skipped++;
|
|
228
|
+
continue;
|
|
229
|
+
}
|
|
230
|
+
// Y2 (heartbeat): the lock TTL used to be a FIXED 60s that was never renewed — a workflow running
|
|
231
|
+
// longer than that let the lock expire, a second poller took it over and fired the SAME trigger
|
|
232
|
+
// again. The core's `RunLock.renew()` (run-lock.ts) exists exactly for this: "long-running jobs
|
|
233
|
+
// should call this at an interval shorter than the ttl". We renew every ttl/3 for as long as the
|
|
234
|
+
// trigger is in flight — the same pattern as @gnldev/queue's createWorker.
|
|
235
|
+
// A renew returning FALSE (a real takeover) or THROWING (a transient journal hiccup) is only
|
|
236
|
+
// LOGGED here: it deliberately does NOT gate the STATE WRITES below, because those are already
|
|
237
|
+
// gated by something stronger and exact — see `commit`. A `lockLost` flag (the queue's approach)
|
|
238
|
+
// would be a delayed approximation for that job and can be flipped by a mere network blip.
|
|
239
|
+
// KNOWN LIMIT — the CAS covers the WRITE, not the EXECUTION: when renew() returns false the
|
|
240
|
+
// workflow this poller already started KEEPS RUNNING to completion, so during a takeover window
|
|
241
|
+
// the same runId can be in flight in two pollers at once (only one of them can commit). The
|
|
242
|
+
// durable run's own per-runId guarantee is what keeps that convergent; making the poller actually
|
|
243
|
+
// STOP would need runWorkflow's AbortSignal to be plumbed through from here — it is not, today.
|
|
244
|
+
// A `lockLost` flag would not have fixed this either (the workflow is already running).
|
|
245
|
+
const heartbeat = setInterval(() => {
|
|
246
|
+
lock
|
|
247
|
+
.renew(lockTtlMs)
|
|
248
|
+
.then((ok) => {
|
|
249
|
+
if (!ok)
|
|
250
|
+
console.warn(`[scheduler] the lock was taken over (trigger ${id}, run ${runId}) — the CAS on the state write will decide the result.`);
|
|
251
|
+
})
|
|
252
|
+
.catch((err) => console.warn(`[scheduler] renew transient error (trigger ${id}), the next tick will retry:`, err));
|
|
253
|
+
}, Math.max(1, Math.floor(lockTtlMs / 3)));
|
|
254
|
+
// Fencing: every state write is a CAS against the state we read AT THE START of this fire
|
|
255
|
+
// (`expected = state`). CAS — not a "do I still hold the lock?" hunch — is the AUTHORITY FOR THE
|
|
256
|
+
// WRITE (and only for the write; execution is not fenced, see the KNOWN LIMIT above): the
|
|
257
|
+
// lock is advisory and any ownership check is inherently a read at a point in time (it can go
|
|
258
|
+
// stale between the check and the write), whereas putIfMatch decides ATOMICALLY at write time,
|
|
259
|
+
// inside the journal. If somebody else advanced the trigger (took the fire over and wrote its
|
|
260
|
+
// own result), our record no longer matches and this write is REJECTED — a late poller can no
|
|
261
|
+
// longer roll fireCount/nextRunAt back or resurrect `attempts`. The write of the poller that
|
|
262
|
+
// genuinely holds the lock always matches, so a correct poller is never blocked.
|
|
263
|
+
// A journal WITHOUT putIfMatch falls back to an unconditional put (old behavior, documented risk
|
|
264
|
+
// — the same fallback as run-lock.ts / claim()).
|
|
265
|
+
const commit = async (next, what) => {
|
|
266
|
+
if (!journal.putIfMatch) {
|
|
267
|
+
await journal.put(STATE(id), next);
|
|
268
|
+
return true;
|
|
269
|
+
}
|
|
270
|
+
if (await journal.putIfMatch(STATE(id), state, next))
|
|
271
|
+
return true;
|
|
272
|
+
console.warn(`[scheduler] stale state write rejected (trigger ${id}, ${what}) — another poller advanced this trigger; this poller's result is DISCARDED.`);
|
|
273
|
+
return false;
|
|
274
|
+
};
|
|
275
|
+
try {
|
|
276
|
+
// 1.4: optional budget/quota hook — checked before runner.runWorkflow is CALLED.
|
|
277
|
+
if (opts.budgetGuard) {
|
|
278
|
+
try {
|
|
279
|
+
await opts.budgetGuard({ triggerId: id, workflowName: def.name, input: def.input, now });
|
|
280
|
+
}
|
|
281
|
+
catch (e) {
|
|
282
|
+
// attempts DOES NOT increase; the diagnostic record is only written if the CAS was won (a
|
|
283
|
+
// stale poller must not leave a budget-skip note on someone else's fire either).
|
|
284
|
+
if (await commit({ ...state, nextRunAt: now + retryMs }, 'budget-skip')) {
|
|
285
|
+
await journal.put(BUDGET_SKIP(id), { error: String(e?.message ?? e), at: now });
|
|
286
|
+
out.skipped++;
|
|
287
|
+
}
|
|
288
|
+
continue;
|
|
289
|
+
}
|
|
290
|
+
}
|
|
291
|
+
let result;
|
|
292
|
+
try {
|
|
293
|
+
result = await runner.runWorkflow(def.name, def.input, { runId });
|
|
294
|
+
}
|
|
295
|
+
catch (e) {
|
|
296
|
+
// DEFERRAL, not attempt: the run is already in flight elsewhere (see isRunBusyAtAcquisition).
|
|
297
|
+
// `attempts` is untouched, `status` stays 'pending', and — this is the half that cost the live
|
|
298
|
+
// investigation its diagnosis — NOTHING is written to `sched:fail:`. That record is where an
|
|
299
|
+
// operator reads WHY a trigger died, and a run-busy message answers a question nobody asked
|
|
300
|
+
// while burying the answer to the one they did (the live record said "already running"; the
|
|
301
|
+
// real reason the workflow could not run was never written down anywhere).
|
|
302
|
+
//
|
|
303
|
+
// Not silent, though: a deferral that repeats forever is a trigger that never fires, so the
|
|
304
|
+
// record carries a RUNNING TOTAL (never reset — a lifetime count is the number an operator can
|
|
305
|
+
// compare against `fireCount`). There is no cap on purpose: "wait for the other holder" has no
|
|
306
|
+
// honest deadline, and the other side is already bounded by its own lock TTL.
|
|
307
|
+
if (isRunBusyAtAcquisition(e)) {
|
|
308
|
+
if (await commit({ ...state, nextRunAt: now + retryMs }, 'run-busy-skip')) {
|
|
309
|
+
const prev = await journal.get(BUSY_SKIP(id));
|
|
310
|
+
await journal.put(BUSY_SKIP(id), {
|
|
311
|
+
error: String(e?.message ?? e),
|
|
312
|
+
at: now,
|
|
313
|
+
runId,
|
|
314
|
+
deferrals: (prev?.deferrals ?? 0) + 1,
|
|
315
|
+
});
|
|
316
|
+
out.skipped++;
|
|
317
|
+
}
|
|
318
|
+
continue;
|
|
319
|
+
}
|
|
320
|
+
const attempts = state.attempts + 1;
|
|
321
|
+
if (attempts >= def.maxAttempts) {
|
|
322
|
+
if (await commit({ ...state, attempts, status: 'failed' }, 'failed')) {
|
|
323
|
+
await journal.put(FAIL(id), { error: String(e?.message ?? e), at: now });
|
|
324
|
+
out.failed++;
|
|
325
|
+
}
|
|
326
|
+
}
|
|
327
|
+
else if (await commit({ ...state, attempts, nextRunAt: now + backoff(attempts) }, 'retry')) {
|
|
328
|
+
out.rescheduled++;
|
|
329
|
+
}
|
|
330
|
+
continue;
|
|
331
|
+
}
|
|
332
|
+
if (result.suspended) {
|
|
333
|
+
// workflow suspended → the same runId should be resumed later (fireCount unchanged).
|
|
334
|
+
if (await commit({ ...state, nextRunAt: now + retryMs }, 'suspended'))
|
|
335
|
+
out.rescheduled++;
|
|
336
|
+
}
|
|
337
|
+
else if (def.kind === 'at') {
|
|
338
|
+
if (await commit({ ...state, status: 'done' }, 'done'))
|
|
339
|
+
out.fired++;
|
|
340
|
+
}
|
|
341
|
+
else {
|
|
342
|
+
const next = {
|
|
343
|
+
nextRunAt: computeNext(def, state.nextRunAt, now),
|
|
344
|
+
attempts: 0,
|
|
345
|
+
fireCount: state.fireCount + 1,
|
|
346
|
+
status: 'pending',
|
|
347
|
+
};
|
|
348
|
+
if (await commit(next, 'reschedule'))
|
|
349
|
+
out.fired++;
|
|
350
|
+
}
|
|
351
|
+
}
|
|
352
|
+
finally {
|
|
353
|
+
// MUST run BEFORE release(): release() keeps the SAME fencing token (it only pushes `expires`
|
|
354
|
+
// into the past), so a heartbeat tick that survives this fire would find its own token still in
|
|
355
|
+
// the record and RESURRECT the lock it just released (expires: 0 → now+ttl) — blocking every
|
|
356
|
+
// later poll of the same runId (a suspended trigger's resume, most visibly). Pinned by test.
|
|
357
|
+
clearInterval(heartbeat);
|
|
358
|
+
// If the lock was genuinely taken over, release() is a no-op anyway (the fencing token no
|
|
359
|
+
// longer matches — run-lock.ts mkLock.release), so we never free somebody else's lock.
|
|
360
|
+
await lock.release();
|
|
361
|
+
}
|
|
362
|
+
}
|
|
363
|
+
return out;
|
|
364
|
+
}
|
|
365
|
+
/**
|
|
366
|
+
* READ-ONLY trigger listing from the journal, WITHOUT needing a scheduler INSTANCE or a runner (for
|
|
367
|
+
* Studio introspection). Reads the SAME `sched:def:`/`sched:state:` keys (+ `sched:fail:` for 'failed')
|
|
368
|
+
* as `pollScheduler`; does NOT change any state, does not take a lock, does not run a workflow. Returns
|
|
369
|
+
* results sorted alphabetically by id (stable list order).
|
|
370
|
+
*/
|
|
371
|
+
export async function listTriggers(journal) {
|
|
372
|
+
if (!journal.listKeys)
|
|
373
|
+
throw new Error('@gnldev/scheduler: listTriggers requires journal.listKeys (trigger enumeration)');
|
|
374
|
+
const defKeys = await journal.listKeys('sched:def:');
|
|
375
|
+
const out = [];
|
|
376
|
+
for (const dkey of defKeys) {
|
|
377
|
+
const id = dkey.slice('sched:def:'.length);
|
|
378
|
+
const def = await journal.get(DEF(id));
|
|
379
|
+
const state = await journal.get(STATE(id));
|
|
380
|
+
if (!def || !state)
|
|
381
|
+
continue; // inconsistent/partial record (theoretical) — skip
|
|
382
|
+
const info = {
|
|
383
|
+
id,
|
|
384
|
+
name: def.name,
|
|
385
|
+
kind: def.kind,
|
|
386
|
+
value: def.value,
|
|
387
|
+
input: def.input,
|
|
388
|
+
nextRunAt: state.nextRunAt,
|
|
389
|
+
attempts: state.attempts,
|
|
390
|
+
maxAttempts: def.maxAttempts,
|
|
391
|
+
fireCount: state.fireCount,
|
|
392
|
+
status: state.status,
|
|
393
|
+
misfire: def.misfire,
|
|
394
|
+
};
|
|
395
|
+
if (state.status === 'failed') {
|
|
396
|
+
const fail = await journal.get(FAIL(id));
|
|
397
|
+
if (fail) {
|
|
398
|
+
info.lastError = fail.error;
|
|
399
|
+
info.lastErrorAt = fail.at;
|
|
400
|
+
}
|
|
401
|
+
}
|
|
402
|
+
// Read on EVERY status, not just 'failed': a deferred trigger is 'pending' by definition, so
|
|
403
|
+
// gating this the way `lastError` is gated would hide it on exactly the rows that carry it.
|
|
404
|
+
const busy = await journal.get(BUSY_SKIP(id));
|
|
405
|
+
if (busy?.deferrals) {
|
|
406
|
+
info.deferrals = busy.deferrals;
|
|
407
|
+
info.lastDeferralAt = busy.at;
|
|
408
|
+
}
|
|
409
|
+
out.push(info);
|
|
410
|
+
}
|
|
411
|
+
return out.sort((a, b) => a.id.localeCompare(b.id));
|
|
412
|
+
}
|
|
413
|
+
/**
|
|
414
|
+
* Scheduler that manages the poll loop (a self-rescheduling setTimeout chain) — the same pattern as
|
|
415
|
+
* @gnldev/queue's createWorker. `backoff` (default OFF — timing is the scheduler's core contract, see the
|
|
416
|
+
* trade-off below): IF ENABLED, when a poll fires NO triggers at all (`fired === 0`) the next poll
|
|
417
|
+
* interval grows ×2 (cap: `maxPollMs ?? pollMs*32`) → prevents tens of thousands of empty queries per
|
|
418
|
+
* second (poll storm) on an empty schedule table; the interval resets to `pollMs` once a trigger fires.
|
|
419
|
+
* Trade-off: `backoff: true` cuts idle poll load by ~32x but can delay a trigger that becomes due after
|
|
420
|
+
* a quiet period by up to `maxPollMs` — for timing-critical use (e.g. minute-level cron) the default
|
|
421
|
+
* should stay OFF; only enable it for deployments with many idle-poller instances that can tolerate delay.
|
|
422
|
+
*/
|
|
423
|
+
export function createScheduler(journal, runner, opts = {}) {
|
|
424
|
+
const pollMs = opts.pollMs ?? 1000;
|
|
425
|
+
const backoffOn = opts.backoff ?? false;
|
|
426
|
+
const maxPollMs = opts.maxPollMs ?? pollMs * 32;
|
|
427
|
+
const poll = (now = Date.now()) => pollScheduler(journal, runner, now, { owner: opts.owner, retryMs: opts.retryMs, budgetGuard: opts.budgetGuard, lockTtlMs: opts.lockTtlMs });
|
|
428
|
+
// Phase 8.1: the tick/backoff/"polling" flag loop now lives in @gnldev/durable's shared createPollLoop
|
|
429
|
+
// (it used to be triplicated across queue/events/scheduler) — behavior is identical: pollScheduler only
|
|
430
|
+
// catches runWorkflow errors internally; the rest (journal I/O etc.) are logged and swallowed by
|
|
431
|
+
// createPollLoop (the chain doesn't die). While `fired === 0` and backoffOn, the interval grows ×2
|
|
432
|
+
// (cap maxPollMs); it resets to pollMs once something fires. Default backoff is OFF (see the comment
|
|
433
|
+
// above — timing-critical).
|
|
434
|
+
const loop = createPollLoop(async () => (await poll()).fired > 0, { pollMs, backoff: backoffOn, maxPollMs });
|
|
435
|
+
return {
|
|
436
|
+
poll,
|
|
437
|
+
start: loop.start,
|
|
438
|
+
stop: loop.stop,
|
|
439
|
+
};
|
|
440
|
+
}
|
|
441
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,4EAA4E;AAC5E,sGAAsG;AACtG,uGAAuG;AACvG,oGAAoG;AACpG,OAAO,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAEjE,OAAO,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AAEzC,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AACrD,OAAO,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAmD1D,MAAM,GAAG,GAAG,CAAC,EAAU,EAAE,EAAE,CAAC,aAAa,EAAE,EAAE,CAAC;AAC9C,MAAM,KAAK,GAAG,CAAC,EAAU,EAAE,EAAE,CAAC,eAAe,EAAE,EAAE,CAAC;AAClD,MAAM,IAAI,GAAG,CAAC,EAAU,EAAE,EAAE,CAAC,cAAc,EAAE,EAAE,CAAC;AAChD,MAAM,WAAW,GAAG,CAAC,EAAU,EAAE,EAAE,CAAC,qBAAqB,EAAE,EAAE,CAAC;AAC9D,MAAM,SAAS,GAAG,CAAC,EAAU,EAAE,EAAE,CAAC,mBAAmB,EAAE,EAAE,CAAC;AAE1D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AACH,MAAM,SAAS,GAAG,CAAC,KAAa,EAAE,EAAE,CAAC,GAAG,KAAK,OAAO,CAAC;AAErD;;;;;;;;;;;;;;;;;;;GAmBG;AACH,SAAS,sBAAsB,CAAC,CAAU;IACxC,OAAQ,CAA8B,EAAE,IAAI,KAAK,cAAc;WACzD,CAAqC,CAAC,iBAAiB,KAAK,IAAI,CAAC;AACzE,CAAC;AAeD,SAAS,UAAU,CAAC,IAAkB,EAAE,GAAW;IACjD,IAAI,IAAI,CAAC,EAAE,IAAI,IAAI;QAAE,OAAO,IAAI,CAAC,EAAE,CAAC;IACpC,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI;QAAE,OAAO,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC;IAChD,IAAI,IAAI,CAAC,IAAI,IAAI,IAAI;QAAE,OAAO,YAAY,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IAC3D,MAAM,IAAI,KAAK,CAAC,wDAAwD,CAAC,CAAC;AAC5E,CAAC;AACD;;;;;GAKG;AACH,SAAS,WAAW,CAAC,GAAe,EAAE,QAAgB,EAAE,GAAW;IACjE,IAAI,GAAG,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;QACzB,MAAM,QAAQ,GAAG,GAAG,CAAC,KAAe,CAAC;QACrC,IAAI,GAAG,CAAC,OAAO,KAAK,SAAS;YAAE,OAAO,QAAQ,GAAG,QAAQ,CAAC,CAAC,wDAAwD;QACnH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,GAAG,QAAQ,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,iDAAiD;QACzG,OAAO,QAAQ,GAAG,QAAQ,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,8DAA8D;IAC3G,CAAC;IACD,mGAAmG;IACnG,sGAAsG;IACtG,sGAAsG;IACtG,sBAAsB;IACtB,OAAO,GAAG,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,KAAe,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,KAAe,EAAE,GAAG,CAAC,CAAC;AAC1H,CAAC;AACD,SAAS,OAAO,CAAC,QAAgB;IAC/B,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;AACtD,CAAC;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,KAAK,UAAU,kBAAkB,CAAC,OAAgB,EAAE,EAAU;IAC5D,IAAI,CAAC,OAAO,CAAC,QAAQ;QAAE,OAAO,CAAC,CAAC;IAChC,MAAM,MAAM,GAAG,SAAS,EAAE,GAAG,CAAC;IAC9B,IAAI,OAAO,GAAG,CAAC,CAAC,CAAC;IACjB,IAAI,CAAC;QACH,KAAK,MAAM,GAAG,IAAI,MAAM,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YACjD,MAAM,CAAC,GAAG,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;YACzD,IAAI,CAAC;gBAAE,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACnD,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,CAAC,CAAC,CAAC,yFAAyF;IACrG,CAAC;IACD,OAAO,OAAO,GAAG,CAAC,CAAC;AACrB,CAAC;AAED,gHAAgH;AAChH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,OAAgB,EAAE,IAAkB,EAAE,MAAc,IAAI,CAAC,GAAG,EAAE;IACnG,MAAM,EAAE,GAAG,IAAI,CAAC,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC;IAChC,IAAI,CAAC,MAAM,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,SAAS,EAAE,CAAC;QAC/C;;;;;;;;;;;;;;;;;;WAkBG;QACH,IAAI,CAAC,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,SAAS,EAAE,CAAC;YACjD,MAAM,KAAK,GAAiB;gBAC1B,SAAS,EAAE,UAAU,CAAC,IAAI,EAAE,GAAG,CAAC;gBAChC,QAAQ,EAAE,CAAC;gBACX,SAAS,EAAE,MAAM,kBAAkB,CAAC,OAAO,EAAE,EAAE,CAAC;gBAChD,MAAM,EAAE,SAAS;aAClB,CAAC;YACF,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,EAAE,KAAK,CAAC,CAAC;YACpC,6FAA6F;YAC7F,+FAA+F;YAC/F,6FAA6F;YAC7F,oEAAoE;YACpE,OAAO,CAAC,IAAI,CACV,wDAAwD,EAAE,cAAc,IAAI,CAAC,IAAI,0DAA0D,EAAE,uJAAuJ,KAAK,CAAC,SAAS,2BAA2B,KAAK,CAAC,SAAS,2HAA2H,CACzd,CAAC;QACJ,CAAC;QACD,OAAO,EAAE,CAAC;IACZ,CAAC;IACD,iFAAiF;IACjF,MAAM,IAAI,GAAS,IAAI,CAAC,EAAE,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC;IAClF,MAAM,KAAK,GAAoB,IAAI,CAAC,EAAE,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,IAAK,CAAC;IACnE,MAAM,GAAG,GAAe;QACtB,IAAI,EAAE,IAAI,CAAC,IAAI;QACf,KAAK,EAAE,IAAI,CAAC,KAAK;QACjB,IAAI;QACJ,KAAK;QACL,WAAW,EAAE,IAAI,CAAC,WAAW,IAAI,CAAC;QAClC,OAAO,EAAE,IAAI,CAAC,OAAO,IAAI,MAAM;KAChC,CAAC;IACF,MAAM,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,GAAG,CAAC,CAAC;IAChC,MAAM,KAAK,GAAiB,EAAE,SAAS,EAAE,UAAU,CAAC,IAAI,EAAE,GAAG,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;IAC/G,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,EAAE,KAAK,CAAC,CAAC;IACpC,OAAO,EAAE,CAAC;AACZ,CAAC;AAiBD;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,OAAgB,EAChB,MAAsB,EACtB,MAAc,IAAI,CAAC,GAAG,EAAE,EACxB,OAA4F,EAAE;IAE9F,IAAI,CAAC,OAAO,CAAC,QAAQ;QAAE,MAAM,IAAI,KAAK,CAAC,uEAAuE,CAAC,CAAC;IAChH,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,SAAS,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;IAC9E,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,MAAM,CAAC;IACvC,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,MAAM,CAAC;IAC3C,MAAM,GAAG,GAAe,EAAE,KAAK,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;IAE5E,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;IACrD,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;QAC3B,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;QAC3C,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,GAAG,CAAa,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;QACnD,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,GAAG,CAAe,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC;QACzD,IAAI,CAAC,GAAG,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,MAAM,KAAK,SAAS,IAAI,GAAG,GAAG,KAAK,CAAC,SAAS;YAAE,SAAS;QAEpF,MAAM,KAAK,GAAG,SAAS,EAAE,IAAI,KAAK,CAAC,SAAS,EAAE,CAAC;QAC/C,kGAAkG;QAClG,2DAA2D;QAC3D,MAAM,IAAI,GAAG,MAAM,cAAc,CAAC,OAAO,EAAE,SAAS,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,CAAC,CAAC;QACpF,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,+FAA+F;YAC/F,iGAAiG;YACjG,wFAAwF;YACxF,0EAA0E;YAC1E,GAAG,CAAC,OAAO,EAAE,CAAC;YACd,SAAS;QACX,CAAC;QAED,kGAAkG;QAClG,gGAAgG;QAChG,gGAAgG;QAChG,iGAAiG;QACjG,2EAA2E;QAC3E,6FAA6F;QAC7F,+FAA+F;QAC/F,iGAAiG;QACjG,2FAA2F;QAC3F,4FAA4F;QAC5F,gGAAgG;QAChG,4FAA4F;QAC5F,kGAAkG;QAClG,gGAAgG;QAChG,wFAAwF;QACxF,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE;YACjC,IAAI;iBACD,KAAK,CAAC,SAAS,CAAC;iBAChB,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE;gBACX,IAAI,CAAC,EAAE;oBAAE,OAAO,CAAC,IAAI,CAAC,gDAAgD,EAAE,SAAS,KAAK,wDAAwD,CAAC,CAAC;YAClJ,CAAC,CAAC;iBACD,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,8CAA8C,EAAE,8BAA8B,EAAE,GAAG,CAAC,CAAC,CAAC;QACvH,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAE3C,0FAA0F;QAC1F,iGAAiG;QACjG,0FAA0F;QAC1F,8FAA8F;QAC9F,+FAA+F;QAC/F,8FAA8F;QAC9F,8FAA8F;QAC9F,6FAA6F;QAC7F,iFAAiF;QACjF,iGAAiG;QACjG,iDAAiD;QACjD,MAAM,MAAM,GAAG,KAAK,EAAE,IAAkB,EAAE,IAAY,EAAoB,EAAE;YAC1E,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC;gBACxB,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,EAAE,IAAI,CAAC,CAAC;gBACnC,OAAO,IAAI,CAAC;YACd,CAAC;YACD,IAAI,MAAM,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC;gBAAE,OAAO,IAAI,CAAC;YAClE,OAAO,CAAC,IAAI,CAAC,mDAAmD,EAAE,KAAK,IAAI,8EAA8E,CAAC,CAAC;YAC3J,OAAO,KAAK,CAAC;QACf,CAAC,CAAC;QAEF,IAAI,CAAC;YACH,iFAAiF;YACjF,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;gBACrB,IAAI,CAAC;oBACH,MAAM,IAAI,CAAC,WAAW,CAAC,EAAE,SAAS,EAAE,EAAE,EAAE,YAAY,EAAE,GAAG,CAAC,IAAI,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;gBAC3F,CAAC;gBAAC,OAAO,CAAC,EAAE,CAAC;oBACX,0FAA0F;oBAC1F,iFAAiF;oBACjF,IAAI,MAAM,MAAM,CAAC,EAAE,GAAG,KAAK,EAAE,SAAS,EAAE,GAAG,GAAG,OAAO,EAAE,EAAE,aAAa,CAAC,EAAE,CAAC;wBACxE,MAAM,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,CAAC,EAAE,EAAE,KAAK,EAAE,MAAM,CAAE,CAAS,EAAE,OAAO,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;wBACzF,GAAG,CAAC,OAAO,EAAE,CAAC;oBAChB,CAAC;oBACD,SAAS;gBACX,CAAC;YACH,CAAC;YACD,IAAI,MAAiD,CAAC;YACtD,IAAI,CAAC;gBACH,MAAM,GAAG,MAAM,MAAM,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;YACpE,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,8FAA8F;gBAC9F,+FAA+F;gBAC/F,6FAA6F;gBAC7F,4FAA4F;gBAC5F,4FAA4F;gBAC5F,2EAA2E;gBAC3E,EAAE;gBACF,4FAA4F;gBAC5F,+FAA+F;gBAC/F,+FAA+F;gBAC/F,8EAA8E;gBAC9E,IAAI,sBAAsB,CAAC,CAAC,CAAC,EAAE,CAAC;oBAC9B,IAAI,MAAM,MAAM,CAAC,EAAE,GAAG,KAAK,EAAE,SAAS,EAAE,GAAG,GAAG,OAAO,EAAE,EAAE,eAAe,CAAC,EAAE,CAAC;wBAC1E,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,GAAG,CAAyB,SAAS,CAAC,EAAE,CAAC,CAAC,CAAC;wBACtE,MAAM,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC,EAAE;4BAC/B,KAAK,EAAE,MAAM,CAAE,CAAS,EAAE,OAAO,IAAI,CAAC,CAAC;4BACvC,EAAE,EAAE,GAAG;4BACP,KAAK;4BACL,SAAS,EAAE,CAAC,IAAI,EAAE,SAAS,IAAI,CAAC,CAAC,GAAG,CAAC;yBACtC,CAAC,CAAC;wBACH,GAAG,CAAC,OAAO,EAAE,CAAC;oBAChB,CAAC;oBACD,SAAS;gBACX,CAAC;gBACD,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,GAAG,CAAC,CAAC;gBACpC,IAAI,QAAQ,IAAI,GAAG,CAAC,WAAW,EAAE,CAAC;oBAChC,IAAI,MAAM,MAAM,CAAC,EAAE,GAAG,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE,QAAQ,CAAC,EAAE,CAAC;wBACrE,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,KAAK,EAAE,MAAM,CAAE,CAAS,EAAE,OAAO,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;wBAClF,GAAG,CAAC,MAAM,EAAE,CAAC;oBACf,CAAC;gBACH,CAAC;qBAAM,IAAI,MAAM,MAAM,CAAC,EAAE,GAAG,KAAK,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,GAAG,OAAO,CAAC,QAAQ,CAAC,EAAE,EAAE,OAAO,CAAC,EAAE,CAAC;oBAC7F,GAAG,CAAC,WAAW,EAAE,CAAC;gBACpB,CAAC;gBACD,SAAS;YACX,CAAC;YAED,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;gBACrB,qFAAqF;gBACrF,IAAI,MAAM,MAAM,CAAC,EAAE,GAAG,KAAK,EAAE,SAAS,EAAE,GAAG,GAAG,OAAO,EAAE,EAAE,WAAW,CAAC;oBAAE,GAAG,CAAC,WAAW,EAAE,CAAC;YAC3F,CAAC;iBAAM,IAAI,GAAG,CAAC,IAAI,KAAK,IAAI,EAAE,CAAC;gBAC7B,IAAI,MAAM,MAAM,CAAC,EAAE,GAAG,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE,MAAM,CAAC;oBAAE,GAAG,CAAC,KAAK,EAAE,CAAC;YACtE,CAAC;iBAAM,CAAC;gBACN,MAAM,IAAI,GAAiB;oBACzB,SAAS,EAAE,WAAW,CAAC,GAAG,EAAE,KAAK,CAAC,SAAS,EAAE,GAAG,CAAC;oBACjD,QAAQ,EAAE,CAAC;oBACX,SAAS,EAAE,KAAK,CAAC,SAAS,GAAG,CAAC;oBAC9B,MAAM,EAAE,SAAS;iBAClB,CAAC;gBACF,IAAI,MAAM,MAAM,CAAC,IAAI,EAAE,YAAY,CAAC;oBAAE,GAAG,CAAC,KAAK,EAAE,CAAC;YACpD,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,8FAA8F;YAC9F,gGAAgG;YAChG,6FAA6F;YAC7F,6FAA6F;YAC7F,aAAa,CAAC,SAAS,CAAC,CAAC;YACzB,0FAA0F;YAC1F,uFAAuF;YACvF,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;QACvB,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAkCD;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,OAAgB;IACjD,IAAI,CAAC,OAAO,CAAC,QAAQ;QAAE,MAAM,IAAI,KAAK,CAAC,iFAAiF,CAAC,CAAC;IAC1H,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;IACrD,MAAM,GAAG,GAAkB,EAAE,CAAC;IAC9B,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;QAC3B,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;QAC3C,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,GAAG,CAAa,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;QACnD,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,GAAG,CAAe,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC;QACzD,IAAI,CAAC,GAAG,IAAI,CAAC,KAAK;YAAE,SAAS,CAAC,mDAAmD;QACjF,MAAM,IAAI,GAAgB;YACxB,EAAE;YACF,IAAI,EAAE,GAAG,CAAC,IAAI;YACd,IAAI,EAAE,GAAG,CAAC,IAAI;YACd,KAAK,EAAE,GAAG,CAAC,KAAK;YAChB,KAAK,EAAE,GAAG,CAAC,KAAK;YAChB,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,WAAW,EAAE,GAAG,CAAC,WAAW;YAC5B,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,OAAO,EAAE,GAAG,CAAC,OAAO;SACrB,CAAC;QACF,IAAI,KAAK,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;YAC9B,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,GAAG,CAAgC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;YACxE,IAAI,IAAI,EAAE,CAAC;gBACT,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC;gBAC5B,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,EAAE,CAAC;YAC7B,CAAC;QACH,CAAC;QACD,6FAA6F;QAC7F,4FAA4F;QAC5F,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,GAAG,CAAqC,SAAS,CAAC,EAAE,CAAC,CAAC,CAAC;QAClF,IAAI,IAAI,EAAE,SAAS,EAAE,CAAC;YACpB,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC;YAChC,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,EAAE,CAAC;QAChC,CAAC;QACD,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACjB,CAAC;IACD,OAAO,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AACtD,CAAC;AAQD;;;;;;;;;GASG;AACH,MAAM,UAAU,eAAe,CAC7B,OAAgB,EAChB,MAAsB,EACtB,OAAoJ,EAAE;IAEtJ,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC;IACnC,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,IAAI,KAAK,CAAC;IACxC,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,MAAM,GAAG,EAAE,CAAC;IAChD,MAAM,IAAI,GAAG,CAAC,MAAc,IAAI,CAAC,GAAG,EAAE,EAAE,EAAE,CACxC,aAAa,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,EAAE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;IAE9I,uGAAuG;IACvG,wGAAwG;IACxG,iGAAiG;IACjG,mGAAmG;IACnG,qGAAqG;IACrG,4BAA4B;IAC5B,MAAM,IAAI,GAAG,cAAc,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,KAAK,GAAG,CAAC,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC,CAAC;IAE7G,OAAO;QACL,IAAI;QACJ,KAAK,EAAE,IAAI,CAAC,KAAK;QACjB,IAAI,EAAE,IAAI,CAAC,IAAI;KAChB,CAAC;AACJ,CAAC","sourcesContent":["// @gnldev/scheduler — durable workflow scheduler on top of @gnldev/durable.\n// Keeps triggers in the journal (definition immutable, state mutable). The poll loop (now=Date.now())\n// fires due triggers exactly-once (acquireRunLock + per-fireCount runId). The workflow run carries its\n// own durable guarantee. Time = DATA (nextRunAt in the journal) → resolve-then-freeze, replay-safe.\nimport { acquireRunLock, createPollLoop } from '@gnldev/durable';\nimport type { Journal } from '@gnldev/durable';\nimport { nextCronTime } from './cron.js';\n\nexport { nextCronTime, parseField } from './cron.js';\nexport { createWorkflowWaker } from './workflow-waker.js';\nexport type { WorkflowWaker, WorkflowWakerOptions, WorkflowWakerTickResult } from './workflow-waker.js';\n\n/** Structurally compatible with what createGnl returns — no hard dependency. */\nexport interface WorkflowRunner {\n runWorkflow(\n name: string,\n input: unknown,\n opts?: { runId?: string },\n ): Promise<{ runId: string; suspended?: boolean; output?: unknown }>;\n}\n\nexport interface ScheduleSpec {\n /** Trigger id (idempotent record; `name` is used if not given). */\n id?: string;\n /** Workflow name to run (passed to runner.runWorkflow). */\n name: string;\n input?: unknown;\n /** Absolute time (epoch ms) — one-shot. */\n at?: number;\n /** Period (ms) — repeats every interval. */\n every?: number;\n /** 5-field cron (UTC, minute resolution) — repeats on every match. */\n cron?: string;\n /** Max attempts on failure (default 5). */\n maxAttempts?: number;\n /**\n * Policy for missed (misfire) fires — only meaningful for every/cron:\n * 'skip' (default) — skip missed fires, jump to the next slot aligned to the planned grid (no drift accumulates).\n * 'catchup' — fire each missed occurrence in sequence (one per poll), none are skipped.\n */\n misfire?: MisfirePolicy;\n}\n\nexport type Kind = 'at' | 'every' | 'cron';\nexport type MisfirePolicy = 'skip' | 'catchup';\ninterface TriggerDef {\n name: string;\n input: unknown;\n kind: Kind;\n value: number | string;\n maxAttempts: number;\n misfire: MisfirePolicy;\n}\ninterface TriggerState {\n nextRunAt: number;\n attempts: number;\n fireCount: number;\n status: 'pending' | 'done' | 'failed';\n}\n\nconst DEF = (id: string) => `sched:def:${id}`;\nconst STATE = (id: string) => `sched:state:${id}`;\nconst FAIL = (id: string) => `sched:fail:${id}`;\nconst BUDGET_SKIP = (id: string) => `sched:budget-skip:${id}`;\nconst BUSY_SKIP = (id: string) => `sched:busy-skip:${id}`;\n\n/**\n * THE FIRE LOCK'S KEY — deliberately NOT the runId.\n *\n * This lock answers \"is another POLLER firing this occurrence?\". The runId's own lock answers \"is\n * this RUN executing anywhere?\". Two different questions, and for a long time they were written to\n * the same place: `acquireRunLock(journal, runId, …)` produces `<runId>:lock`, so a poller holding\n * the fire lock was holding the run's lock too.\n *\n * That was invisible until a run started taking its own lock. Measured, in a `preset: 'critical'`\n * app: the poller locked `sched:saglik-5dk:0:lock`, then called `runWorkflow` with that same runId,\n * and the critical preset's run-lock (registry.ts) found the key occupied and refused —\n * `RunBusyError`, \"already running, locked by another process\". The other process was the caller.\n * Five polls, five refusals, `status: 'failed'`, and not one step ever ran. Nothing raced; it simply\n * could not work.\n *\n * The two locks now live on separate keys and are free to be held at the same time, which is the\n * correct arrangement: they nest rather than compete. Fire exclusivity is unchanged (the key is still\n * per-occurrence and still CAS-claimed).\n *\n * SUFFIXED rather than moved to a `sched:fire:…` namespace of its own, and that is not a style\n * choice. `<runId>:fire:lock` stays UNDER the run's own key prefix, which is the prefix `purgeRun` /\n * `sweepRuns` delete by — a sibling namespace would have left one small orphan record per fire\n * behind forever, and a five-minute trigger fires a hundred thousand times a year. `parseJournalKey`\n * still ignores it (it claims only `:model:`/`:tool:`), so it stays invisible to replay exactly as\n * `<runId>:lock` always has.\n *\n * MIXED-VERSION NOTE, honestly: during a rolling deploy an old poller and a new one hold DIFFERENT\n * keys for the same occurrence, so both can start the fire. That window is covered by what already\n * covers every other lock loss here — the state write is a CAS (`commit`), so only one poller's\n * result is recorded, and under the critical preset the run's own lock refuses the second executor\n * outright. UNDER THE CRITICAL PRESET it degrades to documented takeover behaviour, not to a double\n * side effect; a non-critical preset has no run lock, so two executors in this window CAN each run a\n * side-effecting tool once — drain the pollers over an upgrade if that matters to the workflow.\n */\nconst FIRE_LOCK = (runId: string) => `${runId}:fire`;\n\n/**\n * \"Somebody else is already running this exact run\" — the durable engine's refusal AT LOCK\n * ACQUISITION, before the run did anything (`registry.ts` for the critical workflow path,\n * `run.ts` for the agent paths; both stamp `atLockAcquisition`).\n *\n * This is NOT a failed attempt. Nothing was tried and nothing went wrong: the work is in flight\n * somewhere else, and the only sane response is to come back later. Counting it against\n * `maxAttempts` is how a busy minute becomes a permanently dead trigger — which is exactly what the\n * live finding was, five deferrals spent as five failures.\n *\n * Deliberately narrower than durable's own `classifyRunError`: that helper also calls a compensated\n * or cancelled run \"not a failure\", and it is right to, but those are TERMINAL — deferring them\n * would retry them forever. A MID-FLIGHT `RunBusyError` (no `atLockAcquisition`: this poller got in,\n * ran, and was fenced out by a concurrent executor) is left as an ordinary failure for the same\n * reason: something did happen, and it is worth a retry budget.\n *\n * Matched by NAME, not `instanceof`: `@gnldev/durable` is a peer dependency here, and a host with\n * two resolved copies of it would silently fail every `instanceof` check — a lock refusal is not\n * where a version skew should get to change the behaviour.\n */\nfunction isRunBusyAtAcquisition(e: unknown): boolean {\n return (e as { name?: string } | null)?.name === 'RunBusyError'\n && (e as { atLockAcquisition?: boolean }).atLockAcquisition === true;\n}\n\n/**\n * 1.4: optional budget/quota hook — called BEFORE the trigger starts (before runner.runWorkflow is\n * CALLED). Throws on overage (typically `@gnldev/durable`'s `assertBudget` — `BudgetExceededError`);\n * `pollScheduler` does NOT RUN this trigger (skip + records/logs to `sched:budget-skip:<id>`,\n * `out.skipped` increments), state is DEFERRED to retry after `retryMs` but `attempts` DOES NOT\n * increase (a budget overage isn't the workflow's fault → doesn't count toward maxAttempts, `status`\n * stays 'pending'). IF NOT GIVEN (default) behavior is UNCHANGED — no quota check (backward compat).\n * Typical host usage:\n * budgetGuard: () => assertBudget(journal, { orgId, fallback })\n * Kept simple: the scheduler does NOT EMBED quota logic itself, the host injects it (same pattern as limits/guard).\n */\nexport type BudgetGuard = (ctx: { triggerId: string; workflowName: string; input: unknown; now: number }) => Promise<unknown> | unknown;\n\nfunction firstRunAt(spec: ScheduleSpec, now: number): number {\n if (spec.at != null) return spec.at;\n if (spec.every != null) return now + spec.every;\n if (spec.cron != null) return nextCronTime(spec.cron, now);\n throw new Error('scheduleWorkflow: one of at | every | cron is required');\n}\n/**\n * Computes the next fire time. `prevSlot` = the PLANNED slot that just fired (state.nextRunAt) —\n * NOT `now` (poll time). This aligns the 'every' calculation to the planned grid rather than the actual\n * elapsed time → poll delay doesn't accumulate drift (part b). Missed occurrences are skipped or caught\n * up in sequence depending on policy (part a).\n */\nfunction computeNext(def: TriggerDef, prevSlot: number, now: number): number {\n if (def.kind === 'every') {\n const interval = def.value as number;\n if (def.misfire === 'catchup') return prevSlot + interval; // next missed occurrence — may be due again immediately\n const missed = Math.floor((now - prevSlot) / interval); // number of fully missed intervals (0 = on time)\n return prevSlot + interval * (missed + 1); // aligned to the planned grid, the first slot right after now\n }\n // cron: nextCronTime already works off the absolute time grid (no drift). The policy difference is\n // where the scan starts from: 'catchup' starts from the last planned slot (finds the next missed one,\n // may be due immediately), 'skip' starts from the current time (skips everything missed, jumps to the\n // next future match).\n return def.misfire === 'catchup' ? nextCronTime(def.value as string, prevSlot) : nextCronTime(def.value as string, now);\n}\nfunction backoff(attempts: number): number {\n return Math.min(1000 * 2 ** (attempts - 1), 60_000);\n}\n\n/**\n * Where a REPAIRED trigger's fire counter has to start, and why it is not zero.\n *\n * `fireCount` is not bookkeeping: it is part of the run's id (`sched:<id>:<fireCount>`), and a\n * durable run is exactly-once PER ID. A repair that started the counter at 0 would point the first\n * fire at a run that already completed — the engine would hand back the recorded answer, no step\n * would execute, the counter would tick to 1 and do it again. The trigger would look alive on every\n * dashboard and do nothing, which is the same silence this repair exists to end.\n *\n * So the journal is asked what it already knows: the highest `<n>` that ever appeared under this\n * trigger's own key prefix, plus one. Every fire leaves something there — the run's entries, and the\n * fire lock even when the run itself was swept — so the answer is a floor, not a guess.\n *\n * WITHOUT `listKeys` there is no honest answer and it returns 0, which is the historical behaviour of\n * a fresh trigger. That is stated rather than hidden: `pollScheduler` already REQUIRES `listKeys`, so\n * a journal that lacks it cannot run this scheduler at all, and this path is reachable only by a host\n * that schedules through one journal and polls through another.\n */\nasync function fireCountAfterLoss(journal: Journal, id: string): Promise<number> {\n if (!journal.listKeys) return 0;\n const prefix = `sched:${id}:`;\n let highest = -1;\n try {\n for (const key of await journal.listKeys(prefix)) {\n const n = /^(\\d+)(?::|$)/.exec(key.slice(prefix.length));\n if (n) highest = Math.max(highest, Number(n[1]));\n }\n } catch {\n return 0; // a reader that cannot answer is not evidence that nothing ever fired — but see the warn\n }\n return highest + 1;\n}\n\n/** Schedules a workflow (at | every | cron). Idempotent: repeating with the same id = no-op. Returns the id. */\nexport async function scheduleWorkflow(journal: Journal, spec: ScheduleSpec, now: number = Date.now()): Promise<string> {\n const id = spec.id ?? spec.name;\n if ((await journal.get(DEF(id))) !== undefined) {\n /**\n * THE DEFINITION EXISTS. That used to end the function — and it was reading only half the record.\n *\n * A trigger is two entries written together, `sched:def:<id>` and `sched:state:<id>`, and only the\n * second one can be lost on its own: the definition is immutable and nothing rewrites it, while\n * the state is written on every fire and is what a too-wide purge or a retention sweep takes. Once\n * it is gone the trigger is not broken loudly, it is INVISIBLE — `pollScheduler` skips it on its\n * `!state` branch, `listTriggers` drops it as a \"partial record\" so it is absent from Studio, and\n * this function said \"already scheduled\" to every restart. Nothing fires and nothing complains.\n * Measured in production: the discovery was somebody noticing that work had not happened.\n *\n * The repair belongs HERE because this is the only place that still holds the spec. `firstRunAt`\n * needs `at | every | cron`, and neither the poller nor a listing has them — they have a def, but\n * a def that has already lost its state has no honest \"next time\" either. Hosts already call this\n * on every boot (that is the idempotency promise), so the trigger for the repair is free.\n *\n * DEF + STATE BOTH PRESENT: nothing is written, not even the definition. A running trigger's\n * schedule is not silently redefined by a redeploy — that is today's behaviour and it stays.\n */\n if ((await journal.get(STATE(id))) === undefined) {\n const state: TriggerState = {\n nextRunAt: firstRunAt(spec, now),\n attempts: 0,\n fireCount: await fireCountAfterLoss(journal, id),\n status: 'pending',\n };\n await journal.put(STATE(id), state);\n // LOUD, and deliberately not a debug line. A system that heals itself in silence never shows\n // the operator the thing that keeps breaking it — and what breaks this is usually a purge or a\n // retention rule that runs again next week. `attempts` is reset because the previous attempt\n // history is genuinely gone; the trigger is honestly starting over.\n console.warn(\n `[scheduler] orphaned trigger state repaired (trigger ${id}, workflow ${spec.name}) — the definition was in the journal but 'sched:state:${id}' was missing, so the trigger was invisible to the poller AND to listTriggers, and could not have fired again. A fresh state was written: nextRunAt=${state.nextRunAt}, attempts=0, fireCount=${state.fireCount}. Find out what deleted it (a too-wide purge or a retention sweep is the usual cause) — this will happen again otherwise.`,\n );\n }\n return id;\n }\n // A trigger nobody has seen before: both halves are written here, and only here.\n const kind: Kind = spec.at != null ? 'at' : spec.every != null ? 'every' : 'cron';\n const value: number | string = spec.at ?? spec.every ?? spec.cron!;\n const def: TriggerDef = {\n name: spec.name,\n input: spec.input,\n kind,\n value,\n maxAttempts: spec.maxAttempts ?? 5,\n misfire: spec.misfire ?? 'skip',\n };\n await journal.put(DEF(id), def);\n const state: TriggerState = { nextRunAt: firstRunAt(spec, now), attempts: 0, fireCount: 0, status: 'pending' };\n await journal.put(STATE(id), state);\n return id;\n}\n\nexport interface PollResult {\n fired: number;\n rescheduled: number;\n failed: number;\n /**\n * Triggers that were NOT RUN and are NOT a failure — the three deferral reasons, counted together\n * because they mean the same thing to a caller (\"nothing happened, come back later\"):\n * - another poller holds the fire lock for this occurrence (no record; the lock IS the record),\n * - `budgetGuard` was given and threw on overage (`sched:budget-skip:<id>`),\n * - the run is already in flight elsewhere (`sched:busy-skip:<id>`).\n * None of them consume an attempt. Without a guard and without contention this stays 0.\n */\n skipped: number;\n}\n\n/**\n * Fires triggers that are due ('pending' && now≥nextRunAt). Double-firing is prevented via the run-lock;\n * the real exactly-once guarantee comes from the durable workflow run (per-fireCount runId).\n *\n * Y2: the lock is kept alive by a heartbeat while the workflow runs (`lockTtlMs`, default 60s, renewed\n * every ttl/3) — a long workflow no longer lets a second poller take the fire over. And EVERY state\n * write is a CAS (`putIfMatch`) against the state read at the start of the fire, so even if a takeover\n * does happen the late poller cannot write its stale result back.\n */\nexport async function pollScheduler(\n journal: Journal,\n runner: WorkflowRunner,\n now: number = Date.now(),\n opts: { owner?: string; retryMs?: number; budgetGuard?: BudgetGuard; lockTtlMs?: number } = {},\n): Promise<PollResult> {\n if (!journal.listKeys) throw new Error('@gnldev/scheduler: journal.listKeys is required (trigger enumeration)');\n const owner = opts.owner ?? `sched-${Math.random().toString(36).slice(2, 8)}`;\n const retryMs = opts.retryMs ?? 30_000;\n const lockTtlMs = opts.lockTtlMs ?? 60_000;\n const out: PollResult = { fired: 0, rescheduled: 0, failed: 0, skipped: 0 };\n\n const defKeys = await journal.listKeys('sched:def:');\n for (const dkey of defKeys) {\n const id = dkey.slice('sched:def:'.length);\n const def = await journal.get<TriggerDef>(DEF(id));\n const state = await journal.get<TriggerState>(STATE(id));\n if (!def || !state || state.status !== 'pending' || now < state.nextRunAt) continue;\n\n const runId = `sched:${id}:${state.fireCount}`;\n // The fire lock, on its OWN key — see FIRE_LOCK. `runId` below is the RUN's name and is passed to\n // the runner untouched; the two must not be the same lock.\n const lock = await acquireRunLock(journal, FIRE_LOCK(runId), owner, lockTtlMs, now);\n if (!lock) {\n // SESSİZ DEĞİL: bu dal `out.skipped`'a da yazmıyordu, log da basmıyordu — yani zamanlanmış bir\n // tetik atlandığında hiçbir iz kalmıyordu. Atlama DOĞRU (başka bir poller o ateşlemeyi tutuyor),\n // ama görünmez olması \"sessiz-VE-görünmez hiçbir şey olamaz\" kuralının ihlali: operatör\n // \"tetik çalışmadı mı, atlandı mı, çakıştı mı\" sorusunu cevaplayamıyordu.\n out.skipped++;\n continue;\n }\n\n // Y2 (heartbeat): the lock TTL used to be a FIXED 60s that was never renewed — a workflow running\n // longer than that let the lock expire, a second poller took it over and fired the SAME trigger\n // again. The core's `RunLock.renew()` (run-lock.ts) exists exactly for this: \"long-running jobs\n // should call this at an interval shorter than the ttl\". We renew every ttl/3 for as long as the\n // trigger is in flight — the same pattern as @gnldev/queue's createWorker.\n // A renew returning FALSE (a real takeover) or THROWING (a transient journal hiccup) is only\n // LOGGED here: it deliberately does NOT gate the STATE WRITES below, because those are already\n // gated by something stronger and exact — see `commit`. A `lockLost` flag (the queue's approach)\n // would be a delayed approximation for that job and can be flipped by a mere network blip.\n // KNOWN LIMIT — the CAS covers the WRITE, not the EXECUTION: when renew() returns false the\n // workflow this poller already started KEEPS RUNNING to completion, so during a takeover window\n // the same runId can be in flight in two pollers at once (only one of them can commit). The\n // durable run's own per-runId guarantee is what keeps that convergent; making the poller actually\n // STOP would need runWorkflow's AbortSignal to be plumbed through from here — it is not, today.\n // A `lockLost` flag would not have fixed this either (the workflow is already running).\n const heartbeat = setInterval(() => {\n lock\n .renew(lockTtlMs)\n .then((ok) => {\n if (!ok) console.warn(`[scheduler] the lock was taken over (trigger ${id}, run ${runId}) — the CAS on the state write will decide the result.`);\n })\n .catch((err) => console.warn(`[scheduler] renew transient error (trigger ${id}), the next tick will retry:`, err));\n }, Math.max(1, Math.floor(lockTtlMs / 3)));\n\n // Fencing: every state write is a CAS against the state we read AT THE START of this fire\n // (`expected = state`). CAS — not a \"do I still hold the lock?\" hunch — is the AUTHORITY FOR THE\n // WRITE (and only for the write; execution is not fenced, see the KNOWN LIMIT above): the\n // lock is advisory and any ownership check is inherently a read at a point in time (it can go\n // stale between the check and the write), whereas putIfMatch decides ATOMICALLY at write time,\n // inside the journal. If somebody else advanced the trigger (took the fire over and wrote its\n // own result), our record no longer matches and this write is REJECTED — a late poller can no\n // longer roll fireCount/nextRunAt back or resurrect `attempts`. The write of the poller that\n // genuinely holds the lock always matches, so a correct poller is never blocked.\n // A journal WITHOUT putIfMatch falls back to an unconditional put (old behavior, documented risk\n // — the same fallback as run-lock.ts / claim()).\n const commit = async (next: TriggerState, what: string): Promise<boolean> => {\n if (!journal.putIfMatch) {\n await journal.put(STATE(id), next);\n return true;\n }\n if (await journal.putIfMatch(STATE(id), state, next)) return true;\n console.warn(`[scheduler] stale state write rejected (trigger ${id}, ${what}) — another poller advanced this trigger; this poller's result is DISCARDED.`);\n return false;\n };\n\n try {\n // 1.4: optional budget/quota hook — checked before runner.runWorkflow is CALLED.\n if (opts.budgetGuard) {\n try {\n await opts.budgetGuard({ triggerId: id, workflowName: def.name, input: def.input, now });\n } catch (e) {\n // attempts DOES NOT increase; the diagnostic record is only written if the CAS was won (a\n // stale poller must not leave a budget-skip note on someone else's fire either).\n if (await commit({ ...state, nextRunAt: now + retryMs }, 'budget-skip')) {\n await journal.put(BUDGET_SKIP(id), { error: String((e as any)?.message ?? e), at: now });\n out.skipped++;\n }\n continue;\n }\n }\n let result: { suspended?: boolean; output?: unknown };\n try {\n result = await runner.runWorkflow(def.name, def.input, { runId });\n } catch (e) {\n // DEFERRAL, not attempt: the run is already in flight elsewhere (see isRunBusyAtAcquisition).\n // `attempts` is untouched, `status` stays 'pending', and — this is the half that cost the live\n // investigation its diagnosis — NOTHING is written to `sched:fail:`. That record is where an\n // operator reads WHY a trigger died, and a run-busy message answers a question nobody asked\n // while burying the answer to the one they did (the live record said \"already running\"; the\n // real reason the workflow could not run was never written down anywhere).\n //\n // Not silent, though: a deferral that repeats forever is a trigger that never fires, so the\n // record carries a RUNNING TOTAL (never reset — a lifetime count is the number an operator can\n // compare against `fireCount`). There is no cap on purpose: \"wait for the other holder\" has no\n // honest deadline, and the other side is already bounded by its own lock TTL.\n if (isRunBusyAtAcquisition(e)) {\n if (await commit({ ...state, nextRunAt: now + retryMs }, 'run-busy-skip')) {\n const prev = await journal.get<{ deferrals?: number }>(BUSY_SKIP(id));\n await journal.put(BUSY_SKIP(id), {\n error: String((e as any)?.message ?? e),\n at: now,\n runId,\n deferrals: (prev?.deferrals ?? 0) + 1,\n });\n out.skipped++;\n }\n continue;\n }\n const attempts = state.attempts + 1;\n if (attempts >= def.maxAttempts) {\n if (await commit({ ...state, attempts, status: 'failed' }, 'failed')) {\n await journal.put(FAIL(id), { error: String((e as any)?.message ?? e), at: now });\n out.failed++;\n }\n } else if (await commit({ ...state, attempts, nextRunAt: now + backoff(attempts) }, 'retry')) {\n out.rescheduled++;\n }\n continue;\n }\n\n if (result.suspended) {\n // workflow suspended → the same runId should be resumed later (fireCount unchanged).\n if (await commit({ ...state, nextRunAt: now + retryMs }, 'suspended')) out.rescheduled++;\n } else if (def.kind === 'at') {\n if (await commit({ ...state, status: 'done' }, 'done')) out.fired++;\n } else {\n const next: TriggerState = {\n nextRunAt: computeNext(def, state.nextRunAt, now),\n attempts: 0,\n fireCount: state.fireCount + 1,\n status: 'pending',\n };\n if (await commit(next, 'reschedule')) out.fired++;\n }\n } finally {\n // MUST run BEFORE release(): release() keeps the SAME fencing token (it only pushes `expires`\n // into the past), so a heartbeat tick that survives this fire would find its own token still in\n // the record and RESURRECT the lock it just released (expires: 0 → now+ttl) — blocking every\n // later poll of the same runId (a suspended trigger's resume, most visibly). Pinned by test.\n clearInterval(heartbeat);\n // If the lock was genuinely taken over, release() is a no-op anyway (the fencing token no\n // longer matches — run-lock.ts mkLock.release), so we never free somebody else's lock.\n await lock.release();\n }\n }\n return out;\n}\n\n/** Introspection view of a single trigger (for the Studio Scheduler view). */\nexport interface TriggerInfo {\n id: string;\n name: string;\n kind: Kind;\n /** kind='at' → absolute time (epoch ms); kind='every' → period (ms); kind='cron' → 5-field cron expression. */\n value: number | string;\n input?: unknown;\n /** Next (planned) fire time, epoch ms — if in the past, the trigger is due/overdue. */\n nextRunAt: number;\n attempts: number;\n maxAttempts: number;\n fireCount: number;\n status: 'pending' | 'done' | 'failed';\n misfire: MisfirePolicy;\n /** Last error if status='failed' (if any, from `sched:fail:<id>`). */\n lastError?: string;\n lastErrorAt?: number;\n /**\n * How many times this trigger was DEFERRED because the run was already in flight elsewhere\n * (`sched:busy-skip:<id>`, lifetime total). Absent when it has never happened.\n *\n * On this list a deferred trigger is otherwise indistinguishable from a healthy one — 'pending',\n * with a nextRunAt in the near future, forever. That is the shape of a trigger that has not run in\n * a week and looks fine, so the count is here rather than only in the journal. Compare it against\n * `fireCount`: a number that keeps climbing while fireCount does not is a trigger that is being\n * refused, not one that is waiting.\n */\n deferrals?: number;\n lastDeferralAt?: number;\n}\n\n/**\n * READ-ONLY trigger listing from the journal, WITHOUT needing a scheduler INSTANCE or a runner (for\n * Studio introspection). Reads the SAME `sched:def:`/`sched:state:` keys (+ `sched:fail:` for 'failed')\n * as `pollScheduler`; does NOT change any state, does not take a lock, does not run a workflow. Returns\n * results sorted alphabetically by id (stable list order).\n */\nexport async function listTriggers(journal: Journal): Promise<TriggerInfo[]> {\n if (!journal.listKeys) throw new Error('@gnldev/scheduler: listTriggers requires journal.listKeys (trigger enumeration)');\n const defKeys = await journal.listKeys('sched:def:');\n const out: TriggerInfo[] = [];\n for (const dkey of defKeys) {\n const id = dkey.slice('sched:def:'.length);\n const def = await journal.get<TriggerDef>(DEF(id));\n const state = await journal.get<TriggerState>(STATE(id));\n if (!def || !state) continue; // inconsistent/partial record (theoretical) — skip\n const info: TriggerInfo = {\n id,\n name: def.name,\n kind: def.kind,\n value: def.value,\n input: def.input,\n nextRunAt: state.nextRunAt,\n attempts: state.attempts,\n maxAttempts: def.maxAttempts,\n fireCount: state.fireCount,\n status: state.status,\n misfire: def.misfire,\n };\n if (state.status === 'failed') {\n const fail = await journal.get<{ error: string; at: number }>(FAIL(id));\n if (fail) {\n info.lastError = fail.error;\n info.lastErrorAt = fail.at;\n }\n }\n // Read on EVERY status, not just 'failed': a deferred trigger is 'pending' by definition, so\n // gating this the way `lastError` is gated would hide it on exactly the rows that carry it.\n const busy = await journal.get<{ at: number; deferrals?: number }>(BUSY_SKIP(id));\n if (busy?.deferrals) {\n info.deferrals = busy.deferrals;\n info.lastDeferralAt = busy.at;\n }\n out.push(info);\n }\n return out.sort((a, b) => a.id.localeCompare(b.id));\n}\n\nexport interface Scheduler {\n poll(now?: number): Promise<PollResult>;\n start(): void;\n stop(): void;\n}\n\n/**\n * Scheduler that manages the poll loop (a self-rescheduling setTimeout chain) — the same pattern as\n * @gnldev/queue's createWorker. `backoff` (default OFF — timing is the scheduler's core contract, see the\n * trade-off below): IF ENABLED, when a poll fires NO triggers at all (`fired === 0`) the next poll\n * interval grows ×2 (cap: `maxPollMs ?? pollMs*32`) → prevents tens of thousands of empty queries per\n * second (poll storm) on an empty schedule table; the interval resets to `pollMs` once a trigger fires.\n * Trade-off: `backoff: true` cuts idle poll load by ~32x but can delay a trigger that becomes due after\n * a quiet period by up to `maxPollMs` — for timing-critical use (e.g. minute-level cron) the default\n * should stay OFF; only enable it for deployments with many idle-poller instances that can tolerate delay.\n */\nexport function createScheduler(\n journal: Journal,\n runner: WorkflowRunner,\n opts: { pollMs?: number; owner?: string; retryMs?: number; backoff?: boolean; maxPollMs?: number; budgetGuard?: BudgetGuard; lockTtlMs?: number } = {},\n): Scheduler {\n const pollMs = opts.pollMs ?? 1000;\n const backoffOn = opts.backoff ?? false;\n const maxPollMs = opts.maxPollMs ?? pollMs * 32;\n const poll = (now: number = Date.now()) =>\n pollScheduler(journal, runner, now, { owner: opts.owner, retryMs: opts.retryMs, budgetGuard: opts.budgetGuard, lockTtlMs: opts.lockTtlMs });\n\n // Phase 8.1: the tick/backoff/\"polling\" flag loop now lives in @gnldev/durable's shared createPollLoop\n // (it used to be triplicated across queue/events/scheduler) — behavior is identical: pollScheduler only\n // catches runWorkflow errors internally; the rest (journal I/O etc.) are logged and swallowed by\n // createPollLoop (the chain doesn't die). While `fired === 0` and backoffOn, the interval grows ×2\n // (cap maxPollMs); it resets to pollMs once something fires. Default backoff is OFF (see the comment\n // above — timing-critical).\n const loop = createPollLoop(async () => (await poll()).fired > 0, { pollMs, backoff: backoffOn, maxPollMs });\n\n return {\n poll,\n start: loop.start,\n stop: loop.stop,\n };\n}\n"]}
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
import type { JournalLike, WorkflowRunStatus } from '@gnldev/workflow';
|
|
2
|
+
export interface WorkflowWakerOptions {
|
|
3
|
+
/** Structurally compatible with @gnldev/workflow's JournalLike (a superset — @gnldev/durable's Journal — also works). */
|
|
4
|
+
journal: JournalLike;
|
|
5
|
+
/**
|
|
6
|
+
* Host-supplied resume: the waker does not know how to rebuild a workflow instance, so it hands
|
|
7
|
+
* the runId + registry record back to the host, which typically calls the matching workflow's
|
|
8
|
+
* `runResumable(input, { runId, journal }, opts)` again (completed steps replay from the journal;
|
|
9
|
+
* the suspended step re-evaluates and either continues or suspends again).
|
|
10
|
+
*/
|
|
11
|
+
resume: (runId: string, status: WorkflowRunStatus) => Promise<unknown>;
|
|
12
|
+
/** Poll interval (ms). Default 5000. */
|
|
13
|
+
intervalMs?: number;
|
|
14
|
+
/**
|
|
15
|
+
* Random delay (0..jitterMs) added BEFORE each tick's scan — spreads multiple waker instances'
|
|
16
|
+
* polls apart so they don't all hit the journal in lockstep. Does NOT push a time-based sleep
|
|
17
|
+
* PAST its `untilMs`: the due-check reads `Date.now()` AFTER the jitter delay, so a run only
|
|
18
|
+
* ever wakes at-or-after its scheduled time (may be noticed up to `jitterMs` later, never earlier
|
|
19
|
+
* than what a poll interval would already allow). Default 0 (off).
|
|
20
|
+
*/
|
|
21
|
+
jitterMs?: number;
|
|
22
|
+
/**
|
|
23
|
+
* Evented (`waitFor`) and HITL (`waitForResume`) suspends have NO time signal — the waker cannot
|
|
24
|
+
* tell whether the event/payload has arrived. Default (false): SKIP them entirely (only time-based
|
|
25
|
+
* sleeps are woken). Opt-in `true`: resume them too, on every tick they remain suspended. Honest
|
|
26
|
+
* cost: since a still-not-ready evented run gets a FRESH `wfrun:` `updatedAt` (and thus a fresh
|
|
27
|
+
* wake ticket) every time `resume()` re-suspends it, this means one `resume()` round-trip (and
|
|
28
|
+
* whatever `check()`/host work it triggers) PER TICK PER still-suspended evented run, for as long
|
|
29
|
+
* as it stays unresolved — a busy-poll, not a push. Fine for a handful of runs; expensive at scale
|
|
30
|
+
* (prefer delivering the event directly, e.g. `runResumable({ resume })`, when you can).
|
|
31
|
+
*/
|
|
32
|
+
wakeEvented?: boolean;
|
|
33
|
+
/**
|
|
34
|
+
* Per-run error hook. Default (omitted): swallowed — `console.warn` ONCE per runId per
|
|
35
|
+
* failure-streak (a success, or the run moving on, resets the streak) so a stuck run doesn't spam
|
|
36
|
+
* logs every tick. The tick loop never dies from a `resume()` throw either way (matches
|
|
37
|
+
* `createPollLoop`'s "the chain doesn't die" contract).
|
|
38
|
+
*/
|
|
39
|
+
onError?: (runId: string, error: unknown) => void;
|
|
40
|
+
}
|
|
41
|
+
export interface WorkflowWakerTickResult {
|
|
42
|
+
/** Runs whose `resume()` was called AND won the wake ticket (or the CAS-less fallback let through). */
|
|
43
|
+
resumed: number;
|
|
44
|
+
/** Runs not due yet, evented/HITL runs skipped (wakeEvented off), or runs that lost the wake-ticket race. */
|
|
45
|
+
skipped: number;
|
|
46
|
+
/** `resume()` calls that threw (swallowed — see `onError`). Included in `resumed` (the call WAS made). */
|
|
47
|
+
errored: number;
|
|
48
|
+
}
|
|
49
|
+
export interface WorkflowWaker {
|
|
50
|
+
/** One scan+wake pass. Exposed directly for tests/manual driving (mirrors pollScheduler/poll). */
|
|
51
|
+
tick(now?: number): Promise<WorkflowWakerTickResult>;
|
|
52
|
+
start(): void;
|
|
53
|
+
stop(): void;
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* Creates a durable sleep/event waker for @gnldev/workflow suspended runs. `start()`/`stop()` mirror
|
|
57
|
+
* `createScheduler`'s lifecycle (a self-rescheduling `setTimeout` chain via `createPollLoop` — no
|
|
58
|
+
* dangling timer after `stop()`).
|
|
59
|
+
*/
|
|
60
|
+
export declare function createWorkflowWaker(opts: WorkflowWakerOptions): WorkflowWaker;
|