@framers/agentos-ext-triggers 0.0.0-stage → 0.1.0

package/dist/scheduler.js

@@ -0,0 +1,743 @@
+import { computeNextRunAtMs, requireSchedule } from './cron.js';
+import { housekeep, requireKeep } from './housekeeping.js';
+import { NAME, WATCH_ROW } from './names.js';
+import { createReporter } from './report.js';
+import { missedViewOf, num, runViewOf, scheduleOf, viewOf } from './rows.js';
+import { randomHex } from './store/ids.js';
+import { dialectOf } from './store/migrations.js';
+import { DEFAULT_PREFIX, requireDialectPrefix, requirePrefix, tableName } from './store/prefix.js';
+import { scheduleWords } from './words.js';
+/** The tick when the host gives none, and the smallest `every`. */
+export const DEFAULT_TICK_MS = 10_000;
+/** The gap after which a job counts as stalled when its definition gives none. */
+export const DEFAULT_STALL_AFTER_MS = 900_000;
+/** Tries before a job pauses itself when its definition gives none. */
+export const DEFAULT_RETRIES = 5;
+/** A run's bound when its definition gives none. */
+export const DEFAULT_TIMEOUT_MS = 600_000;
+/** Hand runs an hour on one instance when the host gives none. */
+export const DEFAULT_HAND_RUNS_PER_HOUR = 20;
+/** How long `stop()` waits for the runs in flight when the host gives no wait. */
+export const DEFAULT_STOP_WAIT_MS = 5_000;
+/** The longest pause note. */
+export const NOTE_MAX = 200;
+export { NAME, WATCH_ROW } from './names.js';
+/**
+ * The most missed due times one restore, one resume or one settlement records for a job. For `skip` and `catch_up_once`
+ * the NEWEST MISSED_MAX are recorded and the older ones only counted (one `job_missed` line with `count`); for `required`
+ * the OLDEST MISSED_MAX are recorded and the rest stay owed: the settlement of the catch-up run that served them records
+ * the next batch, so no required due time is stepped past unrecorded.
+ */
+export const MISSED_MAX = 1_000;
+/** One hour, the hand runs' bound's window and the housekeeping's. */
+const HOUR = 3_600_000;
+/** Jobs a `list` page holds. */
+export const LIST_PAGE = 200;
+/** The wait before try `attempt + 1`: 30 s doubling to 10 minutes. */
+export function defaultBackoffMs(attempt) {
+    return Math.min(30_000 * 2 ** Math.max(0, attempt - 1), 600_000);
+}
+/** Thrown inside the restore's transaction when its conditional claim changed no row, so the missed rows it wrote roll back with it. */
+const RESTORE_LOST = Symbol('restore_lost');
+let uniqueCheck = null;
+/** Whether `error` is a unique violation, through the adapter's own `isUniqueViolation` loaded at the first call (the adapter is an optional peer, so the entry imports without it). */
+function isUnique(error) {
+    uniqueCheck ??= import('@framers/sql-storage-adapter').then((adapter) => adapter.isUniqueViolation);
+    return uniqueCheck.then((check) => check(error));
+}
+/**
+ * Builds a scheduler over `options.db` under `options.prefix`. A process that runs jobs registers them and calls
+ * `start()`; a process that only reads and commands builds one and never starts it. `internals.arm: false` arms no
+ * interval: the caller drives `tick()` (the tests, and `createSchedulerForTests`).
+ */
+export function createScheduler(options, internals = {}) {
+    const db = options.db;
+    const prefix = requireDialectPrefix(requirePrefix(options.prefix ?? DEFAULT_PREFIX), dialectOf(db));
+    const t = (name) => tableName(prefix, name);
+    const now = options.now ?? Date.now;
+    const tickMs = options.tickMs ?? DEFAULT_TICK_MS;
+    const logger = options.logger;
+    // A `keep` field that is not a whole number above 0 throws here, never at the first hourly housekeeping.
+    const keep = requireKeep(options.keep);
+    const reporter = createReporter(db, t, now, logger);
+    const holder = `${options.process}:${randomHex(4)}`;
+    const definitions = new Map();
+    const inFlight = new Map();
+    const runHandlers = new Set();
+    const failedHandlers = new Set();
+    const handRunsAt = [];
+    let started = false;
+    /** The start in progress: a second `start()` while it runs waits for it rather than writing, restoring and arming again. */
+    let starting = null;
+    let stopping = false;
+    let ticking = false;
+    /** The tick in progress, which `stop()` waits for before it reads the runs in flight. */
+    let currentTick = Promise.resolve();
+    let timer = null;
+    const scheduleJson = (schedule) => JSON.stringify(schedule);
+    /**
+     * The due time a new or changed definition starts from: the schedule's next one after `at`, or a one-shot's own time when
+     * it has already passed, so the restore applies the job's rule to it rather than leaving a row no claim can take.
+     */
+    function firstDue(schedule, at) {
+        return computeNextRunAtMs(schedule, at) ?? (schedule.kind === 'at' && Number.isFinite(schedule.at) ? schedule.at : null);
+    }
+    /** The next due time strictly after `fromMs` that is also after `afterMs`: stepped from the due time, never from the run's end, so a slow run does not drift the schedule. */
+    function nextAfter(schedule, fromMs, afterMs) {
+        let next = computeNextRunAtMs(schedule, fromMs);
+        while (next !== undefined && next <= afterMs)
+            next = computeNextRunAtMs(schedule, next);
+        return next;
+    }
+    /** The due times from `fromMs` (exclusive) up to `untilMs` (inclusive), at most MISSED_MAX, oldest first. */
+    function dueTimes(schedule, fromMs, untilMs) {
+        const times = [];
+        let next = computeNextRunAtMs(schedule, fromMs);
+        while (next !== undefined && next <= untilMs && times.length < MISSED_MAX) {
+            times.push(next);
+            next = computeNextRunAtMs(schedule, next);
+        }
+        return times;
+    }
+    /**
+     * The missed window a rule records: `first` (a due time already owed, or undefined) and the due times after `fromMs` up to
+     * `untilMs`. `required` keeps the oldest MISSED_MAX (`more` says due times stay owed past them); the other rules keep the
+     * newest MISSED_MAX and count the older ones in `dropped`, so a catch-up serves the latest due time missed.
+     */
+    function missedWindow(schedule, rule, first, fromMs, untilMs) {
+        const head = first === undefined ? [] : [first];
+        if (rule === 'required') {
+            const times = [...head, ...dueTimes(schedule, fromMs, untilMs)].slice(0, MISSED_MAX);
+            const after = computeNextRunAtMs(schedule, times[times.length - 1] ?? fromMs);
+            return { times, dropped: 0, more: times.length > 0 && after !== undefined && after <= untilMs };
+        }
+        let kept = [...head];
+        let dropped = 0;
+        let next = computeNextRunAtMs(schedule, fromMs);
+        while (next !== undefined && next <= untilMs) {
+            kept.push(next);
+            // Trimmed in halves, so a long outage costs one pass over its due times, never a shift per due time.
+            if (kept.length >= 2 * MISSED_MAX) {
+                dropped += kept.length - MISSED_MAX;
+                kept = kept.slice(-MISSED_MAX);
+            }
+            next = computeNextRunAtMs(schedule, next);
+        }
+        if (kept.length > MISSED_MAX) {
+            dropped += kept.length - MISSED_MAX;
+            kept = kept.slice(-MISSED_MAX);
+        }
+        return { times: kept, dropped, more: false };
+    }
+    function register(definition) {
+        // A start in progress has read, or is reading, the definitions: a job added now might get no row or no restore.
+        if (started || starting !== null)
+            throw new Error('register() runs before start()');
+        if (!NAME.test(definition.name))
+            throw new RangeError('a job name is a lower-case letter, then at most 63 lower-case letters, digits or underscores');
+        if (definitions.has(definition.name))
+            throw new Error(`duplicate job name: ${definition.name}`);
+        requireSchedule(definition.schedule, tickMs);
+        if (definition.legalDuty === true && definition.rule !== 'required')
+            throw new Error(`a legal-duty job's rule is required: ${definition.name}`);
+        definitions.set(definition.name, definition);
+    }
+    /**
+     * The definition's columns written in. The stored schedule is read BEFORE the write and compared: a changed schedule, or a
+     * finished one-shot (`enabled = 0`) whose time changed, is rescheduled from now and enabled again in the same transaction, so
+     * the restore below never reads the old schedule's `next_due_at` as the new one's (a weekly job changed to every five
+     * minutes would otherwise keep its weekly due time).
+     */
+    async function upsertDefinition(def, at) {
+        const schedule = scheduleJson(def.schedule);
+        await db.transaction(async (trx) => {
+            const stored = await trx.get(`SELECT schedule, enabled FROM ${t('jobs')} WHERE name = ?`, [def.name]);
+            const changed = stored !== undefined && stored !== null && (stored.schedule === null || stored.schedule !== schedule);
+            await trx.run(`INSERT INTO ${t('jobs')} (name, title, process, kind, schedule, schedule_words, rule, stall_after_ms, safe_to_repeat, retries, timeout_ms, legal_duty, enabled, next_due_at, registered_at, updated_at)
+         VALUES (?, ?, ?, 'scheduled', ?, ?, ?, ?, ?, ?, ?, ?, 1, ?, ?, ?)
+         ON CONFLICT (name) DO UPDATE SET title = excluded.title, process = excluded.process, kind = 'scheduled', schedule = excluded.schedule,
+           schedule_words = excluded.schedule_words, rule = excluded.rule, stall_after_ms = excluded.stall_after_ms, safe_to_repeat = excluded.safe_to_repeat,
+           retries = excluded.retries, timeout_ms = excluded.timeout_ms, legal_duty = excluded.legal_duty, updated_at = excluded.updated_at`, [def.name, def.title, def.process, schedule, scheduleWords(def.schedule), def.rule, def.stallAfterMs ?? DEFAULT_STALL_AFTER_MS, def.safeToRepeat ? 1 : 0,
+                def.retries ?? DEFAULT_RETRIES, def.timeoutMs ?? DEFAULT_TIMEOUT_MS, def.legalDuty === true ? 1 : 0, firstDue(def.schedule, at), at, at]);
+            if (changed) {
+                // A run in flight under the old schedule, in this process or another, settles onto the stored schedule (the settlement reads it), so it never writes the old schedule's next due time back.
+                await trx.run(`UPDATE ${t('jobs')} SET next_due_at = ?, enabled = 1, updated_at = ? WHERE name = ?`, [firstDue(def.schedule, at), at, def.name]);
+            }
+        });
+    }
+    /**
+     * Writes the missed rows for `times` by `rule` and answers whether a catch-up run is owed now. The ledger's unique index on
+     * `(job, due_at)` refuses a due time recorded twice: the insert throws, and the caller's transaction rolls back. Under
+     * `catch_up_once` an earlier catch-up still owed (a row `caught_up` with no run) is moved to `skipped`, so one catch-up is
+     * owed whatever number of restores and resumes recorded windows before it ran. `dropped` older due times are counted on
+     * the first line.
+     */
+    async function applyRule(writer, name, rule, cause, times, at, dropped = 0) {
+        if (rule === 'catch_up_once' && times.length > 0) {
+            await writer.run(`UPDATE ${t('missed')} SET action = 'skipped', acted_at = ? WHERE job = ? AND action = 'caught_up' AND run_id IS NULL AND due_at < ?`, [at, name, times[0]]);
+        }
+        for (let i = 0; i < times.length; i += 1) {
+            const caught = rule === 'required' || (rule === 'catch_up_once' && i === times.length - 1);
+            await writer.run(`INSERT INTO ${t('missed')} (id, job, due_at, cause, action, acted_at) VALUES (?, ?, ?, ?, ?, ?)`, [randomHex(16), name, times[i], cause, caught ? 'caught_up' : 'skipped', caught ? null : at]);
+            logger.info('job_missed', { job: name, rule, cause, action: caught ? 'caught_up' : 'skipped', ...(i === 0 && dropped > 0 ? { count: dropped } : {}) });
+        }
+        return { catchUp: times.length > 0 && rule !== 'skip' };
+    }
+    /**
+     * The restore at start(): what was missed while no process ran a job is recorded by its rule, atomically per job (two
+     * starting processes read the same overdue `next_due_at`; one restores it and the other changes nothing). The missed rows
+     * and the move of `next_due_at` land in one transaction whose last statement is a conditional claim on the `next_due_at`
+     * this process read, skipping a row another process holds under a live lease; a claim that changes no row throws
+     * `RESTORE_LOST`, so the transaction rolls back and the other process's rows stand. A catch-up sets `next_due_at` to the
+     * LAST missed due time, never to "now": the claim then runs it at once (`next_due_at <= now`), the run's `dueAt` is a due
+     * time the schedule had, and `covers` is read from the missed rows. A `next_due_at` within the last two ticks is left to
+     * the claim: it shows nothing about whether another process is running.
+     */
+    async function restore(at) {
+        for (const def of definitions.values()) {
+            const row = await db.get(`SELECT * FROM ${t('jobs')} WHERE name = ?`, [def.name]);
+            if (!row || row.enabled !== 1 || row.paused_by !== null)
+                continue;
+            const nextDue = num(row.next_due_at);
+            if (nextDue === null) {
+                await db.run(`UPDATE ${t('jobs')} SET next_due_at = ?, updated_at = ? WHERE name = ? AND next_due_at IS NULL`, [computeNextRunAtMs(def.schedule, at) ?? null, at, def.name]);
+                continue;
+            }
+            // A due time of the last two ticks is the claim's, never missed: a live process claims it within a tick, and if none
+            // runs, this process's first tick does. Only a due time older than that shows that no process ran the job.
+            if (nextDue > at - 2 * tickMs)
+                continue;
+            if (row.lease_until !== null && Number(row.lease_until) > at)
+                continue; // a live process holds the job: nothing was missed
+            // A pending retry (`retry_at` set) serves `next_due_at` itself when its backoff ends: that due time is owed, never missed,
+            // and the window starts after it; the missed-run rule never reads `retry_at`. So is a due time whose run's process died
+            // holding the lease (a token under an expired lease): the claim reads that run lost first, then runs the due time again or,
+            // for a job not safe to repeat, pauses it by system.
+            const retrying = row.retry_at !== null || row.lease_token !== null;
+            const { times: window, dropped } = missedWindow(def.schedule, def.rule, retrying ? undefined : nextDue, nextDue, at);
+            // Due times an earlier restore or resume already recorded are not recorded again: a second restore before the queued
+            // catch-up ran finds its window recorded and changes nothing.
+            const recorded = window.length === 0 ? new Set() : new Set((await db.all(`SELECT due_at FROM ${t('missed')} WHERE job = ? AND due_at >= ? AND due_at <= ?`, [def.name, window[0], window[window.length - 1]])).map((m) => Number(m.due_at)));
+            const times = window.filter((due) => !recorded.has(due));
+            if (times.length === 0)
+                continue;
+            const last = times[times.length - 1];
+            try {
+                await db.transaction(async (trx) => {
+                    const { catchUp } = await applyRule(trx, def.name, def.rule, 'worker_down', times, at, dropped);
+                    // A pending retry keeps its due time; the catch-up it leaves owed runs after the retry settles (settle reads the ledger).
+                    // A required window with more owed runs its batch now; the settlement records the next batch.
+                    const next = retrying ? nextDue : catchUp ? last : (nextAfter(def.schedule, last, at) ?? null);
+                    const changed = await trx.run(`UPDATE ${t('jobs')} SET next_due_at = ?, updated_at = ? WHERE name = ? AND next_due_at = ? AND (lease_until IS NULL OR lease_until < ?)`, [next, at, def.name, nextDue, at]);
+                    if (Number(changed.changes) !== 1)
+                        throw RESTORE_LOST;
+                });
+            }
+            catch (error) {
+                // Another process restored first: its conditional claim moved the row, or its rows hold the ledger's unique index.
+                if (error !== RESTORE_LOST && !(await isUnique(error)))
+                    throw error;
+            }
+        }
+    }
+    function start() {
+        if (started)
+            return Promise.resolve();
+        // Cleared when it settles, so a start that threw can be called again.
+        starting ??= begin().finally(() => { starting = null; });
+        return starting;
+    }
+    async function begin() {
+        const at = now();
+        for (const def of definitions.values())
+            await upsertDefinition(def, at);
+        await restore(at);
+        started = true;
+        if (internals.arm !== false) {
+            timer = setInterval(() => void tick(), tickMs);
+            timer.unref?.();
+            void tick();
+        }
+    }
+    /**
+     * Each tick, for every run this process has in flight: the timeout read from the clock (the first tick at or past
+     * `startedAt + timeoutMs` aborts the signal with `timeout`; no real timer, so a fake clock sees it), then the renewal.
+     * A renewal that changes no row, or that the database refuses or cannot reach, is a lost lease: the run's signal is
+     * aborted with `lease_lost` at once, and its settlement stays fenced by the token.
+     */
+    async function renewLeases(at) {
+        for (const [name, flight] of inFlight) {
+            if (!flight.controller.signal.aborted && at - flight.startedAt >= flight.timeoutMs)
+                flight.controller.abort(new Error('timeout'));
+            let changes = 0;
+            try {
+                changes = Number((await db.run(`UPDATE ${t('jobs')} SET lease_until = ?, updated_at = ? WHERE name = ? AND lease_token = ?`, [at + 2 * tickMs, at, name, flight.token])).changes);
+            }
+            catch {
+                changes = 0;
+            }
+            if (changes !== 1 && !flight.controller.signal.aborted)
+                flight.controller.abort(new Error('lease_lost'));
+        }
+    }
+    /** Marks a run another process left open as lost; a job not safe to repeat is paused by system, its alert owed, and not run. Answers whether the claimant may run. */
+    async function settleLost(row, lost, token, at) {
+        await db.run(`UPDATE ${t('runs')} SET outcome = 'failed', ended_at = ?, ms = ?, kind = 'lost' WHERE id = ? AND outcome = 'running'`, [at, at - Number(lost.started_at), lost.id]);
+        logger.warn('job_run', { job: row.name, outcome: 'failed', ms: at - Number(lost.started_at), tries: lost.tries });
+        if (row.safe_to_repeat === 1)
+            return true;
+        await db.run(`UPDATE ${t('jobs')} SET paused_by = 'system', paused_at = ?, failed_at = ?, failed_kind = 'lost', alerted_at = ?, tries = tries + 1, retry_at = NULL, lease_holder = NULL, lease_token = NULL, lease_until = NULL, hand_run_by = NULL, hand_run_at = NULL, updated_at = ? WHERE name = ? AND lease_token = ?`, [at, at, at, at, row.name, token]);
+        logger.warn('job_failed', { job: row.name, kind: 'lost', tries: row.tries + 1 });
+        const view = await get(row.name);
+        const run = await runById(lost.id);
+        if (view && run)
+            for (const handler of failedHandlers) {
+                try {
+                    handler(view, run);
+                }
+                catch { /* a handler's throw is its own */ }
+            }
+        return false;
+    }
+    /** The claim: a conditional update that writes this process's lease and a fresh token on a job that is due (its retry ready) or asked for by hand, and free. */
+    async function tryClaim(def, at) {
+        const token = randomHex(16);
+        const changed = await db.run(`UPDATE ${t('jobs')} SET lease_holder = ?, lease_token = ?, lease_until = ?, updated_at = ?
+       WHERE name = ? AND kind = 'scheduled' AND (lease_until IS NULL OR lease_until < ?)
+         AND ((enabled = 1 AND paused_by IS NULL AND next_due_at IS NOT NULL AND next_due_at <= ? AND (retry_at IS NULL OR retry_at <= ?)) OR hand_run_at IS NOT NULL)`, [holder, token, at + 2 * tickMs, at, def.name, at, at, at]);
+        if (Number(changed.changes) !== 1)
+            return null;
+        const row = (await db.get(`SELECT * FROM ${t('jobs')} WHERE name = ?`, [def.name]));
+        const lost = await db.get(`SELECT * FROM ${t('runs')} WHERE job = ? AND outcome = 'running' AND lease_token <> ? ORDER BY started_at DESC LIMIT 1`, [def.name, token]);
+        if (lost && !(await settleLost(row, lost, token, at)))
+            return null;
+        return { row, token };
+    }
+    /** When the housekeeping last ran on this instance: it runs on the tick once an hour at most. */
+    let lastHousekeepingAt = null;
+    /** The retention of runs and missed runs (`housekeep`), once an hour at most, with the host's `keep` over the defaults. */
+    async function housekeeping(at) {
+        if (lastHousekeepingAt !== null && at - lastHousekeepingAt < HOUR)
+            return;
+        lastHousekeepingAt = at;
+        await housekeep(db, t, at, keep);
+    }
+    async function tick() {
+        if (ticking || stopping)
+            return;
+        ticking = true;
+        const pass = claimPass();
+        currentTick = pass;
+        await pass;
+    }
+    /** Frees a lease this process took and will not run on (a claim that landed after `stop()` began), fenced by its token. */
+    async function release(name, token) {
+        await db.run(`UPDATE ${t('jobs')} SET lease_holder = NULL, lease_token = NULL, lease_until = NULL, updated_at = ? WHERE name = ? AND lease_token = ?`, [now(), name, token]);
+    }
+    async function claimPass() {
+        try {
+            const at = now();
+            await renewLeases(at);
+            for (const def of definitions.values()) {
+                if (stopping)
+                    break;
+                if (inFlight.has(def.name))
+                    continue;
+                const claim = await tryClaim(def, at);
+                if (claim === null)
+                    continue;
+                // `stop()` began while this claim was in the database: it starts no run, and the job is free for the next process.
+                if (stopping) {
+                    await release(def.name, claim.token);
+                    break;
+                }
+                // Waits until the run's row is written and its body called, never for the body to settle.
+                await startRun(def, claim, at);
+            }
+            await housekeeping(at);
+        }
+        catch (error) {
+            logger.error('tick_failed', { kind: options.describe?.(error)?.kind ?? 'error' });
+        }
+        finally {
+            ticking = false;
+        }
+    }
+    /**
+     * Starts a claimed run: writes its row, calls the job's body and settles it when the body settles. Answers a promise that
+     * resolves once the body has been called (or the run failed before it), so a tick never returns ahead of the runs it began.
+     */
+    function startRun(def, claim, at) {
+        const { row, token } = claim;
+        let byHand = row.hand_run_at !== null ? { accountId: row.hand_run_by ?? 'system' } : null;
+        const runId = randomHex(16);
+        const attempt = row.tries + 1;
+        const controller = new AbortController();
+        const timeoutMs = def.timeoutMs ?? DEFAULT_TIMEOUT_MS;
+        let begun = () => { };
+        const bodyCalled = new Promise((resolve) => { begun = resolve; });
+        const promise = (async () => {
+            // The due time this run serves: a hand run's request time; else the job's `next_due_at`, which a retry leaves as the due
+            // time it serves (its readiness is `retry_at`), so a retry serves its original due time with no lookup.
+            const dueAt = byHand ? Number(row.hand_run_at) : Number(row.next_due_at);
+            // What the run covers: the caught-up missed rows not yet served, up to this due time (the restore wrote the last of them as
+            // `next_due_at`, so it is among them), else this due time alone. Never a time the schedule did not have.
+            const pending = byHand ? [] : (await db.all(`SELECT due_at FROM ${t('missed')} WHERE job = ? AND action = 'caught_up' AND run_id IS NULL AND due_at <= ? ORDER BY due_at`, [def.name, dueAt])).map((m) => Number(m.due_at));
+            const covers = pending.length > 0 ? pending : [dueAt];
+            // The run row is written under the job row's lock, with a hand run's account read again inside that lock: an account's
+            // erasure (`forgetActor`, which nulls the job row first) either committed before it, so the run reads the account gone,
+            // or waits for it and then nulls the run row it wrote. The account read at the claim is never written after an erasure.
+            // A lock that finds the token gone means another process took the lease before this run began: that process runs the
+            // due time, so this one writes no run row and calls no body.
+            const written = await db.transaction(async (trx) => {
+                const held = Number((await trx.run(`UPDATE ${t('jobs')} SET updated_at = updated_at WHERE name = ? AND lease_token = ?`, [def.name, token])).changes);
+                if (held !== 1)
+                    return null;
+                const asked = byHand === null ? null : { accountId: (await trx.get(`SELECT hand_run_by FROM ${t('jobs')} WHERE name = ?`, [def.name]))?.hand_run_by ?? 'system' };
+                await trx.run(`INSERT INTO ${t('runs')} (id, job, started_at, outcome, tries, by_hand, covers_from, covers_to, lease_token) VALUES (?, ?, ?, 'running', ?, ?, ?, ?, ?)`, [runId, def.name, at, attempt, asked?.accountId ?? null, covers[0], covers[covers.length - 1], token]);
+                return { asked };
+            });
+            if (written === null) {
+                logger.warn('claim_lost', { job: def.name });
+                return;
+            }
+            byHand = written.asked;
+            let summary;
+            let error;
+            // A throw or a rejection fails the try whatever its value, `undefined` included.
+            let threw = false;
+            try {
+                const view = (await get(def.name));
+                const context = { now: at, job: view, attempt, byHand, covers, runId, dueAt, signal: controller.signal, logger };
+                const body = def.run(context);
+                begun();
+                summary = (await body);
+            }
+            catch (caught) {
+                threw = true;
+                error = caught;
+            }
+            begun();
+            const reason = controller.signal.aborted ? controller.signal.reason?.message : undefined;
+            await settle(def, row, token, runId, at, dueAt, byHand, attempt, summary, threw ? { error } : null, reason === 'timeout');
+        })().catch((caught) => {
+            logger.error('settle_failed', { job: def.name, kind: options.describe?.(caught)?.kind ?? 'error' });
+        }).finally(() => {
+            begun();
+            inFlight.delete(def.name);
+        });
+        // Set before the first await settles: `renewLeases` on the next tick reads it, and drives the timeout from the clock.
+        inFlight.set(def.name, { token, runId, controller, promise, startedAt: at, timeoutMs });
+        return bodyCalled;
+    }
+    /**
+     * The settlement, fenced by the token: every job write is conditional on `lease_token`, so a run whose lease another
+     * process took changes no job row, and its run is left open for that process's claim, which reads it `lost`. A success steps to the next due time (or a catch-up the
+     * ledger still owes, or a required batch now overdue); a failure below the bound sets `retry_at`; at the bound the job
+     * is paused by the system and `onFailed` fires.
+     */
+    async function settle(def, row, token, runId, startedAt, dueAt, byHand, attempt, summary, thrown, timedOut) {
+        const ended = now();
+        const ms = ended - startedAt;
+        const outcome = thrown !== null || timedOut || summary?.outcome === 'failed' ? 'failed' : summary?.outcome === 'skipped' ? 'skipped' : 'ok';
+        const kind = timedOut ? 'timeout' : thrown !== null ? (options.describe?.(thrown.error)?.kind ?? 'error') : summary?.outcome === 'failed' ? 'failed' : null;
+        const did = summary?.did ?? null;
+        const retries = row.retries;
+        const result = await db.transaction(async (trx) => {
+            let changed;
+            let settlement = 'settled';
+            let backlog = [];
+            let dropped = 0;
+            let rule = def.rule;
+            let paused = false;
+            if (byHand) {
+                // A hand run leaves the schedule and the tries as they were; its outcome lands on the run and the last-run columns.
+                changed = Number((await trx.run(`UPDATE ${t('jobs')} SET last_run_at = ?, last_outcome = ?, last_ms = ?, last_did = ?, lease_holder = NULL, lease_token = NULL, lease_until = NULL, hand_run_by = NULL, hand_run_at = NULL, updated_at = ? WHERE name = ? AND lease_token = ?`, [ended, outcome, ms, did, ended, def.name, token])).changes);
+            }
+            else if (outcome === 'failed' && attempt < retries) {
+                const backoff = (def.backoffMs ?? defaultBackoffMs)(attempt);
+                // `next_due_at` stays the due time the retry serves; `retry_at` holds the readiness, `failed_kind` the try's kind (the view's `retrying`).
+                changed = Number((await trx.run(`UPDATE ${t('jobs')} SET tries = ?, retry_at = ?, failed_kind = ?, last_run_at = ?, last_outcome = 'failed', last_ms = ?, last_did = ?, lease_holder = NULL, lease_token = NULL, lease_until = NULL, updated_at = ? WHERE name = ? AND lease_token = ?`, [attempt, ended + backoff, kind, ended, ms, did, ended, def.name, token])).changes);
+            }
+            else if (outcome === 'failed') {
+                settlement = 'failed_out';
+                changed = Number((await trx.run(`UPDATE ${t('jobs')} SET tries = ?, retry_at = NULL, paused_by = 'system', paused_at = ?, failed_at = ?, failed_kind = ?, alerted_at = ?, last_run_at = ?, last_outcome = 'failed', last_ms = ?, last_did = ?, lease_holder = NULL, lease_token = NULL, lease_until = NULL, updated_at = ? WHERE name = ? AND lease_token = ?`, [attempt, ended, ended, kind, ended, ended, ms, did, ended, def.name, token])).changes);
+            }
+            else {
+                // The next due time never steps past a due time owed: a catch-up the ledger still holds after this one (a restore during a
+                // pending retry left it) runs next; a `required` job whose next due time has passed records the overdue due times, a batch
+                // of MISSED_MAX at most, and runs them next; else the next due time after this one and after now. The schedule and the rule
+                // are the stored ones: a process that registered a changed definition while this run was in flight wrote them, and this
+                // process's own definition may be the older one.
+                const stored = await trx.get(`SELECT schedule, rule, next_due_at, paused_by FROM ${t('jobs')} WHERE name = ?`, [def.name]);
+                const schedule = (stored ? scheduleOf(stored) : null) ?? def.schedule;
+                rule = stored?.rule ?? def.rule;
+                // A schedule changed while this run was in flight: the registration that changed it set the next due time from its own
+                // start, and that due time stands; stepping the new schedule from this run's due time would move it.
+                const changedUnder = stored !== undefined && stored !== null && stored.schedule !== row.schedule;
+                // Paused while this run was in flight: the due times that passed meanwhile are recorded by the rule, cause `paused`, as
+                // a resume would have recorded them, so the catch-up a rule owes never depends on whether the resume or this settlement came first.
+                paused = stored?.paused_by !== null && stored?.paused_by !== undefined;
+                const owed = await trx.get(`SELECT MAX(due_at) AS d FROM ${t('missed')} WHERE job = ? AND action = 'caught_up' AND run_id IS NULL AND due_at > ?`, [def.name, dueAt]);
+                let next = owed?.d !== null && owed?.d !== undefined ? Number(owed.d) : undefined;
+                if (next === undefined && changedUnder)
+                    next = num(stored.next_due_at) ?? undefined;
+                if (next === undefined && (rule === 'required' || paused)) {
+                    const following = computeNextRunAtMs(schedule, dueAt);
+                    if (following !== undefined && following <= ended) {
+                        const window = missedWindow(schedule, rule, following, following, ended);
+                        backlog = window.times;
+                        dropped = window.dropped;
+                        if (rule !== 'skip')
+                            next = backlog[backlog.length - 1];
+                    }
+                }
+                next ??= nextAfter(schedule, dueAt, ended);
+                changed = Number((await trx.run(`UPDATE ${t('jobs')} SET tries = 0, retry_at = NULL, failed_at = NULL, failed_kind = NULL, next_due_at = ?, enabled = ?, last_run_at = ?, last_outcome = ?, last_ms = ?, last_did = ?, lease_holder = NULL, lease_token = NULL, lease_until = NULL, updated_at = ? WHERE name = ? AND lease_token = ?`, [next ?? null, next === undefined ? 0 : 1, ended, outcome, ms, did, ended, def.name, token])).changes);
+            }
+            if (changed !== 1)
+                return 'lost';
+            // Recorded only by the settlement that holds the token (a lost settlement writes nothing): cause `paused` for a pause that
+            // came during the run, else `worker_down`, since no run started at their time. Due times the ledger holds (a resume during
+            // the run recorded them) are not recorded again.
+            if (backlog.length > 0) {
+                const have = new Set((await trx.all(`SELECT due_at FROM ${t('missed')} WHERE job = ? AND due_at >= ? AND due_at <= ?`, [def.name, backlog[0], backlog[backlog.length - 1]])).map((m) => Number(m.due_at)));
+                const fresh = backlog.filter((due) => !have.has(due));
+                if (fresh.length > 0)
+                    await applyRule(trx, def.name, rule, paused ? 'paused' : 'worker_down', fresh, ended, dropped);
+            }
+            await trx.run(`UPDATE ${t('runs')} SET ended_at = ?, outcome = ?, ms = ?, tries = ?, kind = ?, did = ? WHERE id = ?`, [ended, outcome, ms, attempt, kind, did, runId]);
+            if (outcome !== 'failed' && !byHand)
+                await trx.run(`UPDATE ${t('missed')} SET run_id = ?, acted_at = ? WHERE job = ? AND action = 'caught_up' AND run_id IS NULL AND due_at <= ?`, [runId, ended, def.name, dueAt]);
+            return settlement;
+        });
+        if (result === 'lost') {
+            // Another process took the lease, so its claim reads this run lost (and pauses a job not safe to repeat). The run is left
+            // open for that claim: closing it here, between the other claim and its look for open runs, would let that claim run a
+            // job not safe to repeat a second time.
+            logger.warn('job_run', { job: def.name, outcome: 'failed', ms, tries: attempt });
+            return;
+        }
+        logger.info('job_run', { job: def.name, outcome, ms, tries: attempt });
+        const view = await get(def.name);
+        const run = await runById(runId);
+        if (!view || !run)
+            return;
+        for (const handler of runHandlers) {
+            try {
+                handler(view, run);
+            }
+            catch { /* the handler's own */ }
+        }
+        if (result === 'failed_out') {
+            logger.warn('job_failed', { job: def.name, kind: kind ?? 'error', tries: attempt });
+            for (const handler of failedHandlers) {
+                try {
+                    handler(view, run);
+                }
+                catch { /* the handler's own */ }
+            }
+        }
+    }
+    const refusal = (refused, status, retryAt) => (retryAt === undefined ? { refused, status } : { refused, status, retryAt });
+    async function pause(name, actor, note) {
+        if (note !== undefined && note.length > NOTE_MAX)
+            throw new RangeError(`a pause note is at most ${NOTE_MAX} characters`);
+        const at = now();
+        return db.transaction(async (trx) => {
+            const row = await trx.get(`SELECT * FROM ${t('jobs')} WHERE name = ?`, [name]);
+            if (!row || name === WATCH_ROW)
+                return refusal('not_found', 404);
+            if (row.kind === 'reports_only')
+                return refusal('reports_only', 409);
+            if (row.paused_by !== null)
+                return refusal('already', 409);
+            // Every command is a conditional update whose changed-row count decides it: two transactions may read the same free
+            // row, and only one may write its audit entry and answer success. The read above gives the refusal its word; the write decides.
+            const changed = await trx.run(`UPDATE ${t('jobs')} SET paused_by = ?, paused_at = ?, paused_note = ?, updated_at = ? WHERE name = ? AND paused_by IS NULL AND kind = 'scheduled'`, [actor.accountId, at, note ?? null, at, name]);
+            if (Number(changed.changes) !== 1)
+                return refusal('already', 409);
+            await options.audit?.(trx, { kind: 'job_paused', staffAccountId: actor.accountId, subject: name, detail: note === undefined ? {} : { note } });
+            logger.info('job_paused', { job: name, actor: 'staff' });
+            return viewFrom(trx, name, at);
+        });
+    }
+    async function resume(name, actor) {
+        const at = now();
+        return db.transaction(async (trx) => {
+            const row = await trx.get(`SELECT * FROM ${t('jobs')} WHERE name = ?`, [name]);
+            if (!row || name === WATCH_ROW)
+                return refusal('not_found', 404);
+            if (row.kind === 'reports_only')
+                return refusal('reports_only', 409);
+            if (row.paused_by === null)
+                return refusal('already', 409);
+            const schedule = scheduleOf(row);
+            const from = num(row.next_due_at);
+            // A scheduled run claimed before the pause and still under its lease serves `next_due_at`: that due time is its, never
+            // missed, and its settlement steps the schedule on, so this resume leaves `next_due_at` to it.
+            const serving = row.lease_until !== null && Number(row.lease_until) >= at && row.hand_run_at === null;
+            // `next_due_at` is always a due time of the schedule (a retry's readiness is `retry_at`, which the resume clears), so the
+            // window starts there; due times the ledger already holds (a catch-up queued before the pause) are not recorded again.
+            const { times: window, dropped } = from !== null && from <= at ? missedWindow(schedule, row.rule, serving ? undefined : from, from, at) : { times: [], dropped: 0 };
+            const recorded = window.length === 0 ? new Set() : new Set((await trx.all(`SELECT due_at FROM ${t('missed')} WHERE job = ? AND due_at >= ? AND due_at <= ?`, [name, window[0], window[window.length - 1]])).map((m) => Number(m.due_at)));
+            const times = window.filter((due) => !recorded.has(due));
+            // The pause is lifted first, conditionally; the missed rows are written only when this transaction lifted it, so a second
+            // resume at once writes nothing and reads `already`.
+            const changed = await trx.run(`UPDATE ${t('jobs')} SET paused_by = NULL, paused_at = NULL, paused_note = NULL, failed_at = NULL, failed_kind = NULL, alerted_at = NULL, tries = 0, retry_at = NULL, updated_at = ? WHERE name = ? AND paused_by IS NOT NULL`, [at, name]);
+            if (Number(changed.changes) !== 1)
+                return refusal('already', 409);
+            const { catchUp } = await applyRule(trx, name, row.rule, 'paused', times, at, dropped);
+            // A catch-up owed (this resume's, or one queued before the pause that the ledger holds) runs at once; else a due time still
+            // ahead is kept (nothing was missed, so the resume moves nothing); else the next due time after now. A run in flight on
+            // `next_due_at` reads the ledger when it settles and runs an owed catch-up next.
+            if (!serving) {
+                const owed = await trx.get(`SELECT MAX(due_at) AS d FROM ${t('missed')} WHERE job = ? AND action = 'caught_up' AND run_id IS NULL`, [name]);
+                const next = catchUp ? times[times.length - 1] : owed?.d !== null && owed?.d !== undefined ? Number(owed.d) : from !== null && from > at ? from : (nextAfter(schedule, window[window.length - 1] ?? at, at) ?? null);
+                await trx.run(`UPDATE ${t('jobs')} SET next_due_at = ?, updated_at = ? WHERE name = ?`, [next, at, name]);
+            }
+            await options.audit?.(trx, { kind: 'job_resumed', staffAccountId: actor.accountId, subject: name, detail: { missed: times.length } });
+            logger.info('job_resumed', { job: name, actor: 'staff' });
+            return viewFrom(trx, name, at);
+        });
+    }
+    async function runNow(name, actor) {
+        const at = now();
+        // The bound counts the hand runs of this UTC clock hour, so `retryAt`, the next whole hour, is when one is taken again.
+        // The place is taken before the first await and given back on any refusal or throw, so two asks at once never both pass.
+        const hourStart = Math.floor(at / HOUR) * HOUR;
+        while (handRunsAt.length > 0 && handRunsAt[0] < hourStart)
+            handRunsAt.shift();
+        if (handRunsAt.length >= (options.handRunsPerHour ?? DEFAULT_HAND_RUNS_PER_HOUR))
+            return refusal('hand_runs_used', 429, hourStart + HOUR);
+        handRunsAt.push(at);
+        const giveBack = () => { const i = handRunsAt.lastIndexOf(at); if (i >= 0)
+            handRunsAt.splice(i, 1); };
+        let answer;
+        try {
+            answer = await queueHandRun(name, actor, at);
+        }
+        catch (error) {
+            giveBack();
+            throw error;
+        }
+        if ('refused' in answer)
+            giveBack();
+        return answer;
+    }
+    async function queueHandRun(name, actor, at) {
+        return db.transaction(async (trx) => {
+            const row = await trx.get(`SELECT * FROM ${t('jobs')} WHERE name = ?`, [name]);
+            if (!row || name === WATCH_ROW)
+                return refusal('not_found', 404);
+            if (row.kind === 'reports_only')
+                return refusal('reports_only', 409);
+            if (row.safe_to_repeat !== 1)
+                return refusal('not_safe', 409);
+            if (row.lease_until !== null && Number(row.lease_until) > at)
+                return refusal('in_flight', 409);
+            if (row.hand_run_at !== null)
+                return refusal('already', 409);
+            const changed = await trx.run(`UPDATE ${t('jobs')} SET hand_run_by = ?, hand_run_at = ?, updated_at = ? WHERE name = ? AND hand_run_at IS NULL AND (lease_until IS NULL OR lease_until < ?)`, [actor.accountId, at, at, name, at]);
+            if (Number(changed.changes) !== 1)
+                return refusal('already', 409);
+            await options.audit?.(trx, { kind: 'job_run', staffAccountId: actor.accountId, subject: name, detail: {} });
+            logger.info('job_hand_run', { job: name, actor: 'staff' });
+            return { queued: true };
+        });
+    }
+    async function missedCounts(reader, at) {
+        const rows = await reader.all(`SELECT job, COUNT(*) AS n FROM ${t('missed')} WHERE due_at >= ? GROUP BY job`, [at - 86_400_000]);
+        return new Map(rows.map((r) => [r.job, Number(r.n)]));
+    }
+    async function runningSince(reader, name) {
+        const run = await reader.get(`SELECT started_at FROM ${t('runs')} WHERE job = ? AND outcome = 'running' ORDER BY started_at DESC LIMIT 1`, [name]);
+        return run ? Number(run.started_at) : null;
+    }
+    async function viewFrom(reader, name, at) {
+        const row = await reader.get(`SELECT * FROM ${t('jobs')} WHERE name = ?`, [name]);
+        if (!row)
+            return refusal('not_found', 404);
+        return viewOf(row, at, (await missedCounts(reader, at)).get(name) ?? 0, await runningSince(reader, name));
+    }
+    async function get(name) {
+        if (name === WATCH_ROW)
+            return null;
+        const view = await viewFrom(db, name, now());
+        return 'refused' in view ? null : view;
+    }
+    const RANK = { failed: 0, stalled: 0, missed: 1, paused: 2, running: 3, no_run: 3, ok: 3 };
+    async function list(filter = {}) {
+        const at = now();
+        const rows = await db.all(`SELECT * FROM ${t('jobs')} WHERE name <> ? ORDER BY name`, [WATCH_ROW]);
+        const missed = await missedCounts(db, at);
+        const views = [];
+        for (const row of rows)
+            views.push(viewOf(row, at, missed.get(row.name) ?? 0, row.lease_until !== null && Number(row.lease_until) > at ? await runningSince(db, row.name) : null));
+        const wanted = views.filter((v) => (filter.process === undefined || v.process === filter.process) && (filter.state === undefined || (filter.state === 'attention' ? v.state === 'failed' || v.state === 'stalled' || v.state === 'missed' : v.state === filter.state)));
+        wanted.sort((a, b) => RANK[a.state] - RANK[b.state] || (a.nextDueAt ?? Number.MAX_SAFE_INTEGER) - (b.nextDueAt ?? Number.MAX_SAFE_INTEGER) || a.name.localeCompare(b.name));
+        // The cursor comes back from a host's query string: anything but a whole number from zero up reads as the first page.
+        const asked = filter.cursor === undefined ? 0 : Number(filter.cursor);
+        const offset = Number.isInteger(asked) && asked >= 0 ? asked : 0;
+        const page = wanted.slice(offset, offset + LIST_PAGE);
+        return { jobs: page, cursor: offset + LIST_PAGE < wanted.length ? String(offset + LIST_PAGE) : null };
+    }
+    async function runs(name, opts = {}) {
+        // A whole number from 1 to 100; a value that is no finite number reads as the default 20.
+        const asked = opts.limit !== undefined && Number.isFinite(opts.limit) ? Math.trunc(opts.limit) : 20;
+        const rows = await db.all(`SELECT * FROM ${t('runs')} WHERE job = ? ORDER BY started_at DESC LIMIT ?`, [name, Math.min(Math.max(asked, 1), 100)]);
+        return rows.map(runViewOf);
+    }
+    async function runById(id) {
+        const row = await db.get(`SELECT * FROM ${t('runs')} WHERE id = ?`, [id]);
+        return row ? runViewOf(row) : null;
+    }
+    async function missed(name, opts = {}) {
+        const rows = await db.all(`SELECT * FROM ${t('missed')} WHERE job = ? AND due_at >= ? ORDER BY due_at DESC LIMIT 50`, [name, opts.sinceMs ?? now() - 7 * 86_400_000]);
+        return rows.map(missedViewOf);
+    }
+    async function forgetActor(trx, accountId) {
+        // The pause stands with its actor forgotten; a hand run's actor is nulled on the job and on its runs.
+        await trx.run(`UPDATE ${t('jobs')} SET paused_by = 'system' WHERE paused_by = ?`, [accountId]);
+        await trx.run(`UPDATE ${t('jobs')} SET hand_run_by = NULL WHERE hand_run_by = ?`, [accountId]);
+        await trx.run(`UPDATE ${t('runs')} SET by_hand = NULL WHERE by_hand = ?`, [accountId]);
+    }
+    async function settled() {
+        await Promise.all([...inFlight.values()].map((f) => f.promise));
+        await reporter.pending();
+    }
+    async function stop(opts = {}) {
+        stopping = true;
+        if (timer !== null) {
+            clearInterval(timer);
+            timer = null;
+        }
+        // A tick in progress claims nothing more and frees a claim it took after this point; its runs join the ones held below.
+        await currentTick;
+        const waitMs = opts.waitMs ?? DEFAULT_STOP_WAIT_MS;
+        // No tick renews a lease once stopping, so each lease in flight is held through the wait at once (fenced by its token):
+        // another process never takes a job whose body is still running inside the wait.
+        const at = now();
+        for (const [name, flight] of inFlight) {
+            try {
+                await db.run(`UPDATE ${t('jobs')} SET lease_until = ?, updated_at = ? WHERE name = ? AND lease_token = ?`, [at + waitMs + 2 * tickMs, at, name, flight.token]);
+            }
+            catch {
+                // The database refused or could not be reached: the lease runs out as it stood, and the settlement stays fenced by the token.
+            }
+        }
+        let wait;
+        await Promise.race([settled(), new Promise((resolve) => { wait = setTimeout(resolve, waitMs); wait.unref?.(); })]);
+        if (wait !== undefined)
+            clearTimeout(wait);
+        for (const flight of inFlight.values())
+            if (!flight.controller.signal.aborted)
+                flight.controller.abort(new Error('stop'));
+    }
+    const subscribe = (set) => (handler) => { set.add(handler); return () => { set.delete(handler); }; };
+    return {
+        options, register, report: reporter.report, start, stop, pause, resume, runNow, list, get, runs, missed,
+        onRun: subscribe(runHandlers), onFailed: subscribe(failedHandlers), forgetActor, tick, settled,
+    };
+}
+//# sourceMappingURL=scheduler.js.map