@harperfast/harper 5.2.0-beta.1 → 5.2.0-beta.2

package/dist/resources/scheduler/engine.js

@@ -0,0 +1,767 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PROMOTION_ESCALATION_MS = exports.FAILOVER_WATCHER_INTERVAL_MS = exports.STALE_THRESHOLD_MS = exports.HEARTBEAT_INTERVAL_MS = void 0;
+exports.safeErrorMessage = safeErrorMessage;
+exports.pickNextLeader = pickNextLeader;
+exports.isHeartbeatStale = isHeartbeatStale;
+exports.promotionWaitMs = promotionWaitMs;
+exports.findMissedCronOccurrence = findMissedCronOccurrence;
+exports.registerComponentJobs = registerComponentJobs;
+exports.unregisterComponentJobs = unregisterComponentJobs;
+exports.startSchedulerEngine = startSchedulerEngine;
+exports.electionSettledForTests = electionSettledForTests;
+exports.runHeartbeatForTests = runHeartbeatForTests;
+exports.runFailoverCheckForTests = runFailoverCheckForTests;
+exports.stopSchedulerEngine = stopSchedulerEngine;
+exports.getEngineRole = getEngineRole;
+exports.getRegisteredJobNames = getRegisteredJobNames;
+exports.sanitizeStoredError = sanitizeStoredError;
+const node_os_1 = require("node:os");
+const databases_ts_1 = require("../databases.js");
+const Server_ts_1 = require("../../server/Server.js");
+const harper_logger_ts_1 = __importDefault(require("../../utility/logging/harper_logger.js"));
+const CronExpression_ts_1 = require("./CronExpression.js");
+const schedulerLogger = harper_logger_ts_1.default.forComponent('scheduler');
+// One row per scheduled job (run state) plus a singleton leader-lease row.
+// Lives in the system database and replicates (auditing enabled) so every node
+// sees the current leader's heartbeat and each job's last run — that shared
+// view is what makes "fires once per cluster" and failover catch-up work.
+const SCHEDULER_STATE_TABLE = 'hdb_scheduler_state';
+const LEADER_ROW_ID = 'leader';
+// Timing constants are env-overridable (same pattern as liveSubscriptionAuth's
+// sweep interval) primarily so multi-node integration tests can exercise
+// failover without waiting out the production thresholds
+function timingFromEnv(name, defaultMs) {
+    const value = Number(process.env[name]);
+    return Number.isFinite(value) && value > 0 ? value : defaultMs;
+}
+exports.HEARTBEAT_INTERVAL_MS = timingFromEnv('HARPER_SCHEDULER_HEARTBEAT_INTERVAL_MS', 60_000);
+const rawStaleThreshold = timingFromEnv('HARPER_SCHEDULER_STALE_THRESHOLD_MS', 5 * 60 * 1000);
+// A stale threshold at or below the heartbeat interval makes every healthy
+// leader look dead (leadership flaps, each promotion re-running catch-up);
+// clamp the misconfiguration rather than honor it (audit finding)
+exports.STALE_THRESHOLD_MS = rawStaleThreshold > exports.HEARTBEAT_INTERVAL_MS ? rawStaleThreshold : exports.HEARTBEAT_INTERVAL_MS * 5;
+exports.FAILOVER_WATCHER_INTERVAL_MS = timingFromEnv('HARPER_SCHEDULER_FAILOVER_WATCHER_INTERVAL_MS', 75_000);
+// setTimeout clamps to a 32-bit signed int; longer delays wrap to ~1ms and busy-loop
+const MAX_TIMEOUT_MS = 0x7fffffff;
+const MAX_STORED_ERROR_LENGTH = 500;
+let engineStarted = false;
+let role = 'inactive';
+let heartbeatTimer;
+let failoverWatcherTimer;
+let catchUpRunning = false;
+// First time this follower observed the cluster leaderless (no row or stale
+// heartbeat); drives the promotion escalation ladder in failoverCheck
+let leaderlessSince;
+// When this node became leader — fallback for the lease row's initializedAt
+// when a heartbeat renews after a failed read
+let leaderInitializedAt;
+// The in-flight (or settled) election kicked off by startSchedulerEngine —
+// held so tests can await the role decision instead of polling
+let electionPromise;
+// Bumped by stopSchedulerEngine: async transitions (electRole, becomeLeader)
+// capture it before their awaits and abandon themselves if it moved — an
+// in-flight election must not resurrect timers after a stop (review finding)
+let engineEpoch = 0;
+const jobsByComponent = new Map();
+let _stateTable;
+function getStateTable() {
+    _stateTable ??= (0, databases_ts_1.table)({
+        database: 'system',
+        table: SCHEDULER_STATE_TABLE,
+        // The cluster-once guarantee depends on every node seeing this table:
+        // declare replication POSITIVELY rather than relying on absence from the
+        // non-replicating list, so an operator-scoped `replication.databases`
+        // config cannot silently exclude it (review finding — that degradation
+        // mode is N independent leaders running every job N times)
+        replicate: true,
+        // Replication of system tables requires auditing
+        audit: true,
+        attributes: [
+            // 'leader' for the lease row; 'job:<component>:<name>' for run state
+            { name: 'id', type: 'string', isPrimaryKey: true },
+            { name: 'leaderNode', type: 'string' },
+            { name: 'lastHeartbeat', type: 'string' },
+            { name: 'initializedAt', type: 'string' },
+            { name: 'firstSeenAt', type: 'string' },
+            { name: 'lastRunAt', type: 'string' },
+            { name: 'lastStatus', type: 'string' },
+            { name: 'lastError', type: 'string' },
+            { name: 'lastDurationMs', type: 'number' },
+        ],
+    });
+    return _stateTable;
+}
+function jobRowId(job) {
+    return `job:${job.componentName}:${job.name}`;
+}
+// Every state-table operation is bounded: observed empirically (2-node
+// failover) that a read/write issued on a freshly promoted leader — while the
+// replication link to the dead peer is still churning — can stall
+// indefinitely, wedging the promotion pipeline behind its await while the
+// already-started heartbeat keeps renewing the lease. The result was a
+// healthy-looking leader running zero jobs, with no error anywhere. A bounded
+// operation instead fails into the existing degraded paths (schedule from
+// now / skip this sweep / retry next heartbeat).
+const STATE_OPERATION_TIMEOUT_MS = 10_000;
+function withStateTimeout(operation, what) {
+    return new Promise((resolve, reject) => {
+        const timer = setTimeout(() => reject(new Error(`${what} timed out after ${STATE_OPERATION_TIMEOUT_MS}ms`)), STATE_OPERATION_TIMEOUT_MS);
+        timer.unref();
+        operation.then((value) => {
+            clearTimeout(timer);
+            resolve(value);
+        }, (error) => {
+            clearTimeout(timer);
+            reject(error);
+        });
+    });
+}
+/**
+ * Job handlers and storage drivers can throw anything — including objects
+ * whose property getters themselves throw — so even reading .message is
+ * guarded here (surfaced by review).
+ */
+function safeErrorMessage(error) {
+    try {
+        return error?.message || String(error);
+    }
+    catch {
+        try {
+            return String(error);
+        }
+        catch {
+            return 'unknown error';
+        }
+    }
+}
+function currentNodeName() {
+    // server.hostname is the node's replication identity; hostname() is only a
+    // local-dev fallback where there is no cluster and the name is just a label
+    return Server_ts_1.server.hostname || (0, node_os_1.hostname)();
+}
+function nodeRoster() {
+    // server.nodes lists PEER nodes only (populated by the replication
+    // component); the current node is not included and the list is absent
+    // entirely on a single standalone instance
+    const peers = (Server_ts_1.server.nodes || []).map((node) => node?.name).filter(Boolean);
+    return [...new Set([...peers, currentNodeName()])].sort();
+}
+/**
+ * Deterministic election: the alphabetically-first node wins, excluding a
+ * stale leader so a wedged node cannot immediately re-elect itself. Every node
+ * evaluates this identically against the same replicated state, so no
+ * consensus round is needed.
+ */
+function pickNextLeader(roster, staleLeader) {
+    if (roster.length === 0)
+        return null;
+    if (!staleLeader)
+        return roster[0];
+    const eligible = roster.filter((name) => name !== staleLeader);
+    return eligible.length > 0 ? eligible[0] : roster[0];
+}
+function isHeartbeatStale(lastHeartbeat, now = Date.now()) {
+    if (!lastHeartbeat)
+        return true;
+    const heartbeatTime = Date.parse(lastHeartbeat);
+    return Number.isNaN(heartbeatTime) || now - heartbeatTime > exports.STALE_THRESHOLD_MS;
+}
+// Escalation ladder for leaderless promotion: each successive fallback node
+// waits this much longer before claiming leadership, giving preferred nodes
+// (which check every FAILOVER_WATCHER_INTERVAL_MS) time to claim it first.
+exports.PROMOTION_ESCALATION_MS = 2 * exports.FAILOVER_WATCHER_INTERVAL_MS;
+/**
+ * How long this node should observe a leaderless cluster before promoting
+ * itself. The preferred (alphabetically-first eligible) node promotes
+ * immediately; each subsequent node adds one escalation interval, so a dead or
+ * never-started preferred node cannot deadlock the cluster — the next node in
+ * line claims leadership one rung later, and sticky leadership plus the
+ * heartbeat takeover check heal any race between rungs.
+ */
+function promotionWaitMs(roster, self, staleLeader) {
+    const eligible = staleLeader ? roster.filter((name) => name !== staleLeader) : roster;
+    const queue = eligible.length > 0 ? eligible : roster;
+    const queueIndex = queue.indexOf(self);
+    // A node not in the queue (it IS the stale leader) goes to the back
+    const rung = queueIndex < 0 ? queue.length : queueIndex;
+    return rung * exports.PROMOTION_ESCALATION_MS;
+}
+/**
+ * The cron occurrence that should have fired but didn't, or null if the job is
+ * up to date. `baseline` is the job's last run (or when it was first seen, so
+ * a newly-deployed job waits for its first scheduled time instead of firing
+ * immediately).
+ */
+function findMissedCronOccurrence(cron, timezone, baseline, now) {
+    const previousOccurrence = cron.previousDate(now, timezone ?? (0, CronExpression_ts_1.getSystemTimezone)());
+    if (previousOccurrence && previousOccurrence.getTime() > baseline.getTime())
+        return previousOccurrence;
+    return null;
+}
+/**
+ * Register (or replace) the scheduled jobs for a component. Called from the
+ * scheduler plugin's handleApplication on the scheduling worker; safe to call
+ * repeatedly — a reload or redeploy replaces the component's whole job set.
+ */
+function registerComponentJobs(componentName, jobs) {
+    unregisterComponentJobs(componentName);
+    const jobMap = new Map();
+    for (const job of jobs) {
+        jobMap.set(job.name, { ...job, running: false });
+    }
+    jobsByComponent.set(componentName, jobMap);
+    if (role === 'leader') {
+        // Fire-and-forget: scheduling reads run state from the DB and must not
+        // block handleApplication (which holds a cross-thread load lock)
+        scheduleComponentJobs(componentName).catch((error) => {
+            schedulerLogger.error?.(`Failed to schedule jobs for ${componentName}`, error);
+        });
+    }
+}
+/**
+ * Cancel timers and forget the jobs of a component (its scope is closing —
+ * worker shutdown, redeploy, or a discarded deploy-validation load).
+ *
+ * Leadership is deliberately retained even if this empties the job set: the
+ * common cause is a reload that re-registers moments later, and stepping down
+ * here would leave the engine unable to re-elect (startSchedulerEngine is
+ * one-shot per worker). An idle leader heartbeating a zero-job cluster is
+ * harmless and resolves on the next worker restart.
+ */
+function unregisterComponentJobs(componentName) {
+    const jobMap = jobsByComponent.get(componentName);
+    if (!jobMap)
+        return;
+    for (const job of jobMap.values()) {
+        if (job.timer)
+            clearTimeout(job.timer);
+    }
+    jobsByComponent.delete(componentName);
+}
+/**
+ * Start the engine's cluster role (leader or follower). Idempotent; called
+ * once per scheduling worker regardless of how many components declare jobs.
+ * All the real work happens asynchronously so the caller (handleApplication,
+ * which holds a cross-thread load lock with a 30s timeout) returns fast.
+ */
+function startSchedulerEngine() {
+    if (engineStarted)
+        return;
+    engineStarted = true;
+    electionPromise = electRole().catch((error) => {
+        schedulerLogger.error?.('Scheduler engine failed to start', error);
+    });
+}
+/** @internal — testing only: resolves when the initial election has settled */
+function electionSettledForTests() {
+    return electionPromise ?? Promise.resolve();
+}
+/**
+ * @internal — testing only: run one heartbeat tick immediately (same body the
+ * interval runs), so tests can drive lease renewal / takeover checks without
+ * waiting out HEARTBEAT_INTERVAL_MS. Same seam pattern as
+ * setCoolingFunctionForTests in transactionLogCooling.
+ */
+function runHeartbeatForTests() {
+    return heartbeat();
+}
+/** @internal — testing only: run one failover-watcher tick immediately */
+function runFailoverCheckForTests() {
+    return failoverCheck();
+}
+/** Reset all engine state and timers. Intended for tests. */
+function stopSchedulerEngine() {
+    for (const componentName of [...jobsByComponent.keys()]) {
+        unregisterComponentJobs(componentName);
+    }
+    if (heartbeatTimer)
+        clearInterval(heartbeatTimer);
+    if (failoverWatcherTimer)
+        clearInterval(failoverWatcherTimer);
+    heartbeatTimer = undefined;
+    failoverWatcherTimer = undefined;
+    role = 'inactive';
+    engineStarted = false;
+    catchUpRunning = false;
+    leaderlessSince = undefined;
+    leaderInitializedAt = undefined;
+    electionPromise = undefined;
+    engineEpoch++;
+    _stateTable = undefined;
+}
+function getEngineRole() {
+    return role;
+}
+/** @internal — testing only */
+function getRegisteredJobNames(componentName) {
+    return [...(jobsByComponent.get(componentName)?.keys() ?? [])];
+}
+async function electRole() {
+    const self = currentNodeName();
+    const epoch = engineEpoch;
+    let leaderRow;
+    try {
+        leaderRow = await withStateTimeout(getStateTable().get(LEADER_ROW_ID), 'leader state read');
+    }
+    catch (error) {
+        if (epoch !== engineEpoch)
+            return; // stopped while electing
+        // Fail toward followership: electing ourselves while the state table is
+        // unreadable risks a second leader. The failover watcher keeps checking
+        // and promotes once reads succeed and show a leaderless cluster.
+        schedulerLogger.warn?.(`Could not read scheduler leader state, defaulting to follower: ${safeErrorMessage(error)}`);
+        becomeFollower();
+        return;
+    }
+    if (epoch !== engineEpoch)
+        return; // stopped while electing
+    // Sticky leadership: a node (re)starting while another node is actively
+    // leading defers to it rather than seizing leadership back
+    if (leaderRow && leaderRow.leaderNode !== self && !isHeartbeatStale(leaderRow.lastHeartbeat)) {
+        schedulerLogger.info?.(`Scheduler leader is ${leaderRow.leaderNode} (heartbeat fresh); ${self} watching for failover`);
+        becomeFollower();
+        return;
+    }
+    // A fresh lease naming THIS node means a prior same-node incarnation was
+    // just leading (overlapping worker restart) — skip the promotion catch-up
+    // so its still-committing runs are not immediately re-fired
+    const previousIncarnationActive = leaderRow != null && leaderRow.leaderNode === self && !isHeartbeatStale(leaderRow.lastHeartbeat);
+    const roster = nodeRoster();
+    if (roster.length <= 1 || pickNextLeader(roster, null) === self) {
+        await becomeLeader(previousIncarnationActive);
+    }
+    else {
+        schedulerLogger.info?.(`Scheduler leader election chose ${pickNextLeader(roster, null)}; ${self} is a follower`);
+        becomeFollower();
+    }
+}
+async function becomeLeader(skipInitialCatchUp = false) {
+    const self = currentNodeName();
+    const epoch = engineEpoch;
+    role = 'leader';
+    if (failoverWatcherTimer) {
+        clearInterval(failoverWatcherTimer);
+        failoverWatcherTimer = undefined;
+    }
+    // A second promotion (e.g. two failover checks in flight) must not orphan
+    // the first heartbeat interval — an orphaned heartbeat would keep renewing
+    // the lease after step-down with no timers armed, silently stopping all
+    // jobs cluster-wide (audit finding)
+    if (heartbeatTimer) {
+        clearInterval(heartbeatTimer);
+        heartbeatTimer = undefined;
+    }
+    schedulerLogger.info?.(`Scheduler leader started on ${self}`);
+    if (getStateTable().replicate === false) {
+        // The cluster-once guarantee is void if this table stops replicating;
+        // fail loudly instead of silently multiplying job executions
+        schedulerLogger.error?.(`${SCHEDULER_STATE_TABLE} is not replicating — scheduled jobs may run on every node instead of once per cluster`);
+    }
+    const now = new Date().toISOString();
+    leaderInitializedAt = now;
+    await putStateRow({ id: LEADER_ROW_ID, leaderNode: self, lastHeartbeat: now, initializedAt: now });
+    // A stop while the lease write was in flight must not resurrect the
+    // heartbeat interval (review finding: in-flight transitions surviving
+    // stopSchedulerEngine)
+    if (epoch !== engineEpoch)
+        return;
+    // The heartbeat interval must be beating BEFORE the promotion catch-up
+    // pass: catch-up runs user handlers serially and can exceed the stale
+    // threshold, and a leader that stops renewing mid-catch-up looks dead —
+    // the next follower would promote and re-run the same occurrences
+    // (review finding). runCatchUp is single-flight, so the first heartbeat
+    // tick overlapping the promotion pass skips its own sweep.
+    heartbeatTimer = setInterval(() => {
+        heartbeat().catch((error) => schedulerLogger.error?.('Scheduler heartbeat failed', error));
+    }, exports.HEARTBEAT_INTERVAL_MS);
+    heartbeatTimer.unref();
+    for (const componentName of jobsByComponent.keys()) {
+        await scheduleComponentJobs(componentName);
+    }
+    // Skipped when a prior same-node incarnation was just leading (overlapping
+    // worker restart): its runs may still be committing, and an immediate
+    // catch-up pass would re-fire the occurrence it is mid-executing. The
+    // heartbeat sweep still catches genuinely missed occurrences within 60s.
+    if (!skipInitialCatchUp)
+        await runCatchUp();
+}
+function becomeFollower() {
+    role = 'follower';
+    leaderlessSince = undefined;
+    if (heartbeatTimer) {
+        clearInterval(heartbeatTimer);
+        heartbeatTimer = undefined;
+    }
+    for (const jobMap of jobsByComponent.values()) {
+        for (const job of jobMap.values()) {
+            if (job.timer) {
+                clearTimeout(job.timer);
+                job.timer = undefined;
+            }
+        }
+    }
+    if (failoverWatcherTimer)
+        return;
+    failoverWatcherTimer = setInterval(() => {
+        failoverCheck().catch((error) => schedulerLogger.error?.('Scheduler failover check failed', error));
+    }, exports.FAILOVER_WATCHER_INTERVAL_MS);
+    failoverWatcherTimer.unref();
+}
+async function heartbeat() {
+    // A tick from a stale interval (already stepped down / re-promoted) must
+    // not renew the lease
+    if (role !== 'leader')
+        return;
+    const self = currentNodeName();
+    let leaderRow;
+    try {
+        leaderRow = await withStateTimeout(getStateTable().get(LEADER_ROW_ID), 'leader state read');
+    }
+    catch (error) {
+        // A failed read must NOT abort the tick: skipping renewal makes a live
+        // leader look stale (a follower would promote and dual-execute), and
+        // skipping the takeover check breaks the path that heals dual
+        // leadership. Renew blind; the step-down check runs on the next tick.
+        schedulerLogger.warn?.(`Failed to read leader state during heartbeat: ${safeErrorMessage(error)}`);
+    }
+    // Split-brain healing: if another node has taken over with a fresh
+    // heartbeat (e.g. we were partitioned long enough to be considered stale),
+    // step down instead of dueling over the lease row
+    if (leaderRow != null && leaderRow.leaderNode !== self && !isHeartbeatStale(leaderRow.lastHeartbeat)) {
+        schedulerLogger.info?.(`Scheduler leadership was taken over by ${leaderRow.leaderNode}; ${self} stepping down`);
+        becomeFollower();
+        return;
+    }
+    await putStateRow({
+        id: LEADER_ROW_ID,
+        leaderNode: self,
+        lastHeartbeat: new Date().toISOString(),
+        initializedAt: leaderRow?.initializedAt ?? leaderInitializedAt,
+    });
+    // Periodic missed-run sweep: catches occurrences lost to DST transitions,
+    // worker restarts, and anything else that slipped past the timers
+    await runCatchUp();
+}
+async function failoverCheck() {
+    if (role !== 'follower')
+        return;
+    const self = currentNodeName();
+    const epoch = engineEpoch;
+    let leaderRow;
+    try {
+        leaderRow = await withStateTimeout(getStateTable().get(LEADER_ROW_ID), 'leader state read');
+    }
+    catch (error) {
+        // Can't tell whether a leader exists; skip this tick (and don't let the
+        // leaderless clock run) rather than risk promoting into a split brain
+        schedulerLogger.warn?.(`Failed to read leader state during failover check: ${safeErrorMessage(error)}`);
+        leaderlessSince = undefined;
+        return;
+    }
+    if (leaderRow != null && !isHeartbeatStale(leaderRow.lastHeartbeat)) {
+        leaderlessSince = undefined;
+        return;
+    }
+    // Re-check after the await: a concurrent (stalled) check may have promoted
+    // this node already (audit finding), or the engine may have been stopped
+    // while the read was in flight (review finding) — either way, do not promote
+    if (role !== 'follower' || epoch !== engineEpoch)
+        return;
+    const now = Date.now();
+    leaderlessSince ??= now;
+    const roster = nodeRoster();
+    const staleLeader = leaderRow?.leaderNode ?? null;
+    const waitMs = promotionWaitMs(roster, self, staleLeader);
+    if (now - leaderlessSince >= waitMs) {
+        schedulerLogger.info?.(staleLeader
+            ? `Scheduler leader ${staleLeader} heartbeat is stale; ${self} promoting itself`
+            : `No active scheduler leader; ${self} promoting itself`);
+        leaderlessSince = undefined;
+        await becomeLeader();
+    }
+    else {
+        schedulerLogger.trace?.(`Scheduler leader is stale or absent; ${self} promotes in ${Math.round((waitMs - (now - leaderlessSince)) / 1000)}s unless a preferred node claims leadership`);
+    }
+}
+async function putStateRow(row) {
+    try {
+        await withStateTimeout(getStateTable().put(row), `state write ${row.id}`);
+    }
+    catch (error) {
+        // State persistence failures must never take the scheduler down; the
+        // next heartbeat retries
+        schedulerLogger.warn?.(`Failed to persist scheduler state row ${row.id}: ${safeErrorMessage(error)}`);
+    }
+}
+async function scheduleComponentJobs(componentName) {
+    const jobMap = jobsByComponent.get(componentName);
+    if (!jobMap || role !== 'leader')
+        return;
+    for (const job of jobMap.values()) {
+        await scheduleNextRun(job);
+    }
+}
+// A job is live only while it is the EXACT object the registry holds:
+// registerComponentJobs replaces objects under the same names, so a name-only
+// check would let an in-flight chain re-arm a replaced job forever (review
+// finding: persistent double-fire after redeploy-during-run)
+function isRegistered(job) {
+    return jobsByComponent.get(job.componentName)?.get(job.name) === job;
+}
+/**
+ * Compute the job's next fire time and arm its timer. Interval jobs anchor to
+ * their persisted last run so cadence survives restarts and failover; cron
+ * jobs fire at the next matching wall-clock time (missed occurrences are
+ * handled by the catch-up sweep instead).
+ *
+ * Concurrent calls for the same job coalesce onto one computation: the
+ * callers (becomeLeader's initial loop, a registration's fire-and-forget, a
+ * post-run reschedule, the heartbeat sweep) can otherwise interleave across
+ * the state read below, each pass the no-timer check, and each arm a timer —
+ * with only the last handle retained, leaving an uncancellable live timer
+ * (review finding).
+ */
+function scheduleNextRun(job) {
+    job.scheduling ??= computeAndArmNextRun(job).finally(() => {
+        job.scheduling = undefined;
+    });
+    return job.scheduling;
+}
+async function computeAndArmNextRun(job) {
+    if (role !== 'leader' || !isRegistered(job))
+        return;
+    if (job.timer)
+        clearTimeout(job.timer);
+    const now = new Date();
+    let fireAt;
+    if (job.cron) {
+        const next = job.cron.nextDate(now, job.timezone ?? (0, CronExpression_ts_1.getSystemTimezone)());
+        if (!next) {
+            // Transient (e.g. a DST-window computation edge): the catch-up sweep
+            // re-arms unarmed cron jobs every heartbeat, so this self-heals
+            schedulerLogger.warn?.(`Job ${jobRowId(job)} has no computable next occurrence; retrying at next heartbeat`);
+            return;
+        }
+        fireAt = next;
+    }
+    else {
+        const stateRow = await getJobStateRow(job);
+        const persistedLastRun = stateRow?.lastRunAt ? Date.parse(stateRow.lastRunAt) : NaN;
+        // Anchor to the LATEST of the persisted run and the in-memory attempt:
+        // the persisted value survives restarts/failover (overdue intervals fire
+        // immediately — the interval catch-up path), while the in-memory value
+        // survives state-WRITE failures so a frozen persisted row cannot cause
+        // an immediate-refire hot loop (review finding). Clamped to now: a
+        // FUTURE timestamp from a clock-skewed leader must not wedge the job
+        // (audit finding).
+        const anchor = Math.min(Math.max(Number.isNaN(persistedLastRun) ? 0 : persistedLastRun, job.lastAttemptAt ?? 0), now.getTime());
+        fireAt = anchor > 0 ? new Date(anchor + job.intervalMs) : new Date(now.getTime() + job.intervalMs);
+    }
+    // Re-check after the await: the registry (or our role) may have changed
+    // while reading job state, and arming a timer for a stale object leaks a
+    // timer nothing can cancel
+    if (role !== 'leader' || !isRegistered(job))
+        return;
+    armTimer(job, fireAt);
+}
+function armTimer(job, fireAt) {
+    const delay = fireAt.getTime() - Date.now();
+    // Fail closed on a non-finite target: setTimeout coerces NaN to ~1ms,
+    // which would hot-loop the job (review finding). Cron jobs re-arm via the
+    // heartbeat sweep; interval bounds are validated at config load.
+    if (!Number.isFinite(delay)) {
+        schedulerLogger.error?.(`Job ${jobRowId(job)} computed a non-finite fire time; not arming`);
+        return;
+    }
+    if (delay > MAX_TIMEOUT_MS) {
+        // Beyond setTimeout's 32-bit range: sleep the maximum and re-arm
+        job.timer = setTimeout(() => {
+            job.timer = undefined;
+            if (role === 'leader' && isRegistered(job))
+                armTimer(job, fireAt);
+        }, MAX_TIMEOUT_MS);
+    }
+    else {
+        job.timer = setTimeout(() => {
+            // Cleared at fire time so "unarmed" is observable: the heartbeat
+            // catch-up sweep re-arms cron jobs whose timer is missing
+            job.timer = undefined;
+            executeJob(job, fireAt, false)
+                .catch((error) => schedulerLogger.error?.(`Job ${jobRowId(job)} execution failed unexpectedly`, error))
+                .finally(() => {
+                scheduleNextRun(job).catch((error) => schedulerLogger.error?.(`Failed to reschedule job ${jobRowId(job)}`, error));
+            });
+        }, Math.max(delay, 0));
+    }
+    job.timer?.unref();
+}
+// The persisted error replicates cluster-wide; strip filesystem paths (which
+// leak node-local layout) and bound the length
+function sanitizeStoredError(message) {
+    return message
+        .replace(/\/(?:Users|home|var|tmp|opt|etc|root|srv|data|mnt)\/[^\s:)]+/g, '[path]')
+        .replace(/[A-Za-z]:\\[^\s:)]+/g, '[path]')
+        .slice(0, MAX_STORED_ERROR_LENGTH);
+}
+// Swallow-to-undefined is deliberate for the timer-arming path (an unreadable
+// row degrades to "schedule from now"); the catch-up sweep must NOT use this —
+// it needs to distinguish "row absent" from "read failed", because treating a
+// transient read error as "job never seen" previously triggered a destructive
+// first-seen overwrite of the whole run-state row (audit finding)
+async function getJobStateRow(job) {
+    try {
+        return await withStateTimeout(getStateTable().get(jobRowId(job)), `job state read ${jobRowId(job)}`);
+    }
+    catch (error) {
+        schedulerLogger.warn?.(`Failed to read state for job ${jobRowId(job)}: ${safeErrorMessage(error)}`);
+        return undefined;
+    }
+}
+async function executeJob(job, scheduledAt, catchUp) {
+    // The registration re-check matters as much as the role check: a timer can
+    // fire after its component's scope closed or after a reload replaced the
+    // job object, and a discarded component's handler must not run against
+    // production state (review finding)
+    if (role !== 'leader' || !isRegistered(job))
+        return;
+    // In-memory dedup for catch-up: the sweep decides from a state snapshot
+    // that can predate a run which started (or finished) while the snapshot
+    // read was in flight; lastAttemptAt is set synchronously at run start on
+    // this thread, so it cannot be stale (audit finding)
+    if (catchUp && job.lastAttemptAt !== undefined && job.lastAttemptAt >= scheduledAt.getTime()) {
+        return;
+    }
+    if (job.running) {
+        // Single-flight: a run that outlasts its own cadence is not stacked
+        schedulerLogger.debug?.(`Job ${jobRowId(job)} is still running; skipping this occurrence`);
+        return;
+    }
+    job.running = true;
+    const startedAt = new Date();
+    // The attempt anchor and persisted lastRunAt cover the occurrence, not
+    // just the wall clock: if a clock step-back lands the run before its own
+    // fireAt, recording the bare start time would make the next sweep
+    // re-deliver the occurrence (audit finding). Catch-up runs (scheduledAt in
+    // the past) keep startedAt. Set synchronously, before any await.
+    job.lastAttemptAt = Math.max(startedAt.getTime(), scheduledAt.getTime());
+    const lastRunAt = new Date(job.lastAttemptAt).toISOString();
+    const existingRow = await getJobStateRow(job);
+    schedulerLogger.trace?.(`Running job ${jobRowId(job)}${catchUp ? ' (catch-up)' : ''}`);
+    try {
+        await job.handler({ jobName: job.name, scheduledAt, catchUp });
+        const durationMs = Date.now() - startedAt.getTime();
+        schedulerLogger.trace?.(`Job ${jobRowId(job)} completed in ${durationMs}ms`);
+        await putStateRow({
+            id: jobRowId(job),
+            firstSeenAt: existingRow?.firstSeenAt ?? startedAt.toISOString(),
+            lastRunAt,
+            lastStatus: 'success',
+            lastError: undefined,
+            lastDurationMs: durationMs,
+        });
+    }
+    catch (error) {
+        const durationMs = Date.now() - startedAt.getTime();
+        const errorMessage = safeErrorMessage(error);
+        schedulerLogger.warn?.(`Job ${jobRowId(job)} failed after ${durationMs}ms: ${errorMessage}`);
+        await putStateRow({
+            id: jobRowId(job),
+            firstSeenAt: existingRow?.firstSeenAt ?? startedAt.toISOString(),
+            lastRunAt,
+            lastStatus: 'error',
+            lastError: sanitizeStoredError(errorMessage),
+            lastDurationMs: durationMs,
+        });
+    }
+    finally {
+        job.running = false;
+    }
+}
+/**
+ * Fire one catch-up run for every cron job whose most recent occurrence was
+ * missed (leader was down, worker restarted, DST skipped the slot, …). Runs on
+ * promotion and on every heartbeat; single-flight so a slow catch-up pass
+ * never overlaps the next one.
+ */
+async function runCatchUp() {
+    if (catchUpRunning || role !== 'leader')
+        return;
+    // TODO(#951): reap orphaned job:<component>:<name> rows whose job no longer
+    // exists in any registered component, so renamed/removed jobs don't
+    // accumulate state rows forever
+    catchUpRunning = true;
+    try {
+        const now = new Date();
+        // Snapshot: each await below yields, and a component reload can mutate
+        // jobsByComponent mid-sweep (surfaced by review)
+        const cronJobs = [];
+        for (const jobMap of jobsByComponent.values()) {
+            for (const job of jobMap.values()) {
+                if (job.cron)
+                    cronJobs.push(job); // interval jobs self-correct in scheduleNextRun
+            }
+        }
+        for (const job of cronJobs) {
+            // A job unregistered (or replaced) while the sweep was underway must
+            // not fire
+            if (!isRegistered(job))
+                continue;
+            try {
+                // Unlike the timer-arming path, a read failure here must THROW
+                // (the per-job catch below logs and skips) — conflating it with
+                // "row absent" fed the destructive re-seed below (audit finding)
+                const stateRow = await withStateTimeout(getStateTable().get(jobRowId(job)), `job state read ${jobRowId(job)}`);
+                // Baseline = the newest of the persisted timestamps and the
+                // in-memory attempt anchor. Including lastAttemptAt closes two
+                // duplicate-execution paths the persisted row alone cannot: a
+                // sustained state-WRITE outage freezing lastRunAt (the sweep
+                // would re-fire a completed occurrence every heartbeat), and a
+                // fast run completing while this sweep's read was in flight.
+                // Clamped to now so a clock-skewed future timestamp cannot
+                // suppress catch-up (audit findings).
+                const parsedBaseline = Math.max(Date.parse(stateRow?.lastRunAt ?? '') || 0, Date.parse(stateRow?.firstSeenAt ?? '') || 0, job.lastAttemptAt ?? 0);
+                if (parsedBaseline === 0) {
+                    // Row absent, or timestamps missing/unparseable: (re)seed the
+                    // baseline without firing. put is full-record replacement, so
+                    // preserve whatever fields exist, and skip while a run is in
+                    // flight so the seed cannot clobber its just-committed result
+                    if (!job.running) {
+                        await putStateRow({
+                            id: jobRowId(job),
+                            firstSeenAt: now.toISOString(),
+                            lastRunAt: stateRow?.lastRunAt,
+                            lastStatus: stateRow?.lastStatus,
+                            lastError: stateRow?.lastError,
+                            lastDurationMs: stateRow?.lastDurationMs,
+                        });
+                    }
+                }
+                else {
+                    const baseline = new Date(Math.min(parsedBaseline, now.getTime()));
+                    const missed = findMissedCronOccurrence(job.cron, job.timezone, baseline, now);
+                    if (missed) {
+                        schedulerLogger.info?.(`Job ${jobRowId(job)} missed its ${missed.toISOString()} occurrence; running catch-up`);
+                        await executeJob(job, missed, true);
+                    }
+                }
+                // Self-heal: a cron job left unarmed (nextDate returned null during
+                // a DST window, or a reschedule failed) gets its timer re-armed here
+                // every heartbeat instead of staying degraded until failover
+                if (!job.running && !job.timer && isRegistered(job)) {
+                    await scheduleNextRun(job);
+                }
+            }
+            catch (error) {
+                schedulerLogger.warn?.(`Catch-up check failed for job ${jobRowId(job)}: ${safeErrorMessage(error)}`);
+            }
+        }
+    }
+    finally {
+        catchUpRunning = false;
+    }
+}
+//# sourceMappingURL=engine.js.map