@north-light/crouter 0.3.154

package/dist/daemon/cron-run.js

@@ -0,0 +1,944 @@
+// src/daemon/cron-run.ts — the daemon-owned cron run executor: subprocess
+// spawn/capture/timeout/lease mechanics for due `crons` rows (the cron-spec
+// scheduler) PLUS the full settle surface — output dispositions, delivery
+// sinks, overlap policies, expiry, and failure escalation.
+//
+// A cron RUN is the enactment — there is no separate action to fire —
+// so a recurrence advances `fire_at` BEFORE the run (crash-safe: a crash
+// mid-run never re-fires a settled slot) while a one-shot row is deleted only
+// AFTER its run settles, so the run can still be observed while in flight.
+//
+// SETTLE RULES (the cron spec's output disposition + failure sections):
+//   silent      — record only, never deliver, never escalate.
+//   on-failure  — (default) no success delivery; a failure pauses the cron and
+//                 spawns a terminal escalation node to deal with it.
+//   always      — stdout becomes a delivery every run, at the cron's tier.
+//   on-change   — deliver only when stdout's hash differs from the previous
+//                 run's (including the very first run); the hash is updated
+//                 after EVERY settled run, delivered or not.
+// A FAILURE is: timeout kill, process error, nonzero exit, or an undeliverable
+// sink (target node gone/terminal, spawn threw, human submit threw). Every
+// disposition except `silent` escalates: pause + spawn — pausing is the
+// natural dedup (a paused row never refires). An out-of-band run (`cron run`)
+// never escalates or pauses: the invoker sees the result directly.
+//
+// Fire-and-forget contract: runDueCrons
+// NEVER awaits a subprocess in the tick — it kicks each child off
+// synchronously and returns one settlement promise per admitted run, each
+// resolving only after its subprocess has closed and its durable
+// record/lease/consume transition is written. The daemon tick registers those
+// promises for graceful-teardown drain but never blocks on a slow run.
+import { spawn } from 'node:child_process';
+import { createHash, randomBytes, randomUUID } from 'node:crypto';
+import { mkdirSync } from 'node:fs';
+import { join } from 'node:path';
+import { setTimeout as delay } from 'node:timers/promises';
+import { notifyDeck, submitDeck } from '@crouton-kit/humanloop';
+import { dueClockCrons, dueOverlapCrons, expiredIdleCrons, runningCrons, acquireCronRunLease, releaseCronRunLease, recordCronRun, listCronRuns, advanceCron, consumeCron, cancelCron, killLeasedCronRun, verifyLeasedCronRun, getCron, setCronState, setCronLastOutputHash, } from '../core/canvas/crons.js';
+import { getNode } from '../core/canvas/canvas.js';
+import { fullName } from '../core/canvas/labels.js';
+import { nodeDir } from '../core/canvas/paths.js';
+import { killProcessGroup } from '../core/canvas/pid.js';
+import { appendInbox } from '../core/feed/inbox.js';
+import { atomicWriteJson } from '../core/fs-utils.js';
+import { emitEvent } from '../core/events/emit.js';
+import { loadProfileManifest } from '../core/profiles/manifest.js';
+import { hasNoNaturalCycle } from '../core/runtime/revive-all.js';
+import { spawnChild } from '../core/runtime/spawn.js';
+import { cronWakeOrigin } from '../core/runtime/bearings.js';
+import { nextSlotAfter, cadenceDisplay } from '../core/wake.js';
+import { buildOperationalEnvBase } from '../core/runtime/spawn-env.js';
+import { parseSink } from './cron-sink.js';
+import { readProfileEnvVars } from '../core/profiles/env-store.js';
+import { registerCrouterRoot } from '../commands/human/shared.js';
+/** Output head caps for the run log (spec: "capped, e.g. 4KB"). */
+export const CRON_OUTPUT_HEAD_BYTES = 4 * 1024;
+/** A byte-capped capture of one output stream's head. Bytes are buffered and
+ *  decoded exactly once, at the end — decoding per chunk both mis-measured the
+ *  cap (UTF-16 code units, not bytes) and corrupted any multi-byte sequence
+ *  that straddled a chunk boundary. Truncation is MARKED, never silent: a
+ *  clipped body reads as complete output to whoever receives the delivery. */
+function newHeadCapture() {
+    const chunks = [];
+    let bytes = 0;
+    let truncated = false;
+    return {
+        take(chunk) {
+            const room = CRON_OUTPUT_HEAD_BYTES - bytes;
+            if (room <= 0) {
+                truncated = true;
+                return;
+            }
+            if (chunk.length > room) {
+                chunks.push(chunk.subarray(0, room));
+                bytes = CRON_OUTPUT_HEAD_BYTES;
+                truncated = true;
+                return;
+            }
+            chunks.push(chunk);
+            bytes += chunk.length;
+        },
+        decode() {
+            const text = Buffer.concat(chunks).toString('utf8');
+            return truncated ? `${text}\n[… output truncated at ${CRON_OUTPUT_HEAD_BYTES} bytes]` : text;
+        },
+    };
+}
+/** Bound between the run-timeout SIGTERM and a forced SIGKILL escalation, so
+ *  a signal-trapping command cannot stall teardown's detached drain. */
+const CRON_KILL_GRACE_MS = 2000;
+/** Poll interval while waiting for a SIGTERMed stale-lease run to actually die
+ *  (each probe is a `ps` call, so this is deliberately coarse). */
+const CRON_DEATH_POLL_MS = 250;
+/** sha256 of empty stdout — the change-detection hash a run that produced no
+ *  output carries (a failed spawn, a recovered stale lease). */
+const EMPTY_STDOUT_SHA256 = createHash('sha256').update('').digest('hex');
+/** Lead of a `cron_runs.delivered` string written by the overlap=skip pass.
+ *  Matched (not just written) — see `latestRunIsSkip`. */
+const SKIP_MARKER = 'skipped (overlap=skip)';
+/** One id per daemon process instance — distinguishes THIS process's own
+ *  in-flight run leases from a stale 'running' row a prior (crashed/restarted)
+ *  daemon left behind. */
+export const cronDaemonInstanceId = randomUUID();
+// ---------------------------------------------------------------------------
+// Broker-capacity admission — the fleet-safety invariant the retired clock pass
+// enforced (it reserved capacity before a due fresh revive and left an
+// unadmitted one due for the next poll, so a fleet of simultaneous alarms was
+// never stampeded). A cron's enactment is an opaque shell command, so the only
+// structural signals the daemon has are the armed sink and the crtr verb the
+// command invokes. Both misreadings are safe: a false negative behaves exactly
+// as an ungated run does today, and a false positive only DEFERS one poll while
+// the fleet is already at its hard cap.
+// ---------------------------------------------------------------------------
+/** crtr verbs that launch a broker: a node revive (single or graph-wide) and a
+ *  node birth. `node message send` is deliberately absent — a message is not a
+ *  launch (its wake rides the daemon's own capacity-gated inbox pass) — EXCEPT
+ *  in its `--fresh` form, which is a revive wearing the message verb. */
+const NODE_LAUNCHING_COMMAND = /\bcrtr\b[^\n]*?\bnode\s+lifecycle\s+revive\b|\bcrtr\b[^\n]*?\bnode\s+new\b|\bcrtr\b[^\n]*?\bcanvas\s+revive\b|\bcrtr\b[^\n]*?\bnode\s+message\s+send\b[^\n]*?\s--fresh\b/;
+/** Will this cron's run consume broker capacity? True when its armed sink
+ *  births a node, or its command invokes a node-launching crtr verb. A cron
+ *  that merely runs a shell command or sends a message reads false and is
+ *  never gated. */
+export function cronLaunchesNode(c) {
+    if (parseSink(c.sink)?.kind === 'spawn')
+        return true;
+    return NODE_LAUNCHING_COMMAND.test(c.command);
+}
+const liveRuns = new Map();
+/** The cron's profile, re-validated at delivery time and returned in canonical
+ *  id form when its manifest still resolves, else null (root fallback per spec). */
+function resolveLiveProfile(c) {
+    if (c.profile == null)
+        return null;
+    try {
+        return loadProfileManifest(c.profile).profileId;
+    }
+    catch {
+        return null;
+    }
+}
+// ---------------------------------------------------------------------------
+// Delivery sinks
+// ---------------------------------------------------------------------------
+/** Deliver stdout to a living node's inbox. Throws when the target is gone or
+ *  has no natural cycle ahead (finalized/terminal-done) — an undeliverable
+ *  sink is a FAILURE per spec, never a silent drop. */
+function deliverNodeSink(c, target, stdout) {
+    const meta = getNode(target);
+    if (meta === null)
+        throw new Error(`node sink ${target} is gone`);
+    if (hasNoNaturalCycle(meta)) {
+        throw new Error(`node sink ${target} has no natural cycle ahead (done or finalized)`);
+    }
+    const from = c.created_by != null && getNode(c.created_by) !== null ? c.created_by : null;
+    appendInbox(target, {
+        from,
+        tier: c.tier,
+        kind: 'message',
+        label: `⏰ cron ${c.name}`,
+        data: { body: stdout },
+    });
+}
+/** Birth a fresh node with stdout as its kickoff (the recommended sink for
+ *  jobs meant to outlive their creator). Returns the newborn's node id. */
+async function deliverSpawnSink(c, kind, stdout) {
+    const creatorAlive = c.created_by != null && getNode(c.created_by) !== null;
+    const prompt = [
+        `Scheduled cron "${c.name}" (${c.cron_id}) fired and delivered its output to you as your kickoff.`,
+        '',
+        '## Command output',
+        '',
+        stdout,
+    ].join('\n');
+    const res = await spawnChild({
+        kind,
+        cwd: c.cwd,
+        prompt,
+        // Creator alive → a managed terminal child of it. Creator gone → a
+        // PARENTLESS TERMINAL node: nobody to report to, but it still owes a final
+        // so it reaps, and it is never brought forefront. A resident root here
+        // would leave an hourly cron accreting ~24 permanently-live nodes a day
+        // (the count-based prune never deletes live nodes) and would steal the
+        // human's tmux focus on every unattended fire.
+        ...(creatorAlive ? { parent: c.created_by } : { root: true, rootLifecycle: 'terminal' }),
+        readiness: 'launch',
+        profile: resolveLiveProfile(c),
+        wakeOrigin: cronWakeOrigin(c, 'node_birth'),
+    });
+    return res.node.node_id;
+}
+/** Deliver stdout through humanloop as a durable notify deck (fire-and-forget
+ *  inbox state — jobs whose output is a person's call). Mirrors `crtr human
+ *  notify` minus the CLI-env source stamp: the deck names the cron instead. */
+function deliverHumanSink(c, stdout) {
+    const id = `nfy-${randomBytes(4).toString('hex')}`;
+    const idir = nodeDir(id);
+    mkdirSync(idir, { recursive: true });
+    const rc = { mode: 'notify' };
+    atomicWriteJson(join(idir, 'run.json'), rc);
+    let deck = notifyDeck(`⏰ cron ${c.name}`, {
+        subtitle: `Scheduled cron "${c.name}" delivered output for your attention.`,
+        body: stdout,
+    });
+    deck = { ...deck, source: { sessionName: `cron ${c.name} (${c.cron_id})` } };
+    submitDeck({ root: registerCrouterRoot(), id, deck });
+}
+/** Route stdout to the cron's armed sink. Returns the delivered string on
+ *  success; throws on an undeliverable sink (the caller's failure path). */
+async function deliverToSink(c, stdout) {
+    const sink = parseSink(c.sink);
+    if (sink === null)
+        return 'no sink armed — nothing delivered';
+    if (sink.kind === 'node') {
+        deliverNodeSink(c, sink.node, stdout);
+        return `delivered: node ${sink.node}`;
+    }
+    if (sink.kind === 'spawn') {
+        const nodeId = await deliverSpawnSink(c, sink.spawn, stdout);
+        return `delivered: spawned ${nodeId}`;
+    }
+    deliverHumanSink(c, stdout);
+    return 'delivered: human notify';
+}
+function failureSummary(ctx, deliveryFailure) {
+    if (deliveryFailure !== null)
+        return deliveryFailure;
+    if (ctx.timedOut)
+        return `run timed out after ${ctx.runTimeoutS}s (killed; recorded exit -1)`;
+    if (ctx.processErrorMessage !== null)
+        return `run process error: ${ctx.processErrorMessage}`;
+    return `command exited ${ctx.exitCode ?? '(no exit observed)'}`;
+}
+/** The escalation node's kickoff: the cron's identity/config, this failure,
+ *  the run-log ring, the creating-node reference, and a mandate that names its
+ *  actual options (resume after verifying / cancel / ask the human). */
+function buildEscalationKickoff(c, ctx, deliveryFailure) {
+    const creator = c.created_by != null ? getNode(c.created_by) : null;
+    const creatorLine = c.created_by == null
+        ? 'created by: (no creating node recorded)'
+        : creator !== null
+            ? `created by: node ${c.created_by} ("${fullName(creator)}") — still on the canvas; consider telling it what happened`
+            : `created by: node ${c.created_by} (now gone)`;
+    const schedule = c.recur != null ? `every ${cadenceDisplay(c.recur)}` : `one-shot at ${c.fire_at}`;
+    const priorRuns = listCronRuns(c.cron_id)
+        .slice(0, 10)
+        .map((r) => {
+        const exit = r.exit_code === null ? 'no exit' : `exit ${r.exit_code}`;
+        const head = (r.stderr_head ?? r.stdout_head ?? '').split('\n')[0]?.slice(0, 120) ?? '';
+        return `- ${r.started} — ${exit}, ${r.duration_ms}ms — ${r.delivered ?? ''}${head !== '' ? ` — ${head}` : ''}`;
+    });
+    return [
+        `A scheduled cron FAILED and has paused itself. You were spawned to deal with it.`,
+        '',
+        `- cron: "${c.name}" (${c.cron_id})`,
+        `- schedule: ${schedule}${c.tz != null ? ` (tz ${c.tz})` : ''}`,
+        `- context: cwd ${c.cwd}${c.profile != null ? `, profile ${c.profile}` : ''}`,
+        `- ${creatorLine}`,
+        `- this failure: ${failureSummary(ctx, deliveryFailure)}`,
+        '',
+        '## The command',
+        '',
+        '```bash',
+        c.command,
+        '```',
+        '',
+        '## This run',
+        '',
+        `- exit: ${ctx.timedOut ? '-1 (timeout)' : String(ctx.exitCode)}`,
+        `- stdout (head): ${ctx.stdoutHead === '' ? '(empty)' : ''}`,
+        ...(ctx.stdoutHead !== '' ? ['```', ctx.stdoutHead, '```'] : []),
+        `- stderr (head): ${ctx.stderrHead === '' ? '(empty)' : ''}`,
+        ...(ctx.stderrHead !== '' ? ['```', ctx.stderrHead, '```'] : []),
+        '',
+        '## Recent runs (most recent first, this failure not yet recorded)',
+        '',
+        ...(priorRuns.length > 0 ? priorRuns : ['(no prior runs recorded)']),
+        '',
+        '## Your mandate',
+        '',
+        `Diagnose why it failed and decide what to do:`,
+        `- Transient or fixable: fix the environment/command context, verify with \`crtr cron run ${c.cron_id}\` (fires now, out of band, shows you the result), then \`crtr cron resume ${c.cron_id}\` to re-arm it.`,
+        `- Obsolete: \`crtr cron cancel ${c.cron_id}\`.`,
+        `- The user's call (or the fix needs a change you cannot make): \`crtr human ask\`.`,
+        '',
+        'The cron stays paused until someone resumes it — nothing refires while you work.',
+    ].join('\n');
+}
+/** Pause the cron and spawn the escalation node in the cron's originating
+ *  profile (root fallback). Returns the run-log `delivered` string. Never
+ *  throws: an escalation spawn failure is recorded and logged, the pause
+ *  stands (the row must not keep refiring a broken command). */
+async function escalateFailure(c, ctx, deliveryFailure) {
+    setCronState(c.cron_id, 'paused');
+    const creatorAlive = c.created_by != null && getNode(c.created_by) !== null;
+    try {
+        const res = await spawnChild({
+            kind: 'general',
+            cwd: c.cwd,
+            prompt: buildEscalationKickoff(c, ctx, deliveryFailure),
+            // The spec's word is TERMINAL: a node that deals with this failure and
+            // reaps. With the creator gone there is nobody to parent it, but
+            // parentless must not mean resident-and-forefront — see deliverSpawnSink.
+            ...(creatorAlive ? { parent: c.created_by } : { root: true, rootLifecycle: 'terminal' }),
+            readiness: 'launch',
+            profile: resolveLiveProfile(c),
+            wakeOrigin: cronWakeOrigin(c, 'node_birth'),
+        });
+        emitEvent({
+            level: 'info',
+            event: 'cron.failure.escalated',
+            fields: { cron_id: c.cron_id, name: c.name, escalation_node: res.node.node_id },
+        });
+        return `failure: cron paused, escalation node ${res.node.node_id} spawned`;
+    }
+    catch (err) {
+        emitEvent({
+            level: 'error',
+            event: 'cron.escalation.spawn_failed',
+            error: err,
+            fields: { cron_id: c.cron_id, name: c.name },
+        });
+        return `failure: cron paused; escalation spawn failed: ${err.message}`;
+    }
+}
+/** Apply the cron's output disposition to one settled run: hash bookkeeping,
+ *  sink delivery, and the failure path. `outOfBand` runs deliver but never
+ *  pause/escalate — the invoker sees the result directly. */
+async function disposeSettledRun(c, ctx, outOfBand) {
+    const prefix = outOfBand ? 'out-of-band: ' : '';
+    // Re-read the row: last_output_hash/state must be current, and a cron
+    // canceled mid-run must not deliver (its record no-ops anyway).
+    const fresh = getCron(c.cron_id);
+    if (fresh === null)
+        return { delivered: `${prefix}cron canceled mid-run — nothing delivered`, paused: false };
+    const failed = ctx.timedOut || ctx.processErrorMessage !== null || ctx.exitCode !== 0;
+    // on-change bookkeeping happens for every SCHEDULED settled run (delivered or
+    // not, failed or not) so the next comparison is against the latest
+    // observation. An out-of-band run COMPARES but never WRITES: `cron run` is a
+    // verification, and a baseline it rewrote would make the next scheduled run
+    // report "unchanged" and skip a delivery the user should have received.
+    let changed = true;
+    if (fresh.on_output === 'on-change') {
+        changed = fresh.last_output_hash !== ctx.stdoutSha256;
+        if (!outOfBand)
+            setCronLastOutputHash(c.cron_id, ctx.stdoutSha256);
+    }
+    if (failed) {
+        if (fresh.on_output === 'silent')
+            return { delivered: `${prefix}silent`, paused: false };
+        if (outOfBand)
+            return { delivered: `out-of-band: ${failureSummary(ctx, null)} — not escalated`, paused: false };
+        return { delivered: await escalateFailure(fresh, ctx, null), paused: true };
+    }
+    switch (fresh.on_output) {
+        case 'silent':
+            return { delivered: `${prefix}silent`, paused: false };
+        case 'on-failure':
+            return { delivered: `${prefix}ok — nothing to deliver (on-failure)`, paused: false };
+        case 'on-change':
+            if (!changed)
+                return { delivered: `${prefix}unchanged — not delivered (on-change)`, paused: false };
+            return deliverOrEscalate(fresh, ctx, outOfBand, prefix);
+        case 'always':
+            return deliverOrEscalate(fresh, ctx, outOfBand, prefix);
+    }
+}
+/** The shared delivering tail of `always` and a CHANGED `on-change`: push
+ *  stdout to the armed sink, or take the failure path when the sink is
+ *  undeliverable. (The two cases used to share a commented, lint-suppressed
+ *  switch fallthrough.) */
+async function deliverOrEscalate(c, ctx, outOfBand, prefix) {
+    if (ctx.stdoutHead.trim() === '') {
+        return { delivered: `${prefix}empty stdout — nothing delivered`, paused: false };
+    }
+    try {
+        return { delivered: `${prefix}${await deliverToSink(c, ctx.stdoutHead)}`, paused: false };
+    }
+    catch (err) {
+        const why = `delivery failed: ${err.message}`;
+        if (outOfBand)
+            return { delivered: `out-of-band: ${why} — not escalated`, paused: false };
+        return { delivered: await escalateFailure(c, ctx, why), paused: true };
+    }
+}
+// ---------------------------------------------------------------------------
+// Stale-lease recovery
+// ---------------------------------------------------------------------------
+/** Recover stale 'running' rows left by a PRIOR daemon instance: a pid that is
+ *  not VERIFIABLY still the process this lease launched resets straight to
+ *  idle; a confirmed-live process group past its own run_timeout_s is killed
+ *  and settled as a timeout run through the SAME failure path a live timeout
+ *  takes (pause + escalation unless silent). Rows this SAME instance owns are
+ *  genuinely in flight and left alone.
+ *
+ *  The verification is `verifyLeasedCronRun`, not `isPidAlive`, and it is
+ *  FAIL-CLOSED: this row was written by a daemon life that has since ended, so
+ *  after a host reboot its `run_pid` names a slot in a dead kernel's pid space
+ *  that the OS has probably recycled, and the timeout comparison below is
+ *  trivially satisfied by any downtime. Liveness alone would therefore SIGTERM
+ *  a stranger's process group. Only a matching launch-time identity authorizes
+ *  the signal; everything else releases the lease. */
+export function recoverStaleCronLeases(now, options = {}) {
+    for (const c of runningCrons()) {
+        if (c.run_lease_owner === cronDaemonInstanceId)
+            continue;
+        if (!verifyLeasedCronRun(c)) {
+            settleCrashedRun(c, now, options);
+            continue;
+        }
+        const startedAtMs = c.run_started_at != null ? new Date(c.run_started_at).getTime() : now;
+        if (now - startedAtMs < c.run_timeout_s * 1000)
+            continue; // within its own window
+        if (!killLeasedCronRun(c)) {
+            // The process exited (or was re-identified) between the two probes: there
+            // is nothing to signal, so settle it as the unobserved run it now is.
+            settleCrashedRun(c, now, options);
+            continue;
+        }
+        const ctx = {
+            runId: randomUUID(),
+            startedAtIso: c.run_started_at ?? new Date(now).toISOString(),
+            exitCode: -1,
+            timedOut: true,
+            runTimeoutS: c.run_timeout_s,
+            processErrorMessage: null,
+            stdoutHead: '',
+            stdoutSha256: EMPTY_STDOUT_SHA256,
+            stderrHead: `run timed out after ${c.run_timeout_s}s (recovered stale lease from a prior daemon instance)`,
+        };
+        const settlement = (async () => {
+            await awaitLeasedRunDeath(c);
+            const disposition = await disposeSettledRun(c, ctx, false);
+            recordCronRun({
+                run_id: ctx.runId,
+                cron_id: c.cron_id,
+                started: ctx.startedAtIso,
+                finished: new Date(now).toISOString(),
+                duration_ms: now - startedAtMs,
+                exit_code: -1,
+                stdout_head: null,
+                stderr_head: ctx.stderrHead,
+                delivered: disposition.delivered,
+            });
+            releaseCronRunLease(c.cron_id);
+            if (c.recur == null && !disposition.paused)
+                consumeCron(c.cron_id);
+        })();
+        options.register?.(settlement);
+        void settlement.catch((err) => {
+            emitEvent({ level: 'error', event: 'cron.stale_lease.settle_failed', error: err, fields: { cron_id: c.cron_id } });
+        });
+    }
+}
+/** Wait for a SIGTERMed stale-lease run to actually die, escalating to SIGKILL
+ *  if it does not — the same bound `killWithEscalation` gives a live timeout.
+ *  Recovery used to record the run as finished the instant it signaled, while a
+ *  signal-trapping command was still running and still producing side effects. */
+async function awaitLeasedRunDeath(c) {
+    const deadline = Date.now() + CRON_KILL_GRACE_MS;
+    while (Date.now() < deadline) {
+        await delay(CRON_DEATH_POLL_MS);
+        if (!verifyLeasedCronRun(c))
+            return; // gone
+    }
+    killLeasedCronRun(c, 'SIGKILL');
+}
+/** Settle a stale lease whose process can no longer be observed: the daemon
+ *  life that owned this run ended (SIGKILL, host reboot) and nothing recorded
+ *  the outcome. Releasing the lease — all this used to do — is wrong twice.
+ *
+ *  1. The run must leave a RECORD. The run log's whole justification is "it
+ *     silently stopped working Tuesday"; a hole exactly where the crash
+ *     happened is the one gap it cannot afford.
+ *  2. A ONE-SHOT must not RE-FIRE. A recurrence advanced `fire_at` before its
+ *     run, so releasing it to idle costs at most a missed slot. A one-shot's
+ *     `fire_at` is still past-due and it is deleted only after a settle that
+ *     never came — so releasing it re-runs a command whose real-world effect
+ *     (post the report, send the money, open the PR) may already have completed
+ *     in full. It is PAUSED instead (the same dedup the failure path uses: a
+ *     paused row is never due) and escalated, so a node decides whether the
+ *     effect landed — `cron run` to redo it, `cron cancel` to drop it.
+ *
+ *  A `silent` cron pauses without an escalation node: silent means record only,
+ *  never deliver, never escalate. */
+function settleCrashedRun(c, now, options) {
+    const oneShot = c.recur == null;
+    const startedAtMs = c.run_started_at != null ? new Date(c.run_started_at).getTime() : now;
+    const why = 'the daemon life that owned this run ended before it settled — the process is gone and its outcome was never observed';
+    const ctx = {
+        runId: randomUUID(),
+        startedAtIso: c.run_started_at ?? new Date(now).toISOString(),
+        exitCode: null,
+        timedOut: false,
+        runTimeoutS: c.run_timeout_s,
+        processErrorMessage: why,
+        stdoutHead: '',
+        stdoutSha256: EMPTY_STDOUT_SHA256,
+        stderrHead: `run outcome unobserved: ${why}`,
+    };
+    const settlement = (async () => {
+        let delivered = `crashed: ${why}${oneShot ? '' : ' — the next occurrence is unaffected'}`;
+        if (oneShot) {
+            // Stop the re-fire FIRST, so even a throwing escalation cannot leave the
+            // row active with a past-due fire_at. (escalateFailure pauses too; the
+            // second write is a harmless no-op.)
+            setCronState(c.cron_id, 'paused');
+            delivered =
+                c.on_output === 'silent'
+                    ? `crashed: ${why} — one-shot paused, not re-fired (silent: not escalated)`
+                    : `crashed one-shot: ${await escalateFailure(c, ctx, crashedOneShotSummary(why))}`;
+        }
+        recordCronRun({
+            run_id: ctx.runId,
+            cron_id: c.cron_id,
+            started: ctx.startedAtIso,
+            finished: new Date(now).toISOString(),
+            duration_ms: Math.max(0, now - startedAtMs),
+            exit_code: null,
+            stdout_head: null,
+            stderr_head: ctx.stderrHead,
+            delivered,
+        });
+        releaseCronRunLease(c.cron_id);
+        emitEvent({
+            level: 'warn',
+            event: 'cron.run.unobserved',
+            fields: { cron_id: c.cron_id, name: c.name, one_shot: oneShot },
+        });
+    })();
+    options.register?.(settlement);
+    void settlement.catch((err) => {
+        emitEvent({ level: 'error', event: 'cron.crashed_run.settle_failed', error: err, fields: { cron_id: c.cron_id } });
+    });
+}
+function crashedOneShotSummary(why) {
+    return [
+        `${why}.`,
+        'This is a ONE-SHOT, so it has been PAUSED rather than released: its side effects may have completed in full,',
+        'and re-firing would enact them a second time. Determine whether the work actually landed, then either',
+        '`crtr cron run` it to redo it deliberately, or `crtr cron cancel` it.',
+    ].join(' ');
+}
+// ---------------------------------------------------------------------------
+// The run executor
+// ---------------------------------------------------------------------------
+/** Kick off ONE cron run and return its settlement promise, resolving with
+ *  the settled run record (null = the idle→running lease was lost, or the row
+ *  was quarantined — nothing was recorded by THIS call). The subprocess
+ *  starts synchronously (spawn + lease + recurrence advance happen before
+ *  this returns); the promise resolves only after the child has CLOSED and
+ *  every durable transition — disposition/delivery, run record, lease
+ *  release, one-shot consumption — is written.
+ *
+ *  `outOfBand` (the `cron run` verb) takes the same lease but does NOT
+ *  advance a recurrence, does NOT consume a one-shot, and never
+ *  escalates/pauses — it delivers per disposition and hands the record back
+ *  to the invoker. */
+export function executeCron(c, opts) {
+    const { outOfBand } = opts;
+    return new Promise((resolve, reject) => {
+        let extraEnv = {};
+        if (c.env_json != null) {
+            try {
+                const parsed = JSON.parse(c.env_json);
+                for (const [k, v] of Object.entries(parsed)) {
+                    if (typeof v === 'string')
+                        extraEnv[k] = v;
+                }
+            }
+            catch {
+                extraEnv = {}; // a malformed snapshot never blocks the run itself
+            }
+        }
+        const runId = randomUUID();
+        // The cron RUNS UNDER THE PROFILE IT WAS ARMED UNDER (re-validated here, root
+        // fallback when the manifest is gone). The spec's central promise is that
+        // "messaging a node, reviving one, birthing one is `crtr node …` inside the
+        // command" — and every one of those resolves against a profile. Passing the
+        // profile to buildOperationalEnvBase only steers `spawnEnv.allow` config
+        // resolution; the child also needs CRTR_PROFILE_ID itself (not an
+        // operational literal name, so it never crosses on its own) and the
+        // profile's own env store, exactly as buildBrokerEnv gives a broker — storing
+        // a value there IS the operator's consent for it to cross this boundary.
+        // Layering: profile store < the arm-time `--env` snapshot (an explicit
+        // choice) < crtr's own constructed CRTR_* names, which always win.
+        const profileId = resolveLiveProfile(c);
+        const env = {
+            ...buildOperationalEnvBase({ targetCwd: c.cwd, targetProfileId: profileId }),
+            ...readProfileEnvVars(profileId),
+            ...extraEnv,
+            ...(profileId !== null ? { CRTR_PROFILE_ID: profileId } : {}),
+            CRTR_CRON_ID: c.cron_id,
+            CRTR_CRON_NAME: c.name,
+            CRTR_RUN_ID: runId,
+        };
+        // An explicit canvas home is part of the runtime's own identity, not host
+        // state a `spawnEnv.allow` list opts into: a cron command's whole promise
+        // is that `crtr node …` inside it talks to THIS daemon. Without it the
+        // child crtr resolves the default `~/.crouter/canvas`, so every crtr call
+        // in a cron body fails to reach the daemon that spawned it. Same
+        // propagation the broker/child-node boundary does (`core/runtime/nodes.ts`).
+        const canvasHomeOverride = process.env['CRTR_HOME'];
+        if (canvasHomeOverride !== undefined && canvasHomeOverride !== '') {
+            env['CRTR_HOME'] = canvasHomeOverride;
+        }
+        if (c.anchor_node != null)
+            env['CRTR_ANCHOR'] = c.anchor_node;
+        const startedAtMs = Date.now();
+        const startedAtIso = new Date(startedAtMs).toISOString();
+        let child;
+        try {
+            child = spawn('bash', ['-c', c.command], {
+                cwd: c.cwd,
+                env,
+                detached: true, // own process group — killProcessGroup(-pid) reaps the whole tree
+                stdio: ['ignore', 'pipe', 'pipe'],
+            });
+        }
+        catch (err) {
+            // Synchronous spawn failure — no child exists (and no lease was taken).
+            // Settle through the failure path so a broken cron pauses + escalates
+            // instead of refiring every tick.
+            const ctx = {
+                runId,
+                startedAtIso,
+                exitCode: null,
+                timedOut: false,
+                runTimeoutS: c.run_timeout_s,
+                processErrorMessage: err.message,
+                stdoutHead: '',
+                stdoutSha256: EMPTY_STDOUT_SHA256,
+                stderrHead: `run failed to spawn: ${err.message}`.slice(0, CRON_OUTPUT_HEAD_BYTES),
+            };
+            void (async () => {
+                const disposition = await disposeSettledRun(c, ctx, outOfBand);
+                const record = {
+                    run_id: runId,
+                    cron_id: c.cron_id,
+                    started: startedAtIso,
+                    finished: new Date().toISOString(),
+                    duration_ms: Date.now() - startedAtMs,
+                    exit_code: null,
+                    stdout_head: null,
+                    stderr_head: ctx.stderrHead,
+                    delivered: disposition.delivered,
+                };
+                recordCronRun(record);
+                if (!outOfBand && c.recur == null && !disposition.paused)
+                    consumeCron(c.cron_id);
+                resolve(record);
+            })().catch(reject);
+            return;
+        }
+        let settled = false;
+        let timer;
+        let killTimer;
+        let timedOut = false;
+        let leaseLost = false;
+        let processError = null;
+        // Output capture. The head is accumulated as raw BYTES and decoded ONCE at
+        // close: decoding per chunk and slicing by `String.length` capped in UTF-16
+        // code units despite the `_BYTES` name, and split any multi-byte sequence
+        // straddling a chunk boundary into replacement characters. The change-
+        // detection hash runs over the FULL stdout stream rather than the head, so
+        // output differing only past the cap is still seen to have changed (it used
+        // to report "unchanged" forever).
+        const stdout = newHeadCapture();
+        const stderr = newHeadCapture();
+        const stdoutHash = createHash('sha256');
+        child.stdout?.on('data', (chunk) => {
+            stdoutHash.update(chunk);
+            stdout.take(chunk);
+        });
+        child.stderr?.on('data', (chunk) => {
+            stderr.take(chunk);
+        });
+        const killWithEscalation = (pid) => {
+            killProcessGroup(pid);
+            if (killTimer === undefined) {
+                killTimer = setTimeout(() => killProcessGroup(pid, 'SIGKILL'), CRON_KILL_GRACE_MS);
+                killTimer.unref?.();
+            }
+        };
+        // A signal, not settlement: 'close' always follows 'error'.
+        child.on('error', (err) => {
+            if (processError == null)
+                processError = err;
+        });
+        child.on('close', (code) => {
+            if (settled)
+                return;
+            settled = true;
+            if (timer !== undefined)
+                clearTimeout(timer);
+            if (killTimer !== undefined)
+                clearTimeout(killTimer);
+            const stdoutHead = stdout.decode();
+            const stderrHead = stderr.decode();
+            const live = liveRuns.get(c.cron_id);
+            const wasReplaced = live !== undefined && live.runId === runId && live.replaced;
+            if (live !== undefined && live.runId === runId)
+                liveRuns.delete(c.cron_id);
+            void (async () => {
+                if (leaseLost) {
+                    // Lost the idle→running race (explicit cancel or another daemon's
+                    // lease) — whoever won owns the row's disposition; abandon silently.
+                    resolve(null);
+                    return;
+                }
+                const finishedAtMs = Date.now();
+                let delivered;
+                let paused = false;
+                if (wasReplaced) {
+                    // overlap=replace killed this run so the next fire could start
+                    // fresh: record it, no disposition, no escalation.
+                    delivered = 'replaced (overlap=replace)';
+                }
+                else {
+                    const disposition = await disposeSettledRun(c, {
+                        runId,
+                        startedAtIso,
+                        exitCode: timedOut ? -1 : code,
+                        timedOut,
+                        runTimeoutS: c.run_timeout_s,
+                        processErrorMessage: processError != null ? processError.message : null,
+                        stdoutHead,
+                        stdoutSha256: stdoutHash.digest('hex'),
+                        stderrHead,
+                    }, outOfBand);
+                    delivered = disposition.delivered;
+                    paused = disposition.paused;
+                }
+                const record = {
+                    run_id: runId,
+                    cron_id: c.cron_id,
+                    started: startedAtIso,
+                    finished: new Date(finishedAtMs).toISOString(),
+                    duration_ms: finishedAtMs - startedAtMs,
+                    exit_code: timedOut ? -1 : code,
+                    stdout_head: stdoutHead === '' ? null : stdoutHead,
+                    stderr_head: processError != null
+                        ? `run process error: ${processError.message}`.slice(0, CRON_OUTPUT_HEAD_BYTES)
+                        : stderrHead === ''
+                            ? null
+                            : stderrHead,
+                    delivered,
+                };
+                recordCronRun(record);
+                if (!outOfBand && c.recur == null && !paused) {
+                    // One-shot: the run is done, delete the row (its history cascades
+                    // with it — matching one-shot consumption). A failing
+                    // one-shot that PAUSED for escalation is kept, so `cron resume` can
+                    // re-arm its still-due fire_at. A racing explicit cancel already
+                    // removed it; consumeCron no-ops.
+                    consumeCron(c.cron_id);
+                }
+                else {
+                    releaseCronRunLease(c.cron_id);
+                }
+                resolve(record);
+            })().catch(reject);
+        });
+        if (child.pid == null ||
+            !acquireCronRunLease(c.cron_id, cronDaemonInstanceId, child.pid, startedAtIso, outOfBand)) {
+            if (child.pid != null)
+                killWithEscalation(child.pid);
+            leaseLost = true;
+            return; // 'close' above performs the actual settlement.
+        }
+        liveRuns.set(c.cron_id, {
+            runId,
+            replaced: false,
+            kill: () => {
+                if (child.pid != null)
+                    killWithEscalation(child.pid);
+            },
+        });
+        // Settle the schedule BEFORE the run: a recurrence advances to its next
+        // slot now, so a crash mid-run never re-fires this slot (the lease guards
+        // the intra-life overlap; the advanced fire_at guards the crash window).
+        // An out-of-band run never touches the schedule.
+        if (!outOfBand && c.recur != null) {
+            let nextFire;
+            try {
+                nextFire = nextSlotAfter(c.recur, new Date(startedAtMs));
+            }
+            catch (err) {
+                // Unparseable recurrence. This row can never be scheduled again as
+                // written — but DELETING it (what this used to do) destroys the user's
+                // config AND its whole run history with no record and no escalation, the
+                // one outcome the spec has no vocabulary for. Take the failure path
+                // instead: kill the just-started run and hand `close` a process error, so
+                // the row PAUSES and an escalation node arrives holding the parse error.
+                // `fire_at` is deliberately left past-due: a paused row is not due, and
+                // once someone fixes the recurrence a `cron resume` fires immediately.
+                emitEvent({
+                    level: 'error',
+                    event: 'cron.recur.unparseable',
+                    error: err,
+                    fields: { cron_id: c.cron_id, name: c.name, recur: c.recur },
+                });
+                processError = new Error(`unparseable recurrence ${JSON.stringify(c.recur)}: ${err.message} — this cron cannot be rescheduled until its recurrence is fixed (\`crtr cron cancel\` and re-arm it), so it has been paused.`);
+                // Pause HERE, not only via the escalation: `fire_at` was never advanced,
+                // so an on_output='silent' row (whose failure path never escalates and
+                // never pauses) would otherwise re-fire this unschedulable command on
+                // every single tick, forever.
+                setCronState(c.cron_id, 'paused');
+                killWithEscalation(child.pid);
+                return; // 'close' settles it through the ordinary failure path.
+            }
+            advanceCron(c.cron_id, nextFire);
+        }
+        timer = setTimeout(() => {
+            if (settled)
+                return;
+            timedOut = true;
+            killWithEscalation(child.pid);
+            // Signal only — 'close' above performs the actual settlement.
+        }, c.run_timeout_s * 1000);
+        timer.unref?.();
+    });
+}
+// ---------------------------------------------------------------------------
+// The per-tick passes: expiry, overlap policy, due runs
+// ---------------------------------------------------------------------------
+/** Delete idle crons whose clock bound has passed. A running expired row is
+ *  left for a later tick (its run settles first). */
+function expireDueCrons(nowIso) {
+    for (const c of expiredIdleCrons(nowIso)) {
+        cancelCron(c.cron_id);
+        emitEvent({
+            level: 'info',
+            event: 'cron.expired',
+            fields: { cron_id: c.cron_id, name: c.name, expires_at: c.expires_at },
+        });
+    }
+}
+/** Apply each recurring cron's overlap policy to a next fire that has landed
+ *  while its previous run is still leased:
+ *    skip    — advance fire_at past the missed slot and record a skip entry
+ *              (the observable evidence a fire was deliberately not started).
+ *    queue   — do nothing: fire_at stays past-due, so the moment the running
+ *              lease releases, the next tick fires immediately. Multiple
+ *              missed fires collapse structurally (fire_at holds ONE
+ *              occurrence) — the spec's depth cap of 1.
+ *    replace — mark THIS instance's in-flight run replaced and kill it; its
+ *              close handler records the replacement (no escalation) and
+ *              releases the lease, and the still-due fire_at starts fresh on
+ *              the next tick. A stale other-instance lease is left to
+ *              recoverStaleCronLeases. */
+/** True when this cron's most recent run record is already a skip marker — the
+ *  streak guard in the overlap=skip pass. */
+function latestRunIsSkip(cronId) {
+    return listCronRuns(cronId)[0]?.delivered?.startsWith(SKIP_MARKER) === true;
+}
+function applyOverlapPolicies(now, nowIso) {
+    for (const c of dueOverlapCrons(nowIso)) {
+        if (c.overlap === 'queue')
+            continue;
+        if (c.overlap === 'skip') {
+            let nextFire;
+            try {
+                nextFire = nextSlotAfter(c.recur, new Date(now));
+            }
+            catch {
+                continue; // paused + escalated by the run path when it next fires
+            }
+            advanceCron(c.cron_id, nextFire);
+            // Record only the FIRST skip of a streak. A run that overruns its cadence
+            // produces one skipped occurrence per period, and recorded
+            // unconditionally a 25-minute run on a 1-minute cadence writes 25 skip
+            // markers — evicting every real run record from the 20-entry ring, which
+            // is the exact loss the ring exists to prevent. The next settled run ends
+            // the streak, so the next overlap after it records again.
+            if (!latestRunIsSkip(c.cron_id)) {
+                recordCronRun({
+                    run_id: randomUUID(),
+                    cron_id: c.cron_id,
+                    started: nowIso,
+                    finished: nowIso,
+                    duration_ms: 0,
+                    exit_code: null,
+                    stdout_head: null,
+                    stderr_head: null,
+                    delivered: `${SKIP_MARKER}: fire landed while the previous run was in flight (consecutive skips are not re-recorded until a run settles)`,
+                });
+            }
+            continue;
+        }
+        // replace
+        const live = liveRuns.get(c.cron_id);
+        if (c.run_lease_owner === cronDaemonInstanceId && live !== undefined && !live.replaced) {
+            live.replaced = true;
+            live.kill();
+        }
+    }
+}
+/** The daemon's cron pass: expire clock-bounded rows, apply overlap policies,
+ *  then kick off every due active idle cron. Each child starts synchronously
+ *  but is NEVER awaited in the tick; returns one settlement promise per
+ *  admitted run.
+ *
+ *  A run that would launch or revive a node (`cronLaunchesNode`) is admitted
+ *  only while the fleet is under its automatic-revive cap. An unadmitted run is
+ *  DEFERRED, never consumed: nothing is recorded, no recurrence advances, no
+ *  one-shot is deleted, and no failure escalates — the row keeps its past-due
+ *  `fire_at` and is retried on the next poll once capacity frees up. */
+export function runDueCrons(now, options = {}) {
+    const nowIso = new Date(now).toISOString();
+    expireDueCrons(nowIso);
+    applyOverlapPolicies(now, nowIso);
+    const settlements = [];
+    let deferred = 0;
+    for (const c of dueClockCrons(nowIso)) {
+        if (options.hasBrokerCapacity !== undefined && cronLaunchesNode(c) && !options.hasBrokerCapacity()) {
+            deferred += 1;
+            continue; // still due — retried next poll, at no cost to the row
+        }
+        const settlement = executeCron(c, { outOfBand: false }).then(() => undefined);
+        settlements.push(settlement);
+        options.register?.(settlement);
+    }
+    // Backstop: even an unregistered settlement's rejection must never surface
+    // as a process-level unhandled rejection.
+    for (const s of settlements)
+        void s.catch(() => { });
+    if (deferred > 0)
+        emitCapacityDeferral(now, deferred);
+    return settlements;
+}
+/** Throttled record of capacity-deferred runs. Unthrottled this would fire on
+ *  every 2s poll for as long as the fleet stays at its cap — the same flood the
+ *  daemon's own capacity event guards against. */
+const CAPACITY_DEFER_LOG_THROTTLE_MS = 30_000;
+let lastCapacityDeferLogAt = Number.NEGATIVE_INFINITY;
+function emitCapacityDeferral(now, deferred) {
+    if (now - lastCapacityDeferLogAt < CAPACITY_DEFER_LOG_THROTTLE_MS)
+        return;
+    lastCapacityDeferLogAt = now;
+    emitEvent({
+        level: 'warn',
+        event: 'cron.capacity.run_deferred',
+        fields: { deferred_runs: deferred },
+    });
+}