@north-light/crouter 0.3.232 → 0.3.234

package/dist/daemon/boot-migrations.js

@@ -0,0 +1,31 @@
+// The ordered boot-migration list — the THIRD of this repo's three migration
+// homes, and the one for filesystem repairs that must run unconditionally at
+// every daemon boot, before supervision can revive anything. No version gate
+// (there is no single database to hold a `user_version`, and no `crtr sys
+// migrate` invocation to drive it): each entry converges on its own, the same
+// way `migrateLegacyInboxIdentities` already does — a re-run over
+// already-migrated state rewrites nothing.
+//
+// The other two homes:
+//   - `../core/canvas/migrations.ts` — canvas.db's versioned schema chain,
+//     gated by `PRAGMA user_version`, applied once per database inside
+//     `migrate()`.
+//   - `../migrations/registry.ts` — the `STATE_MIGRATIONS` registry for
+//     on-disk memory documents, profile manifests, and the corpus lane,
+//     driven by `crtr sys migrate`.
+//
+// Authoring contract: one entry per boot-time filesystem repair, appended
+// here in order. `crtrd.ts` iterates this list once at boot and emits one
+// `daemon.boot_migration.applied` event per entry that reports a change —
+// add a step here instead of a bespoke call in the boot sequence.
+import { migrateLegacyInboxIdentities } from '../core/feed/inbox-migrate.js';
+export const BOOT_MIGRATIONS = [
+    {
+        id: 'legacy-inbox-identities',
+        description: 'normalize pre-cutover inbox entry/operation ids (UUID-spelled or missing) and cursors (ISO-timestamp boundaries) into canonical 32-lowercase-hex identities the strict inbox reader requires.',
+        apply: () => {
+            const summary = migrateLegacyInboxIdentities();
+            return { changed: summary.nodes > 0 || summary.failures > 0, summary: { ...summary } };
+        },
+    },
+];

package/dist/daemon/cron/armed-context.d.ts

@@ -0,0 +1,21 @@
+import type { Cron } from '../../core/canvas/crons.js';
+/** The cron's profile, re-validated now and returned in canonical id form.
+ *
+ *  A DELETED profile is the spec's root fallback: null. An UNREADABLE one is
+ *  not — the profile is still there, so running as root would execute the
+ *  command under an identity nobody armed, with the profile's own env store and
+ *  purview silently replaced. That throws, and the run fails loudly once
+ *  instead of running wrong on every tick. */
+export declare function resolveLiveProfile(c: Cron): string | null;
+export declare function resolveLiveProfileOrRoot(c: Cron): string | null;
+export declare function resolveLiveProfileName(c: Cron): string | undefined;
+/** A stored context cannot be rebuilt; the run fails rather than running under a context nobody armed. */
+export declare class ArmedContextError extends Error {
+    constructor(message: string);
+}
+export interface ArmedRunContext {
+    profileId: string | null;
+    env: NodeJS.ProcessEnv;
+}
+/** Rebuild the armed context. Corrupt snapshots and unreadable profiles fail rather than silently changing its identity or environment. */
+export declare function buildArmedRunContext(c: Cron, runId: string): ArmedRunContext;

package/dist/daemon/cron/armed-context.js

@@ -0,0 +1,100 @@
+import { envHomeOverride } from '../../shared/env.js';
+import { loadProfileManifest, lookupProfile } from '../../core/profiles/manifest.js';
+import { readProfileEnvVars } from '../../core/profiles/env-store.js';
+import { ADMITTED_HOST_PATH_ENV, buildOperationalEnvBase } from '../../core/runtime/spawn-env.js';
+/** The cron's profile, re-validated now and returned in canonical id form.
+ *
+ *  A DELETED profile is the spec's root fallback: null. An UNREADABLE one is
+ *  not — the profile is still there, so running as root would execute the
+ *  command under an identity nobody armed, with the profile's own env store and
+ *  purview silently replaced. That throws, and the run fails loudly once
+ *  instead of running wrong on every tick. */
+export function resolveLiveProfile(c) {
+    if (c.profile == null)
+        return null;
+    let lookup;
+    try {
+        lookup = lookupProfile(c.profile);
+    }
+    catch {
+        return null; // an ambiguous NAME resolves no better than a deleted profile
+    }
+    if (lookup.kind === 'resolved')
+        return lookup.entry.profileId;
+    if (lookup.kind === 'absent')
+        return null;
+    throw new ArmedContextError(`the armed profile "${c.profile}" is still on this host but its manifest could not be read (${lookup.manifestPath}) — running as root would silently give this cron a different identity, env store, and purview. Repair the manifest, or re-arm the cron under a profile that resolves.`);
+}
+// The escalation node must still report an unreadable manifest, so its own spawn falls back to root.
+export function resolveLiveProfileOrRoot(c) {
+    try {
+        return resolveLiveProfile(c);
+    }
+    catch {
+        return null;
+    }
+}
+export function resolveLiveProfileName(c) {
+    if (c.profile == null)
+        return undefined;
+    try {
+        return loadProfileManifest(c.profile).manifest.name;
+    }
+    catch {
+        return undefined;
+    }
+}
+/** A stored context cannot be rebuilt; the run fails rather than running under a context nobody armed. */
+export class ArmedContextError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = 'ArmedContextError';
+    }
+}
+// Invalid snapshots are corruption: dropping `--env` would run under an environment nobody armed, so fail instead.
+function parseArmedEnv(envJson) {
+    if (envJson == null)
+        return {};
+    let parsed;
+    try {
+        parsed = JSON.parse(envJson);
+    }
+    catch (err) {
+        throw new ArmedContextError(`the armed env snapshot is not valid JSON (${err.message}) — this cron's stored env_json is corrupt, so the run would silently drop every armed variable. Re-arm the cron with a valid --env.`);
+    }
+    if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) {
+        throw new ArmedContextError("the armed env snapshot is not a JSON object — this cron's stored env_json is corrupt. Re-arm the cron with a valid --env.");
+    }
+    const extraEnv = {};
+    for (const [k, v] of Object.entries(parsed)) {
+        if (typeof v !== 'string') {
+            throw new ArmedContextError(`the armed env snapshot's value for "${k}" is not a string — this cron's stored env_json is corrupt, so the run would silently drop that variable. Re-arm the cron with a valid --env.`);
+        }
+        extraEnv[k] = v;
+    }
+    return extraEnv;
+}
+/** Rebuild the armed context. Corrupt snapshots and unreadable profiles fail rather than silently changing its identity or environment. */
+export function buildArmedRunContext(c, runId) {
+    const extraEnv = parseArmedEnv(c.env_json);
+    const profileId = resolveLiveProfile(c);
+    const operationalEnv = buildOperationalEnvBase({ targetCwd: c.cwd, targetProfileId: profileId });
+    const env = {
+        ...operationalEnv,
+        ...readProfileEnvVars(profileId),
+        ...extraEnv,
+        ...(profileId !== null ? { CRTR_PROFILE_ID: profileId } : {}),
+        CRTR_CRON_ID: c.cron_id,
+        CRTR_CRON_NAME: c.name,
+        CRTR_RUN_ID: runId,
+    };
+    // Preserve the daemon identity: otherwise child `crtr` resolves the default canvas.
+    const canvasHomeOverride = envHomeOverride();
+    if (canvasHomeOverride !== undefined && canvasHomeOverride !== '') {
+        env['CRTR_HOME'] = canvasHomeOverride;
+    }
+    if (c.anchor_node != null)
+        env['CRTR_ANCHOR'] = c.anchor_node;
+    env[ADMITTED_HOST_PATH_ENV] = operationalEnv[ADMITTED_HOST_PATH_ENV];
+    return { profileId, env };
+}

package/dist/daemon/cron/capture.d.ts

@@ -0,0 +1,20 @@
+/** Output head caps for the run log (spec: "capped, e.g. 4KB"). */
+export declare const CRON_OUTPUT_HEAD_BYTES: number;
+/** Bound between the run-timeout SIGTERM and a forced SIGKILL escalation, so
+ *  a signal-trapping command cannot stall teardown's detached drain. */
+export declare const CRON_KILL_GRACE_MS = 2000;
+/** Poll interval while waiting for a SIGTERMed stale-lease run to actually die
+ *  (each probe is a `ps` call, so this is deliberately coarse). */
+export declare const CRON_DEATH_POLL_MS = 250;
+/** sha256 of empty stdout — the change-detection hash a run that produced no
+ *  output carries (a failed spawn, a recovered stale lease). */
+export declare const EMPTY_STDOUT_SHA256: string;
+/** A byte-capped capture of one output stream's head. Bytes are buffered and
+ *  decoded exactly once, at the end — decoding per chunk both mis-measured the
+ *  cap (UTF-16 code units, not bytes) and corrupted any multi-byte sequence
+ *  that straddled a chunk boundary. Truncation is MARKED, never silent: a
+ *  clipped body reads as complete output to whoever receives the delivery. */
+export declare function newHeadCapture(): {
+    take(chunk: Buffer): void;
+    decode(): string;
+};

package/dist/daemon/cron/capture.js

@@ -0,0 +1,45 @@
+// src/daemon/cron/capture.ts — output capture for a cron run's child process,
+// plus the timing/hash constants the capture and kill mechanics share.
+import { createHash } from 'node:crypto';
+/** Output head caps for the run log (spec: "capped, e.g. 4KB"). */
+export const CRON_OUTPUT_HEAD_BYTES = 4 * 1024;
+/** Bound between the run-timeout SIGTERM and a forced SIGKILL escalation, so
+ *  a signal-trapping command cannot stall teardown's detached drain. */
+export const CRON_KILL_GRACE_MS = 2000;
+/** Poll interval while waiting for a SIGTERMed stale-lease run to actually die
+ *  (each probe is a `ps` call, so this is deliberately coarse). */
+export const CRON_DEATH_POLL_MS = 250;
+/** sha256 of empty stdout — the change-detection hash a run that produced no
+ *  output carries (a failed spawn, a recovered stale lease). */
+export const EMPTY_STDOUT_SHA256 = createHash('sha256').update('').digest('hex');
+/** A byte-capped capture of one output stream's head. Bytes are buffered and
+ *  decoded exactly once, at the end — decoding per chunk both mis-measured the
+ *  cap (UTF-16 code units, not bytes) and corrupted any multi-byte sequence
+ *  that straddled a chunk boundary. Truncation is MARKED, never silent: a
+ *  clipped body reads as complete output to whoever receives the delivery. */
+export function newHeadCapture() {
+    const chunks = [];
+    let bytes = 0;
+    let truncated = false;
+    return {
+        take(chunk) {
+            const room = CRON_OUTPUT_HEAD_BYTES - bytes;
+            if (room <= 0) {
+                truncated = true;
+                return;
+            }
+            if (chunk.length > room) {
+                chunks.push(chunk.subarray(0, room));
+                bytes = CRON_OUTPUT_HEAD_BYTES;
+                truncated = true;
+                return;
+            }
+            chunks.push(chunk);
+            bytes += chunk.length;
+        },
+        decode() {
+            const text = Buffer.concat(chunks).toString('utf8');
+            return truncated ? `${text}\n[… output truncated at ${CRON_OUTPUT_HEAD_BYTES} bytes]` : text;
+        },
+    };
+}

package/dist/daemon/cron/disposition.d.ts

@@ -0,0 +1,19 @@
+import { type Cron } from '../../core/canvas/crons.js';
+export interface RunSettleContext {
+    runId: string;
+    startedAtIso: string;
+    exitCode: number | null;
+    timedOut: boolean;
+    runTimeoutS: number;
+    processErrorMessage: string | null;
+    stdoutHead: string;
+    /** Hash the full output: a capped head can hide a change. */
+    stdoutSha256: string;
+    stderrHead: string;
+}
+export interface Disposition {
+    delivered: string;
+    /** A paused row is retained so `cron resume` can re-arm it. */
+    paused: boolean;
+}
+export declare function disposeSettledRun(c: Cron, ctx: RunSettleContext, outOfBand: boolean, selfCanceled?: boolean): Promise<Disposition>;

package/dist/daemon/cron/disposition.js

@@ -0,0 +1,52 @@
+import { getCron, setCronLastOutputHash } from '../../core/canvas/crons.js';
+import { escalateFailure, failureSummary } from './escalation.js';
+import { deliverToSink } from './sinks.js';
+export async function disposeSettledRun(c, ctx, outOfBand, selfCanceled = false) {
+    const prefix = outOfBand ? 'out-of-band: ' : '';
+    // A self-cancel still delivers its already-produced output; an external cancellation does not.
+    const fresh = getCron(c.cron_id);
+    if (fresh === null && !selfCanceled)
+        return { delivered: `${prefix}cron canceled mid-run — nothing delivered`, paused: false };
+    const effective = fresh ?? c;
+    const failed = ctx.timedOut || ctx.processErrorMessage !== null || ctx.exitCode !== 0;
+    // `cron run` compares but never rewrites the scheduled baseline.
+    let changed = true;
+    if (effective.on_output === 'on-change') {
+        changed = effective.last_output_hash !== ctx.stdoutSha256;
+        if (!outOfBand && fresh !== null)
+            setCronLastOutputHash(c.cron_id, ctx.stdoutSha256);
+    }
+    if (failed) {
+        if (effective.on_output === 'silent')
+            return { delivered: `${prefix}silent`, paused: false };
+        if (outOfBand)
+            return { delivered: `out-of-band: ${failureSummary(ctx, null)} — not escalated`, paused: false };
+        return { delivered: await escalateFailure(effective, ctx, null), paused: true };
+    }
+    switch (effective.on_output) {
+        case 'silent':
+            return { delivered: `${prefix}silent`, paused: false };
+        case 'on-failure':
+            return { delivered: `${prefix}ok — nothing to deliver (on-failure)`, paused: false };
+        case 'on-change':
+            if (!changed)
+                return { delivered: `${prefix}unchanged — not delivered (on-change)`, paused: false };
+            return deliverOrEscalate(effective, ctx, outOfBand, prefix);
+        case 'always':
+            return deliverOrEscalate(effective, ctx, outOfBand, prefix);
+    }
+}
+async function deliverOrEscalate(c, ctx, outOfBand, prefix) {
+    if (ctx.stdoutHead.trim() === '') {
+        return { delivered: `${prefix}empty stdout — nothing delivered`, paused: false };
+    }
+    try {
+        return { delivered: `${prefix}${await deliverToSink(c, ctx.stdoutHead)}`, paused: false };
+    }
+    catch (err) {
+        const why = `delivery failed: ${err.message}`;
+        if (outOfBand)
+            return { delivered: `out-of-band: ${why} — not escalated`, paused: false };
+        return { delivered: await escalateFailure(c, ctx, why), paused: true };
+    }
+}

package/dist/daemon/cron/escalation.d.ts

@@ -0,0 +1,12 @@
+import { type Cron } from '../../core/canvas/crons.js';
+import type { RunSettleContext } from './disposition.js';
+export declare function failureSummary(ctx: RunSettleContext, deliveryFailure: string | null): string;
+/** Pause the cron and spawn the escalation node in the cron's originating
+ *  profile (root fallback). Returns the run-log `delivered` string. Never
+ *  throws: an escalation spawn failure is recorded and logged, the pause
+ *  stands (the row must not keep refiring a broken command). */
+export declare function escalateFailure(c: Cron, ctx: RunSettleContext, deliveryFailure: string | null): Promise<string>;
+/** The delivery-failure line an escalated crashed ONE-SHOT carries: its side
+ *  effects may already have landed in full, so a human/agent decides whether to
+ *  redo it rather than the daemon re-firing it. */
+export declare function crashedOneShotSummary(why: string): string;

package/dist/daemon/cron/escalation.js

@@ -0,0 +1,124 @@
+// src/daemon/cron/escalation.ts — the cron failure path: pause the row and
+// spawn a terminal node holding everything it needs to deal with the failure.
+// Pausing IS the dedup — a paused row is never due, so nothing refires while
+// the escalation node works.
+import { getNode } from '../../core/canvas/canvas.js';
+import { fullName } from '../../core/canvas/labels.js';
+import { listCronRuns, setCronState } from '../../core/canvas/crons.js';
+import { emitEvent } from '../../core/events/emit.js';
+import { spawnChild } from '../../core/runtime/spawn.js';
+import { cronWakeOrigin } from '../../core/runtime/bearings.js';
+import { cadenceDisplay } from '../../core/wake.js';
+import { resolveLiveProfileOrRoot } from './armed-context.js';
+export function failureSummary(ctx, deliveryFailure) {
+    if (deliveryFailure !== null)
+        return deliveryFailure;
+    if (ctx.timedOut)
+        return `run timed out after ${ctx.runTimeoutS}s (killed; recorded exit -1)`;
+    if (ctx.processErrorMessage !== null)
+        return `run process error: ${ctx.processErrorMessage}`;
+    return `command exited ${ctx.exitCode ?? '(no exit observed)'}`;
+}
+/** The escalation node's kickoff: the cron's identity/config, this failure,
+ *  the run-log ring, the creating-node reference, and a mandate that names its
+ *  actual options (resume after verifying / cancel / ask the human). */
+function buildEscalationKickoff(c, ctx, deliveryFailure) {
+    const creator = c.created_by != null ? getNode(c.created_by) : null;
+    const creatorLine = c.created_by == null
+        ? 'created by: (no creating node recorded)'
+        : creator !== null
+            ? `created by: node ${c.created_by} ("${fullName(creator)}") — still on the canvas; consider telling it what happened`
+            : `created by: node ${c.created_by} (now gone)`;
+    const schedule = c.recur != null ? `every ${cadenceDisplay(c.recur)}` : `one-shot at ${c.fire_at}`;
+    const priorRuns = listCronRuns(c.cron_id)
+        .slice(0, 10)
+        .map((r) => {
+        const exit = r.exit_code === null ? 'no exit' : `exit ${r.exit_code}`;
+        const head = (r.stderr_head ?? r.stdout_head ?? '').split('\n')[0]?.slice(0, 120) ?? '';
+        return `- ${r.started} — ${exit}, ${r.duration_ms}ms — ${r.delivered ?? ''}${head !== '' ? ` — ${head}` : ''}`;
+    });
+    return [
+        `A scheduled cron FAILED and has paused itself. You were spawned to deal with it.`,
+        '',
+        `- cron: "${c.name}" (${c.cron_id})`,
+        `- schedule: ${schedule}${c.tz != null ? ` (tz ${c.tz})` : ''}`,
+        `- context: cwd ${c.cwd}${c.profile != null ? `, profile ${c.profile}` : ''}`,
+        `- ${creatorLine}`,
+        `- this failure: ${failureSummary(ctx, deliveryFailure)}`,
+        '',
+        '## The command',
+        '',
+        '```bash',
+        c.command,
+        '```',
+        '',
+        '## This run',
+        '',
+        `- exit: ${ctx.timedOut ? '-1 (timeout)' : String(ctx.exitCode)}`,
+        `- stdout (head): ${ctx.stdoutHead === '' ? '(empty)' : ''}`,
+        ...(ctx.stdoutHead !== '' ? ['```', ctx.stdoutHead, '```'] : []),
+        `- stderr (head): ${ctx.stderrHead === '' ? '(empty)' : ''}`,
+        ...(ctx.stderrHead !== '' ? ['```', ctx.stderrHead, '```'] : []),
+        '',
+        '## Recent runs (most recent first, this failure not yet recorded)',
+        '',
+        ...(priorRuns.length > 0 ? priorRuns : ['(no prior runs recorded)']),
+        '',
+        '## Your mandate',
+        '',
+        `Diagnose why it failed and decide what to do:`,
+        `- Transient or fixable: fix the environment/command context, verify with \`crtr cron run ${c.cron_id}\` (fires now, out of band, shows you the result), then \`crtr cron resume ${c.cron_id}\` to re-arm it.`,
+        `- Obsolete: \`crtr cron cancel ${c.cron_id}\`.`,
+        `- The user's call (or the fix needs a change you cannot make): \`crtr human send\`.`,
+        '',
+        'The cron stays paused until someone resumes it — nothing refires while you work.',
+    ].join('\n');
+}
+/** Pause the cron and spawn the escalation node in the cron's originating
+ *  profile (root fallback). Returns the run-log `delivered` string. Never
+ *  throws: an escalation spawn failure is recorded and logged, the pause
+ *  stands (the row must not keep refiring a broken command). */
+export async function escalateFailure(c, ctx, deliveryFailure) {
+    setCronState(c.cron_id, 'paused');
+    const creatorAlive = c.created_by != null && getNode(c.created_by) !== null;
+    try {
+        const res = await spawnChild({
+            kind: 'general',
+            cwd: c.cwd,
+            prompt: buildEscalationKickoff(c, ctx, deliveryFailure),
+            // The spec's word is TERMINAL: a node that deals with this failure and
+            // reaps. With the creator gone there is nobody to parent it, but
+            // parentless must not mean resident-and-forefront — see deliverSpawnSink.
+            ...(creatorAlive ? { parent: c.created_by } : { root: true, rootLifecycle: 'terminal' }),
+            readiness: 'launch',
+            profile: resolveLiveProfileOrRoot(c),
+            wakeOrigin: cronWakeOrigin(c, 'node_birth'),
+        });
+        emitEvent({
+            level: 'info',
+            event: 'cron.failure.escalated',
+            fields: { cron_id: c.cron_id, name: c.name, escalation_node: res.node.node_id },
+        });
+        return `failure: cron paused, escalation node ${res.node.node_id} spawned`;
+    }
+    catch (err) {
+        emitEvent({
+            level: 'error',
+            event: 'cron.escalation.spawn_failed',
+            error: err,
+            fields: { cron_id: c.cron_id, name: c.name },
+        });
+        return `failure: cron paused; escalation spawn failed: ${err.message}`;
+    }
+}
+/** The delivery-failure line an escalated crashed ONE-SHOT carries: its side
+ *  effects may already have landed in full, so a human/agent decides whether to
+ *  redo it rather than the daemon re-firing it. */
+export function crashedOneShotSummary(why) {
+    return [
+        `${why}.`,
+        'This is a ONE-SHOT, so it has been PAUSED rather than released: its side effects may have completed in full,',
+        'and re-firing would enact them a second time. Determine whether the work actually landed, then either',
+        '`crtr cron run` it to redo it deliberately, or `crtr cron cancel` it.',
+    ].join(' ');
+}

package/dist/daemon/cron/held.d.ts

@@ -0,0 +1,4 @@
+import { type Cron, type HeldSettle } from '../../core/canvas/crons.js';
+export declare const EXIT_HELD = 75;
+export declare const HELD_DELIVERED = "deferred (exit 75) \u2014 held for poke";
+export declare function decideHeldSettle(c: Cron, startedAtIso: string, finishedAtMs: number): HeldSettle | undefined;

package/dist/daemon/cron/held.js

@@ -0,0 +1,20 @@
+import { getCron } from '../../core/canvas/crons.js';
+// Exit 75 parks an owed occurrence; the daemon honors the gate but does not implement it.
+export const EXIT_HELD = 75;
+export const HELD_DELIVERED = 'deferred (exit 75) — held for poke';
+// A held one-shot without expiry is poke-only; leaving it past due would re-fire every tick.
+const HELD_ONE_SHOT_FAR_FUTURE = '9999-12-31T23:59:59.999Z';
+// Re-read `last_poke_at`: a poke can land while the gate check runs.
+export function decideHeldSettle(c, startedAtIso, finishedAtMs) {
+    const fresh = getCron(c.cron_id);
+    if (fresh === null)
+        return undefined;
+    if (fresh.state === 'active' && fresh.last_poke_at !== null && startedAtIso < fresh.last_poke_at) {
+        // A mid-run poke cannot unpark a lease, so re-due an active row; paused rows must remain parked.
+        return { kind: 'redue', nowIso: new Date(finishedAtMs).toISOString() };
+    }
+    if (c.recur != null) {
+        return { kind: 'park' };
+    }
+    return { kind: 'park', fireAt: fresh.expires_at ?? HELD_ONE_SHOT_FAR_FUTURE };
+}

package/dist/daemon/cron/lease-recovery.d.ts

@@ -0,0 +1,3 @@
+import type { RunDueCronOptions } from './passes.js';
+/** Recover leases from prior daemon instances. Verify launch identity, not pid liveness: a reboot can recycle the pid, and killing its new owner would signal a stranger. */
+export declare function recoverStaleCronLeases(now: number, options?: RunDueCronOptions): void;

package/dist/daemon/cron/lease-recovery.js

@@ -0,0 +1,119 @@
+import { randomUUID } from 'node:crypto';
+import { setTimeout as delay } from 'node:timers/promises';
+import { CRON_DEATH_POLL_MS, CRON_KILL_GRACE_MS, EMPTY_STDOUT_SHA256 } from './capture.js';
+import { cronDaemonInstanceId } from './live-runs.js';
+import { crashedOneShotSummary, escalateFailure } from './escalation.js';
+import { settleRun } from './settle.js';
+import { killLeasedCronRun, runningCrons, setCronState, verifyLeasedCronRun, } from '../../core/canvas/crons.js';
+import { emitEvent } from '../../core/events/emit.js';
+/** Recover leases from prior daemon instances. Verify launch identity, not pid liveness: a reboot can recycle the pid, and killing its new owner would signal a stranger. */
+export function recoverStaleCronLeases(now, options = {}) {
+    for (const c of runningCrons()) {
+        if (c.run_lease_owner === cronDaemonInstanceId)
+            continue;
+        if (!verifyLeasedCronRun(c)) {
+            settleCrashedRun(c, now, options);
+            continue;
+        }
+        const startedAtMs = c.run_started_at != null ? new Date(c.run_started_at).getTime() : now;
+        if (now - startedAtMs < c.run_timeout_s * 1000)
+            continue;
+        if (!killLeasedCronRun(c)) {
+            settleCrashedRun(c, now, options);
+            continue;
+        }
+        const ctx = {
+            runId: randomUUID(),
+            startedAtIso: c.run_started_at ?? new Date(now).toISOString(),
+            exitCode: -1,
+            timedOut: true,
+            runTimeoutS: c.run_timeout_s,
+            processErrorMessage: null,
+            stdoutHead: '',
+            stdoutSha256: EMPTY_STDOUT_SHA256,
+            stderrHead: `run timed out after ${c.run_timeout_s}s (recovered stale lease from a prior daemon instance)`,
+        };
+        const settlement = (async () => {
+            await awaitLeasedRunDeath(c);
+            await settleRun(c, {
+                outOfBand: false,
+                outcome: { kind: 'disposed', ctx },
+                record: {
+                    runId: ctx.runId,
+                    startedAtIso: ctx.startedAtIso,
+                    finishedAtMs: now,
+                    durationMs: now - startedAtMs,
+                    exitCode: -1,
+                    stdoutHead: null,
+                    stderrHead: ctx.stderrHead,
+                },
+                lease: { kind: 'clear' },
+            });
+        })();
+        options.register?.(settlement);
+        void settlement.catch((err) => {
+            emitEvent({ level: 'error', event: 'cron.stale_lease.settle_failed', error: err, fields: { cron_id: c.cron_id } });
+        });
+    }
+}
+// Wait after SIGTERM: a signal-trapping command can still produce side effects.
+async function awaitLeasedRunDeath(c) {
+    const deadline = Date.now() + CRON_KILL_GRACE_MS;
+    while (Date.now() < deadline) {
+        await delay(CRON_DEATH_POLL_MS);
+        if (!verifyLeasedCronRun(c))
+            return; // gone
+    }
+    killLeasedCronRun(c, 'SIGKILL');
+}
+/** Record an unobserved crashed run. Pause one-shots rather than re-fire them: their side effects may already have landed. */
+function settleCrashedRun(c, now, options) {
+    const oneShot = c.recur == null;
+    const startedAtMs = c.run_started_at != null ? new Date(c.run_started_at).getTime() : now;
+    const why = 'the daemon life that owned this run ended before it settled — the process is gone and its outcome was never observed';
+    const ctx = {
+        runId: randomUUID(),
+        startedAtIso: c.run_started_at ?? new Date(now).toISOString(),
+        exitCode: null,
+        timedOut: false,
+        runTimeoutS: c.run_timeout_s,
+        processErrorMessage: why,
+        stdoutHead: '',
+        stdoutSha256: EMPTY_STDOUT_SHA256,
+        stderrHead: `run outcome unobserved: ${why}`,
+    };
+    const settlement = (async () => {
+        let delivered = `crashed: ${why}${oneShot ? '' : ' — the next occurrence is unaffected'}`;
+        if (oneShot) {
+            // Pause before escalation so a throw cannot re-fire the past-due one-shot.
+            setCronState(c.cron_id, 'paused');
+            delivered =
+                c.on_output === 'silent'
+                    ? `crashed: ${why} — one-shot paused, not re-fired (silent: not escalated)`
+                    : `crashed one-shot: ${await escalateFailure(c, ctx, crashedOneShotSummary(why))}`;
+        }
+        await settleRun(c, {
+            outOfBand: false,
+            outcome: { kind: 'stated', delivered, paused: oneShot },
+            record: {
+                runId: ctx.runId,
+                startedAtIso: ctx.startedAtIso,
+                finishedAtMs: now,
+                durationMs: Math.max(0, now - startedAtMs),
+                exitCode: null,
+                stdoutHead: null,
+                stderrHead: ctx.stderrHead,
+            },
+            lease: 'release',
+        });
+        emitEvent({
+            level: 'warn',
+            event: 'cron.run.unobserved',
+            fields: { cron_id: c.cron_id, name: c.name, one_shot: oneShot },
+        });
+    })();
+    options.register?.(settlement);
+    void settlement.catch((err) => {
+        emitEvent({ level: 'error', event: 'cron.crashed_run.settle_failed', error: err, fields: { cron_id: c.cron_id } });
+    });
+}

package/dist/daemon/cron/live-runs.d.ts

@@ -0,0 +1,11 @@
+export declare const cronDaemonInstanceId: `${string}-${string}-${string}-${string}-${string}`;
+export interface LiveRun {
+    runId: string;
+    replaced: boolean;
+    selfCanceled: boolean;
+    kill(): void;
+}
+export declare function liveRunFor(cronId: string): LiveRun | undefined;
+export declare function registerLiveRun(cronId: string, run: LiveRun): void;
+export declare function clearLiveRun(cronId: string, runId: string): void;
+export declare function markSelfCanceledCronRun(cronId: string, runId: string): boolean;

package/dist/daemon/cron/live-runs.js

@@ -0,0 +1,23 @@
+import { randomUUID } from 'node:crypto';
+export const cronDaemonInstanceId = randomUUID();
+const liveRuns = new Map();
+export function liveRunFor(cronId) {
+    return liveRuns.get(cronId);
+}
+export function registerLiveRun(cronId, run) {
+    liveRuns.set(cronId, run);
+}
+// An earlier close handler must not evict a later run.
+export function clearLiveRun(cronId, runId) {
+    const live = liveRuns.get(cronId);
+    if (live !== undefined && live.runId === runId)
+        liveRuns.delete(cronId);
+}
+// Only that exact process may finish after its row is removed.
+export function markSelfCanceledCronRun(cronId, runId) {
+    const live = liveRuns.get(cronId);
+    if (live === undefined || live.runId !== runId)
+        return false;
+    live.selfCanceled = true;
+    return true;
+}

package/dist/daemon/cron/passes.d.ts

@@ -0,0 +1,11 @@
+import { type Cron } from '../../core/canvas/crons.js';
+export interface RunDueCronOptions {
+    /** Registers settlement work for graceful-teardown draining. */
+    register?: (work: Promise<void>) => void;
+    /** Node-launching runs need capacity; absent means all runs are admitted. */
+    hasBrokerCapacity?: () => boolean;
+}
+export declare function cronLaunchesNode(c: Cron): boolean;
+export declare function expireDueCrons(nowIso: string): void;
+export declare function applyOverlapPolicies(now: number, nowIso: string): void;
+export declare function emitCapacityDeferral(now: number, deferred: number): void;