@north-light/crouter 0.3.233 → 0.3.235

package/dist/daemon/cron/lease-recovery.js

@@ -0,0 +1,119 @@
+import { randomUUID } from 'node:crypto';
+import { setTimeout as delay } from 'node:timers/promises';
+import { CRON_DEATH_POLL_MS, CRON_KILL_GRACE_MS, EMPTY_STDOUT_SHA256 } from './capture.js';
+import { cronDaemonInstanceId } from './live-runs.js';
+import { crashedOneShotSummary, escalateFailure } from './escalation.js';
+import { settleRun } from './settle.js';
+import { killLeasedCronRun, runningCrons, setCronState, verifyLeasedCronRun, } from '../../core/canvas/crons.js';
+import { emitEvent } from '../../core/events/emit.js';
+/** Recover leases from prior daemon instances. Verify launch identity, not pid liveness: a reboot can recycle the pid, and killing its new owner would signal a stranger. */
+export function recoverStaleCronLeases(now, options = {}) {
+    for (const c of runningCrons()) {
+        if (c.run_lease_owner === cronDaemonInstanceId)
+            continue;
+        if (!verifyLeasedCronRun(c)) {
+            settleCrashedRun(c, now, options);
+            continue;
+        }
+        const startedAtMs = c.run_started_at != null ? new Date(c.run_started_at).getTime() : now;
+        if (now - startedAtMs < c.run_timeout_s * 1000)
+            continue;
+        if (!killLeasedCronRun(c)) {
+            settleCrashedRun(c, now, options);
+            continue;
+        }
+        const ctx = {
+            runId: randomUUID(),
+            startedAtIso: c.run_started_at ?? new Date(now).toISOString(),
+            exitCode: -1,
+            timedOut: true,
+            runTimeoutS: c.run_timeout_s,
+            processErrorMessage: null,
+            stdoutHead: '',
+            stdoutSha256: EMPTY_STDOUT_SHA256,
+            stderrHead: `run timed out after ${c.run_timeout_s}s (recovered stale lease from a prior daemon instance)`,
+        };
+        const settlement = (async () => {
+            await awaitLeasedRunDeath(c);
+            await settleRun(c, {
+                outOfBand: false,
+                outcome: { kind: 'disposed', ctx },
+                record: {
+                    runId: ctx.runId,
+                    startedAtIso: ctx.startedAtIso,
+                    finishedAtMs: now,
+                    durationMs: now - startedAtMs,
+                    exitCode: -1,
+                    stdoutHead: null,
+                    stderrHead: ctx.stderrHead,
+                },
+                lease: { kind: 'clear' },
+            });
+        })();
+        options.register?.(settlement);
+        void settlement.catch((err) => {
+            emitEvent({ level: 'error', event: 'cron.stale_lease.settle_failed', error: err, fields: { cron_id: c.cron_id } });
+        });
+    }
+}
+// Wait after SIGTERM: a signal-trapping command can still produce side effects.
+async function awaitLeasedRunDeath(c) {
+    const deadline = Date.now() + CRON_KILL_GRACE_MS;
+    while (Date.now() < deadline) {
+        await delay(CRON_DEATH_POLL_MS);
+        if (!verifyLeasedCronRun(c))
+            return; // gone
+    }
+    killLeasedCronRun(c, 'SIGKILL');
+}
+/** Record an unobserved crashed run. Pause one-shots rather than re-fire them: their side effects may already have landed. */
+function settleCrashedRun(c, now, options) {
+    const oneShot = c.recur == null;
+    const startedAtMs = c.run_started_at != null ? new Date(c.run_started_at).getTime() : now;
+    const why = 'the daemon life that owned this run ended before it settled — the process is gone and its outcome was never observed';
+    const ctx = {
+        runId: randomUUID(),
+        startedAtIso: c.run_started_at ?? new Date(now).toISOString(),
+        exitCode: null,
+        timedOut: false,
+        runTimeoutS: c.run_timeout_s,
+        processErrorMessage: why,
+        stdoutHead: '',
+        stdoutSha256: EMPTY_STDOUT_SHA256,
+        stderrHead: `run outcome unobserved: ${why}`,
+    };
+    const settlement = (async () => {
+        let delivered = `crashed: ${why}${oneShot ? '' : ' — the next occurrence is unaffected'}`;
+        if (oneShot) {
+            // Pause before escalation so a throw cannot re-fire the past-due one-shot.
+            setCronState(c.cron_id, 'paused');
+            delivered =
+                c.on_output === 'silent'
+                    ? `crashed: ${why} — one-shot paused, not re-fired (silent: not escalated)`
+                    : `crashed one-shot: ${await escalateFailure(c, ctx, crashedOneShotSummary(why))}`;
+        }
+        await settleRun(c, {
+            outOfBand: false,
+            outcome: { kind: 'stated', delivered, paused: oneShot },
+            record: {
+                runId: ctx.runId,
+                startedAtIso: ctx.startedAtIso,
+                finishedAtMs: now,
+                durationMs: Math.max(0, now - startedAtMs),
+                exitCode: null,
+                stdoutHead: null,
+                stderrHead: ctx.stderrHead,
+            },
+            lease: 'release',
+        });
+        emitEvent({
+            level: 'warn',
+            event: 'cron.run.unobserved',
+            fields: { cron_id: c.cron_id, name: c.name, one_shot: oneShot },
+        });
+    })();
+    options.register?.(settlement);
+    void settlement.catch((err) => {
+        emitEvent({ level: 'error', event: 'cron.crashed_run.settle_failed', error: err, fields: { cron_id: c.cron_id } });
+    });
+}

package/dist/daemon/cron/live-runs.d.ts

@@ -0,0 +1,11 @@
+export declare const cronDaemonInstanceId: `${string}-${string}-${string}-${string}-${string}`;
+export interface LiveRun {
+    runId: string;
+    replaced: boolean;
+    selfCanceled: boolean;
+    kill(): void;
+}
+export declare function liveRunFor(cronId: string): LiveRun | undefined;
+export declare function registerLiveRun(cronId: string, run: LiveRun): void;
+export declare function clearLiveRun(cronId: string, runId: string): void;
+export declare function markSelfCanceledCronRun(cronId: string, runId: string): boolean;

package/dist/daemon/cron/live-runs.js

@@ -0,0 +1,23 @@
+import { randomUUID } from 'node:crypto';
+export const cronDaemonInstanceId = randomUUID();
+const liveRuns = new Map();
+export function liveRunFor(cronId) {
+    return liveRuns.get(cronId);
+}
+export function registerLiveRun(cronId, run) {
+    liveRuns.set(cronId, run);
+}
+// An earlier close handler must not evict a later run.
+export function clearLiveRun(cronId, runId) {
+    const live = liveRuns.get(cronId);
+    if (live !== undefined && live.runId === runId)
+        liveRuns.delete(cronId);
+}
+// Only that exact process may finish after its row is removed.
+export function markSelfCanceledCronRun(cronId, runId) {
+    const live = liveRuns.get(cronId);
+    if (live === undefined || live.runId !== runId)
+        return false;
+    live.selfCanceled = true;
+    return true;
+}

package/dist/daemon/cron/passes.d.ts

@@ -0,0 +1,11 @@
+import { type Cron } from '../../core/canvas/crons.js';
+export interface RunDueCronOptions {
+    /** Registers settlement work for graceful-teardown draining. */
+    register?: (work: Promise<void>) => void;
+    /** Node-launching runs need capacity; absent means all runs are admitted. */
+    hasBrokerCapacity?: () => boolean;
+}
+export declare function cronLaunchesNode(c: Cron): boolean;
+export declare function expireDueCrons(nowIso: string): void;
+export declare function applyOverlapPolicies(now: number, nowIso: string): void;
+export declare function emitCapacityDeferral(now: number, deferred: number): void;

package/dist/daemon/cron/passes.js

@@ -0,0 +1,78 @@
+import { randomUUID } from 'node:crypto';
+import { cronDaemonInstanceId, liveRunFor } from './live-runs.js';
+import { advanceCron, cancelCron, dueOverlapCrons, expiredIdleCrons, listCronRuns, recordCronRun, } from '../../core/canvas/crons.js';
+import { emitEvent } from '../../core/events/emit.js';
+import { nextSlotAfter } from '../../core/wake.js';
+import { parseSink } from '../cron-sink.js';
+/** Lead of a `cron_runs.delivered` string written by the overlap=skip pass.
+ *  Matched (not just written) — see `latestRunIsSkip`. */
+const SKIP_MARKER = 'skipped (overlap=skip)';
+// A false negative is an ungated run; a false positive delays one poll at capacity.
+const NODE_LAUNCHING_COMMAND = /\bcrtr\b[^\n]*?\bnode\s+lifecycle\s+revive\b|\bcrtr\b[^\n]*?\bnode\s+new\b|\bcrtr\b[^\n]*?\bcanvas\s+revive\b|\bcrtr\b[^\n]*?\bnode\s+message\s+send\b[^\n]*?\s--fresh\b/;
+export function cronLaunchesNode(c) {
+    if (parseSink(c.sink)?.kind === 'spawn')
+        return true;
+    return NODE_LAUNCHING_COMMAND.test(c.command);
+}
+export function expireDueCrons(nowIso) {
+    for (const c of expiredIdleCrons(nowIso)) {
+        cancelCron(c.cron_id);
+        emitEvent({
+            level: 'info',
+            event: 'cron.expired',
+            fields: { cron_id: c.cron_id, name: c.name, expires_at: c.expires_at },
+        });
+    }
+}
+function latestRunIsSkip(cronId) {
+    return listCronRuns(cronId)[0]?.delivered?.startsWith(SKIP_MARKER) === true;
+}
+export function applyOverlapPolicies(now, nowIso) {
+    for (const c of dueOverlapCrons(nowIso)) {
+        if (c.overlap === 'queue')
+            continue;
+        if (c.overlap === 'skip') {
+            let nextFire;
+            try {
+                nextFire = nextSlotAfter(c.recur, new Date(now));
+            }
+            catch {
+                continue; // paused + escalated by the run path when it next fires
+            }
+            advanceCron(c.cron_id, nextFire);
+            // One marker per streak preserves real runs in the 20-entry ring.
+            if (!latestRunIsSkip(c.cron_id)) {
+                recordCronRun({
+                    run_id: randomUUID(),
+                    cron_id: c.cron_id,
+                    started: nowIso,
+                    finished: nowIso,
+                    duration_ms: 0,
+                    exit_code: null,
+                    stdout_head: null,
+                    stderr_head: null,
+                    delivered: `${SKIP_MARKER}: fire landed while the previous run was in flight (consecutive skips are not re-recorded until a run settles)`,
+                });
+            }
+            continue;
+        }
+        // replace
+        const live = liveRunFor(c.cron_id);
+        if (c.run_lease_owner === cronDaemonInstanceId && live !== undefined && !live.replaced) {
+            live.replaced = true;
+            live.kill();
+        }
+    }
+}
+const CAPACITY_DEFER_LOG_THROTTLE_MS = 30_000;
+let lastCapacityDeferLogAt = Number.NEGATIVE_INFINITY;
+export function emitCapacityDeferral(now, deferred) {
+    if (now - lastCapacityDeferLogAt < CAPACITY_DEFER_LOG_THROTTLE_MS)
+        return;
+    lastCapacityDeferLogAt = now;
+    emitEvent({
+        level: 'warn',
+        event: 'cron.capacity.run_deferred',
+        fields: { deferred_runs: deferred },
+    });
+}

package/dist/daemon/cron/settle.d.ts

@@ -0,0 +1,28 @@
+import { type Cron, type CronRunRecord, type HeldSettle } from '../../core/canvas/crons.js';
+import { type RunSettleContext } from './disposition.js';
+export type LeaseSettlement = 'none' | 'release' | HeldSettle;
+export type SettleOutcome = {
+    kind: 'disposed';
+    ctx: RunSettleContext;
+    selfCanceled?: boolean;
+} | {
+    kind: 'stated';
+    delivered: string;
+    paused?: boolean;
+};
+export interface SettleRecord {
+    runId: string;
+    startedAtIso: string;
+    finishedAtMs: number;
+    durationMs: number;
+    exitCode: number | null;
+    stdoutHead: string | null;
+    stderrHead: string | null;
+}
+export interface SettleRunInput {
+    outOfBand: boolean;
+    outcome: SettleOutcome;
+    record: SettleRecord;
+    lease: LeaseSettlement;
+}
+export declare function settleRun(c: Cron, input: SettleRunInput): Promise<CronRunRecord>;

package/dist/daemon/cron/settle.js

@@ -0,0 +1,37 @@
+import { consumeCron, recordCronRun, releaseCronRunLease, } from '../../core/canvas/crons.js';
+import { disposeSettledRun } from './disposition.js';
+function occurrenceStaysOwed(lease) {
+    return typeof lease === 'object' && (lease.kind === 'park' || lease.kind === 'redue');
+}
+export async function settleRun(c, input) {
+    const { outOfBand, outcome, record, lease } = input;
+    let delivered;
+    let paused;
+    if (outcome.kind === 'disposed') {
+        const disposition = await disposeSettledRun(c, outcome.ctx, outOfBand, outcome.selfCanceled ?? false);
+        delivered = disposition.delivered;
+        paused = disposition.paused;
+    }
+    else {
+        delivered = outcome.delivered;
+        paused = outcome.paused ?? false;
+    }
+    const written = {
+        run_id: record.runId,
+        cron_id: c.cron_id,
+        started: record.startedAtIso,
+        finished: new Date(record.finishedAtMs).toISOString(),
+        duration_ms: record.durationMs,
+        exit_code: record.exitCode,
+        stdout_head: record.stdoutHead,
+        stderr_head: record.stderrHead,
+        delivered,
+    };
+    recordCronRun(written);
+    if (lease !== 'none')
+        releaseCronRunLease(c.cron_id, lease === 'release' ? undefined : lease);
+    // Retain out-of-band, paused, and held rows; their occurrence is not spent.
+    if (!outOfBand && c.recur == null && !paused && !occurrenceStaysOwed(lease))
+        consumeCron(c.cron_id);
+    return written;
+}

package/dist/daemon/cron/sinks.d.ts

@@ -0,0 +1,2 @@
+import type { Cron } from '../../core/canvas/crons.js';
+export declare function deliverToSink(c: Cron, stdout: string): Promise<string>;

package/dist/daemon/cron/sinks.js

@@ -0,0 +1,100 @@
+import { randomBytes } from 'node:crypto';
+import { rmSync } from 'node:fs';
+import { synthesizedPageDocument, writeSynthesizedPage } from '../../core/human/page-synth.js';
+import { ticketDir } from '../../core/human/root.js';
+import { preparePage, publishPage } from '../../core/human/tickets.js';
+import { resolvePageComponents } from '../../core/config.js';
+import { getNode } from '../../core/canvas/canvas.js';
+import { appendInbox } from '../../core/feed/inbox.js';
+import { hasNoNaturalCycle } from '../../core/runtime/revive-all.js';
+import { isParked } from '../../core/runtime/lifecycle.js';
+import { reviveNode } from '../../core/runtime/revive.js';
+import { isBrokerLive } from '../../core/runtime/model-swap.js';
+import { spawnChild } from '../../core/runtime/spawn.js';
+import { cronWakeOrigin } from '../../core/runtime/bearings.js';
+import { parseSink } from '../cron-sink.js';
+import { resolveLiveProfile, resolveLiveProfileName } from './armed-context.js';
+function deliverNodeSink(c, target, stdout) {
+    const meta = getNode(target);
+    if (meta === null)
+        throw new Error(`node sink ${target} is gone`);
+    // A cron is an explicit next cycle for a clock-parked node, not other done nodes.
+    if (hasNoNaturalCycle(meta) && !isParked(meta)) {
+        throw new Error(`node sink ${target} has no natural cycle ahead (done or finalized)`);
+    }
+    const from = c.created_by != null && getNode(c.created_by) !== null ? c.created_by : null;
+    appendInbox(target, {
+        from,
+        tier: c.tier,
+        kind: 'message',
+        label: `⏰ cron ${c.name}`,
+        data: { body: stdout },
+    });
+    // The durable append succeeds even if its best-effort wake fails.
+    if (c.tier !== 'deferred' && !isBrokerLive(meta)) {
+        try {
+            reviveNode(target, { resume: true });
+        }
+        catch {
+            /* best-effort wake — the entry is durably appended regardless */
+        }
+    }
+}
+async function deliverSpawnSink(c, kind, stdout) {
+    const creatorAlive = c.created_by != null && getNode(c.created_by) !== null;
+    const prompt = [
+        `Scheduled cron "${c.name}" (${c.cron_id}) fired and delivered its output to you as your kickoff.`,
+        '',
+        '## Command output',
+        '',
+        stdout,
+    ].join('\n');
+    const res = await spawnChild({
+        kind,
+        cwd: c.cwd,
+        prompt,
+        // Creatorless sinks are terminal roots so each run reaps without stealing focus.
+        ...(creatorAlive ? { parent: c.created_by } : { root: true, rootLifecycle: 'terminal' }),
+        readiness: 'launch',
+        profile: resolveLiveProfile(c),
+        wakeOrigin: cronWakeOrigin(c, 'node_birth'),
+    });
+    return res.node.node_id;
+}
+function deliverHumanSink(c, stdout) {
+    const synthesized = writeSynthesizedPage(synthesizedPageDocument(`⏰ cron ${c.name}`, `Scheduled cron "${c.name}" delivered output for your attention.`, stdout, false));
+    // Validate verbatim stdout before minting a ticket to avoid a half-built page.
+    let prepared;
+    try {
+        const profileName = resolveLiveProfileName(c);
+        prepared = preparePage({
+            sourceFile: synthesized.path,
+            source: {
+                sessionName: `cron ${c.name} (${c.cron_id})`,
+                ...(profileName === undefined ? {} : { profileName }),
+            },
+            productKinds: resolvePageComponents(),
+            delivery: { placement: 'inline', inbox: true, reply: false },
+        });
+    }
+    finally {
+        rmSync(synthesized.dir, { recursive: true, force: true });
+    }
+    const id = `page-${randomBytes(4).toString('hex')}`;
+    publishPage(ticketDir(id), prepared);
+}
+export async function deliverToSink(c, stdout) {
+    const sink = parseSink(c.sink);
+    if (sink === null)
+        return 'no sink armed — nothing delivered';
+    if (sink.kind === 'node') {
+        deliverNodeSink(c, sink.node, stdout);
+        return `delivered: node ${sink.node}`;
+    }
+    if (sink.kind === 'spawn') {
+        const nodeId = await deliverSpawnSink(c, sink.spawn, stdout);
+        return `delivered: spawned ${nodeId}`;
+    }
+    deliverHumanSink(c, stdout);
+    return 'delivered: human send --inbox';
+}

package/dist/daemon/cron-run.d.ts

@@ -1,45 +1,9 @@
+import { type RunDueCronOptions } from './cron/passes.js';
 import { type Cron, type CronRunRecord } from '../core/canvas/crons.js';
-/** Output head caps for the run log (spec: "capped, e.g. 4KB"). */
-export declare const CRON_OUTPUT_HEAD_BYTES: number;
-/** One id per daemon process instance — distinguishes THIS process's own
- *  in-flight run leases from a stale 'running' row a prior (crashed/restarted)
- *  daemon left behind. */
-export declare const cronDaemonInstanceId: `${string}-${string}-${string}-${string}-${string}`;
-export interface RunDueCronOptions {
-    /** Invoked synchronously the instant each settlement promise is created, so
-     *  the caller can register it for a graceful-teardown drain. */
-    register?: (work: Promise<void>) => void;
-    /** Broker-capacity admission for a run that will LAUNCH OR REVIVE a node
-     *  (`cronLaunchesNode`). Returns false when the live fleet is at the
-     *  automatic-revive cap; the daemon wires in its own `hasBrokerCapacity`
-     *  (crtrd.ts), which owns the fleet count and the throttled capacity event.
-     *  Absent → every run is admitted (out-of-band/test callers own no fleet). */
-    hasBrokerCapacity?: () => boolean;
-}
-/** Will this cron's run consume broker capacity? True when its armed sink
- *  births a node, or its command invokes a node-launching crtr verb. A cron
- *  that merely runs a shell command or sends a message reads false and is
- *  never gated. */
-export declare function cronLaunchesNode(c: Cron): boolean;
-/** Mark a cancellation as originating from the live cron run it names. Only
- * that exact process may finish after its row is removed, so an external cancel
- * retains its normal process-group kill behavior. */
-export declare function markSelfCanceledCronRun(cronId: string, runId: string): boolean;
-/** Recover stale 'running' rows left by a PRIOR daemon instance: a pid that is
- *  not VERIFIABLY still the process this lease launched resets straight to
- *  idle; a confirmed-live process group past its own run_timeout_s is killed
- *  and settled as a timeout run through the SAME failure path a live timeout
- *  takes (pause + escalation unless silent). Rows this SAME instance owns are
- *  genuinely in flight and left alone.
- *
- *  The verification is `verifyLeasedCronRun`, not `isPidAlive`, and it is
- *  FAIL-CLOSED: this row was written by a daemon life that has since ended, so
- *  after a host reboot its `run_pid` names a slot in a dead kernel's pid space
- *  that the OS has probably recycled, and the timeout comparison below is
- *  trivially satisfied by any downtime. Liveness alone would therefore SIGTERM
- *  a stranger's process group. Only a matching launch-time identity authorizes
- *  the signal; everything else releases the lease. */
-export declare function recoverStaleCronLeases(now: number, options?: RunDueCronOptions): void;
+export { CRON_OUTPUT_HEAD_BYTES } from './cron/capture.js';
+export { cronDaemonInstanceId, markSelfCanceledCronRun } from './cron/live-runs.js';
+export { cronLaunchesNode, type RunDueCronOptions } from './cron/passes.js';
+export { recoverStaleCronLeases } from './cron/lease-recovery.js';
 /** Kick off ONE cron run and return its settlement promise, resolving with
  *  the settled run record (null = the idle→running lease was lost, or the row
  *  was quarantined — nothing was recorded by THIS call). The subprocess