@north-light/crouter 0.3.255 → 0.3.256

package/dist/daemon/reconcilers/managed-worktree-sweep.js

@@ -0,0 +1,190 @@
+// The daemon lane that drives managed-worktree reconciliation.
+//
+// It owns WHEN and HOW MANY; `core/worktree-sweep.ts` owns what a single
+// reconciliation does. The lane never blocks the supervision tick: it registers
+// one detached pass and returns immediately, and only one pass is ever in
+// flight, so passes cannot overlap however long a repository takes to answer.
+//
+// NOTHING that spawns a subprocess runs on the tick, and nothing spawns one PER
+// CANDIDATE. `run()` does no work beyond timing; the detached pass reads the
+// canvas, applies the budget, and then takes ONE asynchronous `ps` snapshot
+// that classifies every budgeted candidate's engine. A per-pid probe here would
+// stall the daemon once per stuck worktree.
+//
+// Cost is flat regardless of backlog. The candidate scan touches only terminal
+// nodes, and only those carrying an unreconciled worktree record reach a Git
+// call at all — normally none, so the pass costs zero subprocesses. When there
+// is a backlog, a fixed per-pass budget walks the candidates round-robin, so
+// work per minute is the same whether three worktrees are stuck or three
+// hundred.
+import { getNode, getRow, listNodes } from '../../core/canvas/index.js';
+import { isSafeNodeId } from '../../core/canvas/paths.js';
+import { captureLivenessSnapshotAsync, recordedPidLiveness } from '../../core/canvas/pid.js';
+import { emitEvent } from '../../core/events/emit.js';
+import { operationIdContext } from '../../core/events/operation-id.js';
+import { isDueForSweep, isReconcilable, reconcileManagedWorktree } from '../../core/worktree-sweep.js';
+export const WORKTREE_SWEEP_INTERVAL_MS = 60 * 1000;
+/** Candidates reconciled per pass. Bounds the work a large backlog can demand. */
+export const WORKTREE_SWEEP_BUDGET = 3;
+const TERMINAL_STATUSES = ['done', 'dead', 'canceled'];
+function isTerminal(row) {
+    return TERMINAL_STATUSES.includes(row.status);
+}
+/** Is this node's engine definitively gone?
+ *
+ *  Terminal status alone is not enough: `push final` marks a node `done` while
+ *  its broker is still running out of the managed checkout, which is exactly
+ *  the case that made cleanup deferred in the first place. So the sweep waits
+ *  for terminal status AND the absence of any live engine, and an
+ *  INDETERMINATE pid probe counts as live — uncertainty must never authorize
+ *  deleting a checkout something may still be writing to.
+ *
+ *  `isClaimed` rather than `has`: a revive that has reserved its slot but not
+ *  yet registered its broker is a node coming back to life, and the checkout it
+ *  is about to run in must not be removed underneath it. */
+function engineIsGone(row, fleet, snapshot) {
+    if (fleet.isClaimed(row.node_id))
+        return false;
+    if (row.frozen_at !== null)
+        return false;
+    if (!isTerminal(row))
+        return false;
+    return recordedPidLiveness(row.pi_pid, row.pi_pid_identity, snapshot) === 'dead';
+}
+/** The eligibility recheck the reconciliation runs immediately before EVERY Git
+ *  mutation, synchronously and with no subprocess: a revive that won the race
+ *  makes this false and the mutation never happens.
+ *
+ *  It re-reads the row rather than trusting the classification the pass began
+ *  with, and it compares the recorded engine identity against the one that was
+ *  classified dead — a row whose pid or pid identity moved has been relaunched,
+ *  whatever its status column currently says. */
+function eligibilityGuard(row, fleet) {
+    const nodeId = row.node_id;
+    const pid = row.pi_pid;
+    const identity = row.pi_pid_identity;
+    return {
+        stillEligible: () => {
+            if (fleet.isClaimed(nodeId))
+                return false;
+            const current = getRow(nodeId);
+            if (current === null || !isTerminal(current) || current.frozen_at !== null)
+                return false;
+            return current.pi_pid === pid && current.pi_pid_identity === identity;
+        },
+    };
+}
+export class ManagedWorktreeSweepReconciler {
+    lastSweepAt = Number.NEGATIVE_INFINITY;
+    inFlight = false;
+    /** Round-robin cursor over candidate node ids, so a permanently stuck record
+     *  at the head of the list cannot starve the ones behind it. */
+    cursor = 0;
+    run(now, ctx) {
+        if (this.inFlight)
+            return;
+        if (now - this.lastSweepAt < WORKTREE_SWEEP_INTERVAL_MS)
+            return;
+        if (!ctx.lifecycle.acceptsDetachedWork())
+            return;
+        this.lastSweepAt = now;
+        this.inFlight = true;
+        const pass = this.sweep(now, ctx.fleet).finally(() => {
+            this.inFlight = false;
+        });
+        ctx.lifecycle.registerDetached(pass);
+    }
+    /** Terminal nodes whose managed worktree is unreconciled and whose backoff has
+     *  elapsed. Canvas reads only — engine liveness is classified later, for the
+     *  budgeted few, from one shared process snapshot. Returns null when the
+     *  canvas could not be read, which authorizes nothing. */
+    candidates(now) {
+        let rows;
+        try {
+            rows = listNodes({ status: [...TERMINAL_STATUSES] });
+        }
+        catch (err) {
+            operationIdContext.fresh(() => {
+                emitEvent({ level: 'error', event: 'worktree.sweep.scan_failed', error: err });
+            });
+            return null;
+        }
+        const due = [];
+        for (const row of rows) {
+            let wt;
+            try {
+                wt = getNode(row.node_id)?.managed_worktree;
+            }
+            catch {
+                continue; // an unreadable record proves no ownership; leave it alone
+            }
+            if (!isReconcilable(wt) || wt == null)
+                continue;
+            if (!isDueForSweep(wt, now))
+                continue;
+            due.push(row.node_id);
+        }
+        return due;
+    }
+    async sweep(now, fleet) {
+        const candidates = this.candidates(now);
+        if (candidates === null || candidates.length === 0)
+            return;
+        if (this.cursor >= candidates.length)
+            this.cursor = 0;
+        const start = this.cursor;
+        const take = Math.min(WORKTREE_SWEEP_BUDGET, candidates.length);
+        const budgeted = [];
+        for (let i = 0; i < take; i += 1)
+            budgeted.push(candidates[(start + i) % candidates.length]);
+        this.cursor = (start + take) % candidates.length;
+        // One process-table read for the whole pass, off the tick. A null snapshot
+        // is unknown liveness, and unknown never authorizes destroying a checkout,
+        // so the pass simply ends and the next one retries.
+        const snapshot = await captureLivenessSnapshotAsync();
+        if (snapshot === null)
+            return;
+        for (const nodeId of budgeted) {
+            const row = getRow(nodeId);
+            if (row === null || !engineIsGone(row, fleet, snapshot))
+                continue;
+            await this.reconcileOne(row, now, fleet);
+        }
+    }
+    async reconcileOne(row, now, fleet) {
+        const nodeId = row.node_id;
+        const target = isSafeNodeId(nodeId) ? { node_id: nodeId } : {};
+        const affected = isSafeNodeId(nodeId) ? {} : { affected_node_id: nodeId };
+        try {
+            const outcome = await reconcileManagedWorktree(nodeId, eligibilityGuard(row, fleet), now);
+            operationIdContext.fresh(() => {
+                if (outcome.status === 'complete') {
+                    emitEvent({
+                        level: 'info',
+                        event: 'worktree.sweep.reconciled',
+                        outcome: 'succeeded',
+                        ...target,
+                        fields: { ...affected, ...(outcome.stash === undefined ? {} : { base_checkout_stash: outcome.stash }) },
+                    });
+                    return;
+                }
+                if (outcome.status === 'skipped')
+                    return;
+                if (outcome.status === 'refused' && outcome.unchanged === true)
+                    return; // nothing moved; nothing to say
+                emitEvent({
+                    level: outcome.status === 'failed' ? 'error' : 'warn',
+                    event: outcome.status === 'failed' ? 'worktree.sweep.failed' : 'worktree.sweep.refused',
+                    outcome: outcome.status === 'failed' ? 'failed' : 'skipped',
+                    ...target,
+                    fields: { ...affected, reason: outcome.reason, ...(outcome.detail === undefined ? {} : { detail: outcome.detail }) },
+                });
+            });
+        }
+        catch (err) {
+            operationIdContext.fresh(() => {
+                emitEvent({ level: 'error', event: 'worktree.sweep.failed', ...target, error: err, fields: affected });
+            });
+        }
+    }
+}

package/dist/daemon/reconcilers/storage-maintenance.d.ts

@@ -1,10 +1,19 @@
+import type { FleetRegistry } from '../../core/runtime/fleet.js';
+import type { DetachedWorkLifecycle } from './broker-supervision.js';
 export declare const DEAD_REAP_GRACE_MS: number;
-export type StorageMaintenanceContext = Record<string, never>;
+export interface StorageMaintenanceContext {
+    /** Carries the managed-worktree sweep's detached Git work off this tick. */
+    lifecycle: DetachedWorkLifecycle;
+    /** A live engine handle is the sweep's proof that a node is NOT disposable. */
+    fleet: FleetRegistry;
+}
 /** Recurring storage cleanup. The in-memory throttle clocks intentionally reset
- * on daemon boot, so the first tick runs both full sweeps. */
+ * on daemon boot, so the first tick runs every full sweep — which is also how
+ * managed-worktree reconciliation gets its start-up pass. */
 export declare class StorageMaintenanceReconciler {
     private lastSpareSweepAt;
     private lastGhostSweepAt;
+    private readonly worktreeSweep;
     run(now: number, ctx: StorageMaintenanceContext): void;
     /** Drop rows whose on-disk node dir is gone. Such a row can never be revived,
      * focused, or closed — every one of those resolves the node through meta.json —

package/dist/daemon/reconcilers/storage-maintenance.js

@@ -6,18 +6,20 @@ import { emitEvent } from '../../core/events/emit.js';
 import { operationIdContext } from '../../core/events/operation-id.js';
 import { listLivePanes, tearDownNode } from '../../core/runtime/placement.js';
 import { reapStaleSpares } from '../../core/runtime/warm-pool.js';
+import { ManagedWorktreeSweepReconciler } from './managed-worktree-sweep.js';
 // How long a dead node's on-disk record must be quiet before its leftover
 // placement is reaped. A fresh crash keeps its pane for inspection.
 export const DEAD_REAP_GRACE_MS = 10 * 60_000;
 const SPARE_SWEEP_INTERVAL_MS = 60 * 1000;
 const GHOST_SWEEP_INTERVAL_MS = 60 * 1000;
 /** Recurring storage cleanup. The in-memory throttle clocks intentionally reset
- * on daemon boot, so the first tick runs both full sweeps. */
+ * on daemon boot, so the first tick runs every full sweep — which is also how
+ * managed-worktree reconciliation gets its start-up pass. */
 export class StorageMaintenanceReconciler {
     lastSpareSweepAt = Number.NEGATIVE_INFINITY;
     lastGhostSweepAt = Number.NEGATIVE_INFINITY;
+    worktreeSweep = new ManagedWorktreeSweepReconciler();
     run(now, ctx) {
-        void ctx; // no consumer today — kept only for six-way call-site uniformity
         // Placement reap MUST precede focus GC, so GC catches a focus row the reap
         // strands. Both run after the lifecycle tick, so a same-tick revive has
         // already had the chance to re-anchor its focus.
@@ -25,6 +27,9 @@ export class StorageMaintenanceReconciler {
         this.gcStaleFocuses();
         this.reapGhostRows(now);
         this.gcWarmPool(now);
+        // Registers detached async Git work and returns immediately: this lane is
+        // called un-awaited from a 2-second supervision tick and must not block it.
+        this.worktreeSweep.run(now, ctx);
     }
     /** Drop rows whose on-disk node dir is gone. Such a row can never be revived,
      * focused, or closed — every one of those resolves the node through meta.json —

package/dist/daemon/startup-block-marker.d.ts

@@ -0,0 +1,25 @@
+import { MigrationBlockedError } from '../migrations/activation.js';
+export declare function daemonStartupBlockMarkerPath(): string;
+/** Record that startup is blocked, so no OTHER process spawns into the same
+ *  wall. Best-effort: a home we cannot write to still gets in-process
+ *  suppression, which is strictly better than nothing. */
+export declare function recordDaemonStartupBlocked(message: string, blockerPaths: readonly string[]): void;
+/** Clear the marker — for a repair path that knows startup should be retried
+ *  (a successful migration, a successful daemon start, an explicit reset). */
+export declare function clearDaemonStartupBlocked(): void;
+/** The recorded block message when startup is still blocked by the SAME corpus
+ *  that produced it, or null when nothing is recorded or the corpus has since
+ *  changed. A changed corpus retires the marker as a side effect of the read,
+ *  so the very next spawner is free to try again.
+ *
+ *  A marker naming no blocker paths at all is not self-healing — nothing about
+ *  it can be observed to change — so it is treated as advisory only and
+ *  retired on read rather than left to wedge autostart forever. */
+export declare function daemonStartupBlocked(): string | null;
+/** The blocked-migration failure, wherever it ended up in a thrown value.
+ *  Startup releases its ownership claim before rethrowing, and a claim release
+ *  that ALSO fails is reported as an aggregate — so the one error the spawner
+ *  must react to can arrive nested. Matching only the bare instance would let
+ *  the aggregate path fall through to a generic exit, and every suppression
+ *  downstream is keyed on getting that exit code right. */
+export declare function blockedMigrationIn(error: unknown): MigrationBlockedError | null;

package/dist/daemon/startup-block-marker.js

@@ -0,0 +1,99 @@
+// A blocked daemon startup is a STANDING condition, and the processes that
+// keep retrying it are not one process. Every CLI invocation, every scheduled
+// job tick, and every long-lived client is its own process with its own module
+// state, so an in-memory suppression flag caps exactly one of them and lets the
+// rest storm unchanged. This marker is that suppression made durable: one file
+// in the canvas home, written by whoever first proved startup is blocked, and
+// read by every would-be spawner before it spawns.
+//
+// The hazard of a durable "stop trying" record is that it outlives the problem
+// and wedges a repaired install. So the marker is self-healing rather than
+// permanent: it records the identity (path + size + mtime) of each blocker the
+// daemon named, and any change to any of them retires the marker on the next
+// read. Repairing the corpus — by `crtr sys migrate`, an editor, or deleting
+// the offending file — is itself the thing that lifts the suppression, so no
+// user is ever required to know this file exists.
+import { rmSync } from 'node:fs';
+import { statSync } from 'node:fs';
+import { join } from 'node:path';
+import { crtrHome } from '../core/canvas/paths.js';
+import { MigrationBlockedError } from '../migrations/activation.js';
+import { atomicWriteJson, readJsonOrNull } from '../core/fs-utils.js';
+export function daemonStartupBlockMarkerPath() {
+    return join(crtrHome(), 'daemon-startup-blocked.json');
+}
+function fingerprint(path) {
+    try {
+        const stat = statSync(path);
+        return { path, sizeBytes: stat.size, mtimeMs: stat.mtimeMs };
+    }
+    catch {
+        return { path, sizeBytes: null, mtimeMs: null };
+    }
+}
+function sameFingerprint(a, b) {
+    return a.sizeBytes === b.sizeBytes && a.mtimeMs === b.mtimeMs;
+}
+/** Record that startup is blocked, so no OTHER process spawns into the same
+ *  wall. Best-effort: a home we cannot write to still gets in-process
+ *  suppression, which is strictly better than nothing. */
+export function recordDaemonStartupBlocked(message, blockerPaths) {
+    const marker = {
+        recordedAt: Date.now(),
+        message,
+        blockers: blockerPaths.map(fingerprint),
+    };
+    try {
+        atomicWriteJson(daemonStartupBlockMarkerPath(), marker);
+    }
+    catch { /* best effort */ }
+}
+/** Clear the marker — for a repair path that knows startup should be retried
+ *  (a successful migration, a successful daemon start, an explicit reset). */
+export function clearDaemonStartupBlocked() {
+    try {
+        rmSync(daemonStartupBlockMarkerPath(), { force: true });
+    }
+    catch { /* best effort */ }
+}
+/** The recorded block message when startup is still blocked by the SAME corpus
+ *  that produced it, or null when nothing is recorded or the corpus has since
+ *  changed. A changed corpus retires the marker as a side effect of the read,
+ *  so the very next spawner is free to try again.
+ *
+ *  A marker naming no blocker paths at all is not self-healing — nothing about
+ *  it can be observed to change — so it is treated as advisory only and
+ *  retired on read rather than left to wedge autostart forever. */
+export function daemonStartupBlocked() {
+    const marker = readJsonOrNull(daemonStartupBlockMarkerPath());
+    if (marker === null || typeof marker.message !== 'string' || !Array.isArray(marker.blockers))
+        return null;
+    if (marker.blockers.length === 0) {
+        clearDaemonStartupBlocked();
+        return null;
+    }
+    const changed = marker.blockers.some((recorded) => !sameFingerprint(recorded, fingerprint(recorded.path)));
+    if (changed) {
+        clearDaemonStartupBlocked();
+        return null;
+    }
+    return marker.message;
+}
+/** The blocked-migration failure, wherever it ended up in a thrown value.
+ *  Startup releases its ownership claim before rethrowing, and a claim release
+ *  that ALSO fails is reported as an aggregate — so the one error the spawner
+ *  must react to can arrive nested. Matching only the bare instance would let
+ *  the aggregate path fall through to a generic exit, and every suppression
+ *  downstream is keyed on getting that exit code right. */
+export function blockedMigrationIn(error) {
+    if (error instanceof MigrationBlockedError)
+        return error;
+    if (error instanceof AggregateError) {
+        for (const nested of error.errors) {
+            const found = blockedMigrationIn(nested);
+            if (found !== null)
+                return found;
+        }
+    }
+    return null;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@north-light/crouter",
-  "version": "0.3.255",
+  "version": "0.3.256",
   "description": "crtr — agent runtime with memory, plugins, and marketplaces",
   "type": "module",
   "main": "dist/index.js",

package/runtime.lock.json

@@ -1,12 +1,12 @@
 {
   "name": "@north-light/crouter",
-  "version": "0.3.255",
+  "version": "0.3.256",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@north-light/crouter",
-      "version": "0.3.255",
+      "version": "0.3.256",
       "hasInstallScript": true,
       "license": "MIT",
       "dependencies": {