@crouton-kit/crouter 0.3.63 → 0.3.65

package/dist/core/__tests__/full/broker-crash-teardown.test.js

@@ -44,7 +44,21 @@ import { existsSync, mkdirSync, writeFileSync } from 'node:fs';
 import { join } from 'node:path';
 import { createHarness } from '../helpers/harness.js';
 import { subscribe } from '../../canvas/canvas.js';
-import { isPidAlive } from '../../canvas/pid.js';
+import { isPidAlive, capturePidIdentities } from '../../canvas/pid.js';
+// Diagnostic-only, gated behind CRTR_DEBUG_PID_LIVENESS (same env var as the
+// isRecordedPidAlive trace in pid.ts): logs whether the SIGKILLed pid reads
+// alive and what identity ps reports for it, right at the two moments that
+// matter for the intermittent Linux-only crash-revive wedge — directly after
+// confirming the kill and again just before the grace-elapsed tick — so a real
+// run shows whether the pid was reused during the grace window. No effect on
+// test behavior when the env var is unset.
+function debugLogPidState(label, pid) {
+    if (!process.env.CRTR_DEBUG_PID_LIVENESS)
+        return;
+    const identities = capturePidIdentities([pid]);
+    const identity = identities === null ? 'probe-null' : identities.has(pid) ? JSON.stringify(identities.get(pid)) : 'no-entry';
+    process.stderr.write(`[broker-crash-teardown] ${label}: pid=${pid} isPidAlive=${isPidAlive(pid)} capturePidIdentities=${identity}\n`);
+}
 // crtrd.ts module const (not exported): the fresh-pi-boot grace window the daemon
 // waits before grace-reviving / boot-failing a pi observed dead. Reference:
 // crtrd.ts `REVIVE_GRACE_MS = 20_000`.
@@ -81,9 +95,11 @@ test('CRASH → grace-revive RESUME (one-writer), then clean teardown on close',
     // Actions run: crouter-development/prove-ci-fixes-in-actions). The waitFor's
     // successful resolution already IS the proof of deadness at this point.
     await h.waitFor(() => !isPidAlive(oldPid), { label: 'crashed broker pid is dead' });
+    debugLogPidState('after waitFor(!isPidAlive(oldPid))', oldPid);
     const NOW = 5_000_000;
     await h.tick(NOW); // pid dead, intent null → handleBrokerLiveness → handleLiveWindow marks pending
     assert.equal(h.bootCount(id), boots, 'inside the grace window → NOT yet revived');
+    debugLogPidState('before grace-elapsed tick', oldPid);
     await h.tick(NOW + REVIVE_GRACE_MS + 1); // grace elapsed → reviveNode(resume:true)
     // This is the file's SECOND real broker boot (crash-revive), on an
     // oversubscribed CI runner sharing the host with the other full-tier files'

package/dist/core/__tests__/helpers/harness.js

@@ -415,14 +415,32 @@ export async function createHarness(opts = {}) {
             const minCount = o.minCount ?? 1;
             const bootsPath = join(nodeDir(nodeId), 'fake-pi.boots.jsonl');
             const errPath = join(nodeDir(nodeId), 'fake-pi.error');
-            const lines = await waitFor(() => {
-                const ls = readLines(bootsPath);
-                return ls.length >= minCount ? ls : null;
-            }, {
-                timeoutMs: o.timeoutMs ?? 30_000,
-                label: `fake-pi boot >= ${minCount} for ${nodeId}` +
-                    (existsSync(errPath) ? ` (error file: ${readFileSync(errPath, 'utf8')})` : ''),
-            });
+            let lines;
+            try {
+                lines = await waitFor(() => {
+                    const ls = readLines(bootsPath);
+                    return ls.length >= minCount ? ls : null;
+                }, {
+                    timeoutMs: o.timeoutMs ?? 30_000,
+                    label: `fake-pi boot >= ${minCount} for ${nodeId}` +
+                        (existsSync(errPath) ? ` (error file: ${readFileSync(errPath, 'utf8')})` : ''),
+                });
+            }
+            catch (err) {
+                // Failure-path-only diagnostic: an intermittent, Linux-CI-only hang
+                // here (recorded-pid-reuse fooling the daemon's revive liveness gate)
+                // was invisible for three CI iterations because the timeout error
+                // alone doesn't say WHY the boot never happened. Dump the node row's
+                // recorded liveness fields plus the current boot count so a future
+                // failing run on Actions is self-explaining from the log alone.
+                closeDb();
+                const n = getNode(nodeId);
+                process.stderr.write(`[awaitBoot] timeout for ${nodeId}: pi_pid=${n?.pi_pid ?? 'null'} ` +
+                    `pi_pid_identity=${n?.pi_pid_identity ?? 'null'} status=${n?.status ?? 'null'} ` +
+                    `intent=${n?.intent ?? 'null'} pi_session_id=${n?.pi_session_id ?? 'null'} ` +
+                    `bootCount=${bootCount(nodeId)} minCount=${minCount}\n`);
+                throw err;
+            }
             const boot = JSON.parse(lines[lines.length - 1]);
             if (typeof boot.pid === 'number')
                 pidsToKill.add(boot.pid);

package/dist/core/canvas/pid.d.ts

@@ -13,6 +13,20 @@
  *  out under a long-lived spawning process (see `isZombie`'s doc) never comes
  *  back on its own. */
 export declare function isPidAlive(pid: number | null | undefined): boolean;
+/** Like `isPidAlive`, but guards a RECORDED node pid against PID REUSE using the
+ *  launch-time identity baseline (`pi_pid_identity`, captured by `recordPid`).
+ *  On a heavily-forking host — esp. Linux, where low pids recycle fast — a
+ *  broker's recorded pid can be reused by an unrelated process after the broker
+ *  dies; a bare `isPidAlive` then reports the STRANGER as "our broker alive",
+ *  wedging every daemon supervise/revive decision (the daemon never revives a
+ *  row that looks alive; `reviveNode`'s double-launch guard never relaunches).
+ *  Returns false the instant the pid's CURRENT identity is POSITIVELY known to
+ *  differ from `expectedIdentity` (the recycle case). Fail-open everywhere else
+ *  — no baseline (legacy row / capture failed at launch), `ps` probe failure, or
+ *  a pid present-but-unreadable — exactly matching `captureTeardownSnapshot`'s
+ *  discipline: a MISMATCH is the ONLY thing that reads dead, never an absence of
+ *  evidence. */
+export declare function isRecordedPidAlive(pid: number | null | undefined, expectedIdentity: string | null | undefined): boolean;
 /** SIGTERM a process GROUP by pid (the negative-pid convention) — best-effort,
  *  swallowing ESRCH (already gone). Shared by triggers.ts (cancel-time reap of
  *  an in-flight predicate evaluation) and daemon/predicate-eval.ts (per-eval

package/dist/core/canvas/pid.js

@@ -60,6 +60,77 @@ export function isPidAlive(pid) {
     }
     return !isZombie(pid);
 }
+/** Like `isPidAlive`, but guards a RECORDED node pid against PID REUSE using the
+ *  launch-time identity baseline (`pi_pid_identity`, captured by `recordPid`).
+ *  On a heavily-forking host — esp. Linux, where low pids recycle fast — a
+ *  broker's recorded pid can be reused by an unrelated process after the broker
+ *  dies; a bare `isPidAlive` then reports the STRANGER as "our broker alive",
+ *  wedging every daemon supervise/revive decision (the daemon never revives a
+ *  row that looks alive; `reviveNode`'s double-launch guard never relaunches).
+ *  Returns false the instant the pid's CURRENT identity is POSITIVELY known to
+ *  differ from `expectedIdentity` (the recycle case). Fail-open everywhere else
+ *  — no baseline (legacy row / capture failed at launch), `ps` probe failure, or
+ *  a pid present-but-unreadable — exactly matching `captureTeardownSnapshot`'s
+ *  discipline: a MISMATCH is the ONLY thing that reads dead, never an absence of
+ *  evidence. */
+export function isRecordedPidAlive(pid, expectedIdentity) {
+    const debug = Boolean(process.env.CRTR_DEBUG_PID_LIVENESS);
+    const alive = isPidAlive(pid);
+    if (!alive) {
+        if (debug)
+            logPidLivenessDecision(pid, alive, expectedIdentity, 'no-probe', 'isPidAlive=false → dead', false);
+        return false; // gone/zombie → dead (unchanged path)
+    }
+    if (expectedIdentity == null) {
+        if (debug)
+            logPidLivenessDecision(pid, alive, expectedIdentity, 'no-probe', 'no baseline → trust isPidAlive', true);
+        return true; // no baseline → trust isPidAlive (legacy behavior)
+    }
+    const current = capturePidIdentities([pid]);
+    if (current == null) {
+        if (debug)
+            logPidLivenessDecision(pid, alive, expectedIdentity, 'probe-null', 'ps probe failed → fail-open alive', true);
+        return true; // probe itself failed → fail-open
+    }
+    const actual = current.get(pid);
+    if (actual === undefined) {
+        if (debug)
+            logPidLivenessDecision(pid, alive, expectedIdentity, 'no-entry', 'no identity row for pid (race) → fail-open alive', true);
+        return true; // present but no identity row (race) → fail-open
+    }
+    const result = actual === expectedIdentity; // different identity → REUSED → dead
+    if (debug)
+        logPidLivenessDecision(pid, alive, expectedIdentity, 'value', result ? 'identity matches expected → alive' : 'identity differs from expected → REUSED → dead', result, actual);
+    return result;
+}
+/** Diagnostic-only decision trace for `isRecordedPidAlive`, gated entirely
+ *  behind `CRTR_DEBUG_PID_LIVENESS` — callers must check the env var
+ *  themselves before calling this so the (tiny) formatting cost is paid ONLY
+ *  when the env var is set; this function does no env check of its own.
+ *  Written to trace the intermittent Linux-only CI crash-revive wedge
+ *  (`broker-crash-teardown.test.ts`'s 'CRASH → grace-revive RESUME'): the
+ *  goal is for a single stderr line per call to make BOTH a passing run ("pid
+ *  read dead → revive") and a wedged run ("read alive via <branch>") fully
+ *  self-explaining, without changing any return value. `capturePidIdentities`
+ *  outcome is reported as one of three DISTINCT cases — `probe-null` (the
+ *  `ps` probe itself failed), `no-entry` (probe succeeded but the map has no
+ *  row for this pid), or `value` (probe succeeded and found an identity,
+ *  printed verbatim) — plus `no-probe` for the two branches that return
+ *  before `capturePidIdentities` is ever called. Kept as permanent
+ *  observability for this primitive, not a throwaway debug print: pid-reuse
+ *  liveness is subtle enough that the next person chasing a wedge here should
+ *  find this already in place. */
+function logPidLivenessDecision(pid, isPidAliveResult, expectedIdentity, probeOutcome, branch, returned, actualIdentity) {
+    const parts = [
+        `pid=${String(pid)}`,
+        `isPidAlive=${isPidAliveResult}`,
+        `expectedIdentity=${expectedIdentity === undefined ? 'undefined' : expectedIdentity === null ? 'null' : JSON.stringify(expectedIdentity)}`,
+        `capturePidIdentities=${probeOutcome}${probeOutcome === 'value' ? `(${JSON.stringify(actualIdentity)})` : ''}`,
+        `branch=${JSON.stringify(branch)}`,
+        `returned=${returned}`,
+    ];
+    process.stderr.write(`[isRecordedPidAlive] ${parts.join(' ')}\n`);
+}
 /** SIGTERM a process GROUP by pid (the negative-pid convention) — best-effort,
  *  swallowing ESRCH (already gone). Shared by triggers.ts (cancel-time reap of
  *  an in-flight predicate evaluation) and daemon/predicate-eval.ts (per-eval

package/dist/core/runtime/revive.js

@@ -29,7 +29,7 @@ import { fanDoctrineWake } from './close.js';
 import { buildPiArgv } from './launch.js';
 import { buildReviveKickoff, drainBearings } from './kickoff.js';
 import { headlessBrokerHost } from './host.js';
-import { isPidAlive } from '../canvas/pid.js';
+import { isRecordedPidAlive } from '../canvas/pid.js';
 import { reconcileBootLiveness } from '../canvas/boot.js';
 import { clearFault } from './fault.js';
 import { clearInjectedDocs } from '../substrate/injected-store.js';
@@ -89,11 +89,14 @@ export function reviveNode(nodeId, opts) {
     // actually matters, so it's safe to call on every revive regardless of
     // entry point (daemon tick, or a direct/manual `node lifecycle revive`).
     reconcileBootLiveness();
-    // Double-revive guard: the broker's isAlive IS isPidAlive(pi_pid), so a node
-    // whose broker pid is still running was already revived by another path —
-    // re-launching would put a SECOND broker on the same session file. No-op.
+    // Double-revive guard: the broker's isAlive is identity-aware liveness on
+    // pi_pid, so a node whose broker pid is still running (and CONFIRMED to
+    // still be that same broker, not a reused pid) was already revived by
+    // another path — re-launching would put a SECOND broker on the same
+    // session file. No-op. A bare isPidAlive would misread a reused pid (heavy
+    // forking, esp. Linux) as "still alive" and skip the relaunch forever.
     const live = getNode(nodeId) ?? meta;
-    if (isPidAlive(live.pi_pid)) {
+    if (isRecordedPidAlive(live.pi_pid, live.pi_pid_identity)) {
         return {
             window: null,
             session: live.tmux_session ?? null,

package/dist/core/substrate/render.js

@@ -430,7 +430,8 @@ const MEMORY_USAGE_GUIDANCE = 'When your task matches a knowledge doc or prefere
     "will make without it, so doing the relevant work without consulting it defeats the doc's " +
     'whole purpose. ' +
     'Before saving any memory, check for an existing doc that already covers it — update that ' +
-    'doc rather than creating a duplicate; delete memories that turn out to be wrong. ' +
+    'doc rather than creating a duplicate, rewriting it as if for the first time instead of ' +
+    'appending the new truth beneath the old; delete memories that turn out to be wrong. ' +
     "Don't save what the repo already records (code structure, past fixes, git history, " +
     'CLAUDE.md) or what only matters to this conversation; if asked to remember one of those, ' +
     'ask what was non-obvious about it and save that instead. Docs auto-surfaced in ' +

package/dist/daemon/crtrd.js

@@ -64,7 +64,7 @@ import { FAULT_QUIET_MS, readFault, recordFault } from '../core/runtime/fault.js
 import { fanDoctrineWake } from '../core/runtime/close.js';
 import { logger } from '../core/log.js';
 export { FAULT_QUIET_MS };
-import { isPidAlive } from '../core/canvas/pid.js';
+import { isPidAlive, isRecordedPidAlive } from '../core/canvas/pid.js';
 import { reconcileBootLiveness } from '../core/canvas/boot.js';
 import { listLivePanes, tearDownNode } from '../core/runtime/placement.js';
 import { reviveNode } from '../core/runtime/revive.js';
@@ -704,7 +704,15 @@ async function handleNodeLiveness(row, now, revivedThisTick) {
     // enforce §H: handleYieldStall no-ops unless this node yielded (intent=refresh),
     // its turn is over, and the engine never exited — past the grace it kills the
     // engine so the dead-pid refresh branch below revives it fresh.
-    if (pid != null && isPidAlive(pid)) {
+    //
+    // Liveness here is IDENTITY-aware, not a bare isPidAlive: on a heavily-
+    // forking host (esp. Linux, where low pids recycle fast) a just-dead
+    // broker's pid can be reused by an unrelated process before this tick
+    // runs, and a bare isPidAlive would report the STRANGER as "our broker
+    // alive" — clearing the grace clock forever and wedging revive (the
+    // recorded-pid-reuse hang). isRecordedPidAlive guards against that with
+    // the launch-time identity baseline (`row.pi_pid_identity`).
+    if (pid != null && isRecordedPidAlive(pid, row.pi_pid_identity)) {
         unhealthySince.delete(id);
         clearStrandedRelaunchState(id); // a live engine means the stranding resolved
         handleYieldStall(row, pid, now);
@@ -1149,8 +1157,10 @@ export async function superviseTick(now = Date.now()) {
             // The in-process inbox-watcher only owns delivery while the engine is LIVE.
             // A released node's broker is DEAD (it freed itself while dormant), so it
             // has no watcher and the daemon must wake it. Gate the skip on engine
-            // liveness alone — viewer panes are irrelevant here.
-            if (r.pi_pid != null && isPidAlive(r.pi_pid))
+            // liveness alone — viewer panes are irrelevant here. Identity-aware: a
+            // bare isPidAlive would misread a reused pid as "still alive" and skip
+            // the revive forever, stranding the node on its unseen inbox entry.
+            if (r.pi_pid != null && isRecordedPidAlive(r.pi_pid, r.pi_pid_identity))
                 continue;
             const entries = readInboxSince(row.node_id, readCursor(row.node_id));
             if (entries.length > 0) {
@@ -1214,7 +1224,9 @@ export async function superviseTick(now = Date.now()) {
                 }
                 if (revivedThisTick.has(target.node_id))
                     continue;
-                if (target.pi_pid != null && isPidAlive(target.pi_pid))
+                // Identity-aware: a bare isPidAlive would misread a reused pid as
+                // "still alive" and skip the wake, stranding the scheduled trigger.
+                if (target.pi_pid != null && isRecordedPidAlive(target.pi_pid, target.pi_pid_identity))
                     continue;
                 process.stderr.write(`[crtrd] wake ${target.node_id} (bare alarm)\n`);
                 reviveNode(target.node_id, { resume: false, wakeReason: wakeOriginFrom(t) });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@crouton-kit/crouter",
-  "version": "0.3.63",
+  "version": "0.3.65",
   "description": "crtr — agent runtime with memory, plugins, and marketplaces",
   "type": "module",
   "main": "dist/index.js",