@crouton-kit/crouter 0.3.64 → 0.3.66

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -384,25 +384,46 @@ export class GraphOverlay {
384
384
  if (idx < 0)
385
385
  idx = Math.max(0, rows.findIndex((r) => r.id === this.self));
386
386
  const cur = rows[idx];
387
+ // Cycle rows (↺ back-refs) carry the SAME id as the real node they echo, so a
388
+ // cursor landing on one resolves back to the earlier real row on the next
389
+ // findIndex — the cursor snaps up and can't pass the echo. They're non-
390
+ // actionable (focusing re-targets the real node you can already reach), so
391
+ // navigation skips them: the cursor only ever rests on a unique, real row.
392
+ const isSelectable = (r) => r !== undefined && !r.cycle;
393
+ const step = (from, dir) => {
394
+ for (let i = from + dir; i >= 0 && i < rows.length; i += dir) {
395
+ if (isSelectable(rows[i]))
396
+ return i;
397
+ }
398
+ return from; // no selectable row that way — hold position
399
+ };
400
+ const edge = (dir) => {
401
+ const start = dir === 1 ? 0 : rows.length - 1;
402
+ for (let i = start; i >= 0 && i < rows.length; i += dir) {
403
+ if (isSelectable(rows[i]))
404
+ return i;
405
+ }
406
+ return idx;
407
+ };
387
408
  if (matchesKey(data, 'j')) {
388
- idx = Math.min(rows.length - 1, idx + 1);
409
+ idx = step(idx, 1);
389
410
  this.cursorId = rows[idx]?.id ?? this.cursorId;
390
411
  this.tui.requestRender();
391
412
  return;
392
413
  }
393
414
  if (matchesKey(data, 'k')) {
394
- idx = Math.max(0, idx - 1);
415
+ idx = step(idx, -1);
395
416
  this.cursorId = rows[idx]?.id ?? this.cursorId;
396
417
  this.tui.requestRender();
397
418
  return;
398
419
  }
399
420
  if (matchesKey(data, 'g')) {
400
- this.cursorId = rows[0]?.id ?? this.cursorId;
421
+ this.cursorId = rows[edge(1)]?.id ?? this.cursorId;
401
422
  this.tui.requestRender();
402
423
  return;
403
424
  }
404
425
  if (matchesKey(data, 'shift+g')) {
405
- this.cursorId = rows[rows.length - 1]?.id ?? this.cursorId;
426
+ this.cursorId = rows[edge(-1)]?.id ?? this.cursorId;
406
427
  this.tui.requestRender();
407
428
  return;
408
429
  }
@@ -20,7 +20,7 @@ import { InputError } from '../core/io.js';
20
20
  import { reviveNode } from '../core/runtime/revive.js';
21
21
  import { waitForBrokerViewSocket } from '../core/runtime/placement.js';
22
22
  import { getNode, fullName } from '../core/canvas/index.js';
23
- import { isPidAlive } from '../core/canvas/pid.js';
23
+ import { recordedPidLiveness } from '../core/canvas/pid.js';
24
24
  import { readFault } from '../core/runtime/fault.js';
25
25
  import { isBusy } from '../core/runtime/busy.js';
26
26
  import { faultNeedsYou } from '../core/canvas/status-glyph.js';
@@ -114,7 +114,14 @@ export const nodeReviveLeaf = defineLeaf({
114
114
  if (now) {
115
115
  const pid = meta.pi_pid;
116
116
  const stall = readFault(nodeId);
117
- if (pid == null || !isPidAlive(pid)) {
117
+ // Identity-aware AND requires CONFIRMED alive — --now's whole job is to
118
+ // SIGTERM this pid, so a bare isPidAlive risks signalling a STRANGER
119
+ // process that reused a dead broker's recycled pid. `dead` (including a
120
+ // positively confirmed reuse) correctly refuses here. `indeterminate`
121
+ // (can't confirm identity either way, e.g. a transient `ps` probe
122
+ // failure) ALSO refuses — only a positively confirmed-alive-and-ours pid
123
+ // is ever signaled by this command.
124
+ if (pid == null || recordedPidLiveness(pid, meta.pi_pid_identity) !== 'alive') {
118
125
  throw new InputError({
119
126
  error: 'not_hanging',
120
127
  message: `${nodeId} has no live broker — nothing to kick. --now is for a HANGING node (live broker parked on an active fault).`,
@@ -44,7 +44,21 @@ import { existsSync, mkdirSync, writeFileSync } from 'node:fs';
44
44
  import { join } from 'node:path';
45
45
  import { createHarness } from '../helpers/harness.js';
46
46
  import { subscribe } from '../../canvas/canvas.js';
47
- import { isPidAlive } from '../../canvas/pid.js';
47
+ import { isPidAlive, capturePidIdentities } from '../../canvas/pid.js';
48
+ // Diagnostic-only, gated behind CRTR_DEBUG_PID_LIVENESS (same env var as the
49
+ // isRecordedPidAlive trace in pid.ts): logs whether the SIGKILLed pid reads
50
+ // alive and what identity ps reports for it, right at the two moments that
51
+ // matter for the intermittent Linux-only crash-revive wedge — directly after
52
+ // confirming the kill and again just before the grace-elapsed tick — so a real
53
+ // run shows whether the pid was reused during the grace window. No effect on
54
+ // test behavior when the env var is unset.
55
+ function debugLogPidState(label, pid) {
56
+ if (!process.env.CRTR_DEBUG_PID_LIVENESS)
57
+ return;
58
+ const identities = capturePidIdentities([pid]);
59
+ const identity = identities === null ? 'probe-null' : identities.has(pid) ? JSON.stringify(identities.get(pid)) : 'no-entry';
60
+ process.stderr.write(`[broker-crash-teardown] ${label}: pid=${pid} isPidAlive=${isPidAlive(pid)} capturePidIdentities=${identity}\n`);
61
+ }
48
62
  // crtrd.ts module const (not exported): the fresh-pi-boot grace window the daemon
49
63
  // waits before grace-reviving / boot-failing a pi observed dead. Reference:
50
64
  // crtrd.ts `REVIVE_GRACE_MS = 20_000`.
@@ -81,9 +95,11 @@ test('CRASH → grace-revive RESUME (one-writer), then clean teardown on close',
81
95
  // Actions run: crouter-development/prove-ci-fixes-in-actions). The waitFor's
82
96
  // successful resolution already IS the proof of deadness at this point.
83
97
  await h.waitFor(() => !isPidAlive(oldPid), { label: 'crashed broker pid is dead' });
98
+ debugLogPidState('after waitFor(!isPidAlive(oldPid))', oldPid);
84
99
  const NOW = 5_000_000;
85
100
  await h.tick(NOW); // pid dead, intent null → handleBrokerLiveness → handleLiveWindow marks pending
86
101
  assert.equal(h.bootCount(id), boots, 'inside the grace window → NOT yet revived');
102
+ debugLogPidState('before grace-elapsed tick', oldPid);
87
103
  await h.tick(NOW + REVIVE_GRACE_MS + 1); // grace elapsed → reviveNode(resume:true)
88
104
  // This is the file's SECOND real broker boot (crash-revive), on an
89
105
  // oversubscribed CI runner sharing the host with the other full-tier files'
@@ -0,0 +1,67 @@
1
+ // The on-read POSITIONAL walk must never treat the crouter home (`~/.crouter`,
2
+ // the user scope root) as a project ancestor. Canvas runtime artifacts live at
3
+ // `~/.crouter/canvas/nodes/<id>/{reports,context}/...`, so a naive ancestor
4
+ // walk from such a read climbs THROUGH `~/.crouter` and would inject whatever
5
+ // lives at `~/.crouter/.crouter/memory/` — e.g. a differently-scoped node whose
6
+ // cwd resolved its private project store there. That leaks one node's private
7
+ // memory into every unrelated node that reads a canvas file. This test pins the
8
+ // fence: reading a canvas artifact must not surface a store nested in the
9
+ // crouter home.
10
+ //
11
+ // Run: node --import tsx/esm --test src/core/__tests__/on-read-crouter-home-fence.test.ts
12
+ import { test, before, beforeEach, after } from 'node:test';
13
+ import assert from 'node:assert/strict';
14
+ import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from 'node:fs';
15
+ import { tmpdir } from 'node:os';
16
+ import { join } from 'node:path';
17
+ import { closeDb } from '../canvas/db.js';
18
+ import { resetScopeCache } from '../scope.js';
19
+ import { spawnNode } from '../runtime/nodes.js';
20
+ import { renderOnReadDocs } from '../substrate/on-read.js';
21
+ let canvasHome;
22
+ let fakeHome;
23
+ let prevHomeEnv;
24
+ before(() => {
25
+ prevHomeEnv = process.env['HOME'];
26
+ });
27
+ beforeEach(() => {
28
+ closeDb();
29
+ resetScopeCache();
30
+ // A private user HOME so `homedir()`/`userScopeRoot()` resolve into our
31
+ // sandbox, not the real machine.
32
+ fakeHome = mkdtempSync(join(tmpdir(), 'crtr-fence-home-'));
33
+ process.env['HOME'] = fakeHome;
34
+ // The node graph (canvas.db) lives in its own CRTR_HOME so spawnNode does not
35
+ // touch the fake user home; the read path we test is constructed by hand.
36
+ canvasHome = mkdtempSync(join(tmpdir(), 'crtr-fence-canvas-'));
37
+ process.env['CRTR_HOME'] = canvasHome;
38
+ });
39
+ after(() => {
40
+ closeDb();
41
+ resetScopeCache();
42
+ rmSync(canvasHome, { recursive: true, force: true });
43
+ rmSync(fakeHome, { recursive: true, force: true });
44
+ delete process.env['CRTR_HOME'];
45
+ if (prevHomeEnv === undefined)
46
+ delete process.env['HOME'];
47
+ else
48
+ process.env['HOME'] = prevHomeEnv;
49
+ });
50
+ test('a store nested in the crouter home does not leak into a canvas-artifact read', () => {
51
+ const node = spawnNode({ kind: 'general', cwd: fakeHome, parent: null }).node_id;
52
+ // A private store misresolved into the crouter home: `~/.crouter/.crouter/memory/`.
53
+ const leakedStore = join(fakeHome, '.crouter', '.crouter', 'memory');
54
+ mkdirSync(leakedStore, { recursive: true });
55
+ writeFileSync(join(leakedStore, 'private-thing.md'), '---\nkind: knowledge\n' +
56
+ "when-and-why-to-read: When X, read this because Y.\n" +
57
+ 'file-read-visibility: content\n---\n' +
58
+ 'PRIVATE ASSISTANT MEMORY MUST NOT LEAK\n');
59
+ // A canvas runtime artifact under the crouter home — the shape of every node
60
+ // report/context file the harness auto-reads.
61
+ const reportFile = join(fakeHome, '.crouter', 'canvas', 'nodes', 'somenode', 'reports', 'r.md');
62
+ mkdirSync(join(fakeHome, '.crouter', 'canvas', 'nodes', 'somenode', 'reports'), { recursive: true });
63
+ writeFileSync(reportFile, '# a node report\n');
64
+ const rendered = renderOnReadDocs(node, reportFile, new Set());
65
+ assert.ok(!rendered.includes('PRIVATE ASSISTANT MEMORY MUST NOT LEAK'), `crouter-home-nested store must not surface on a canvas read; got: ${rendered}`);
66
+ assert.ok(!rendered.includes('private-thing'), `crouter-home-nested store name must not surface on a canvas read; got: ${rendered}`);
67
+ });
@@ -13,19 +13,77 @@
13
13
  * out under a long-lived spawning process (see `isZombie`'s doc) never comes
14
14
  * back on its own. */
15
15
  export declare function isPidAlive(pid: number | null | undefined): boolean;
16
- /** Like `isPidAlive`, but guards a RECORDED node pid against PID REUSE using the
17
- * launch-time identity baseline (`pi_pid_identity`, captured by `recordPid`).
18
- * On a heavily-forking host — esp. Linux, where low pids recycle fast — a
19
- * broker's recorded pid can be reused by an unrelated process after the broker
20
- * dies; a bare `isPidAlive` then reports the STRANGER as "our broker alive",
21
- * wedging every daemon supervise/revive decision (the daemon never revives a
22
- * row that looks alive; `reviveNode`'s double-launch guard never relaunches).
23
- * Returns false the instant the pid's CURRENT identity is POSITIVELY known to
24
- * differ from `expectedIdentity` (the recycle case). Fail-open everywhere else
25
- * — no baseline (legacy row / capture failed at launch), `ps` probe failure, or
26
- * a pid present-but-unreadable exactly matching `captureTeardownSnapshot`'s
27
- * discipline: a MISMATCH is the ONLY thing that reads dead, never an absence of
28
- * evidence. */
16
+ /** THE single precision-aware identity compare, used EVERYWHERE two
17
+ * `composeIdentity` fingerprints are compared (this module's own
18
+ * `recordedPidLiveness`, `captureTeardownSnapshot`'s reuse check, and
19
+ * `killProcessTreePids`'s reuse check) centralized so no call site can
20
+ * drift from this rule (review Major #1, Round 2 of the pid-reuse-liveness
21
+ * fix).
22
+ *
23
+ * The hazard this closes: `composeIdentity` appends the `#<ticks>` Linux
24
+ * discriminator only when `/proc/<pid>/stat` happens to be readable at
25
+ * capture time so the SAME live process can be recorded plain `lstart` on
26
+ * one capture and `lstart#ticks` on another (a transient `/proc` read
27
+ * failure, or capturing on a platform/container without procfs). A naive
28
+ * `===` then reads that as a reuse (different string) and reports `dead` for
29
+ * a broker that never actually died — the false-DEAD direction, which can
30
+ * make `reviveNode`'s double-launch guard wrongly relaunch a SECOND broker
31
+ * onto a still-live session.
32
+ *
33
+ * Rule: split both on `#`. Bases (`lstart`) differ → definitely NOT a match
34
+ * (different process, or the same pid reused in a different second). Bases
35
+ * match and BOTH carry a `#ticks` suffix → match iff the suffixes are equal
36
+ * (this still catches genuine same-second reuse, the case `procStartTicks`
37
+ * exists for). Bases match and EITHER lacks the suffix → coarse-vs-fine,
38
+ * can't distinguish → MATCH (fail-open to same-process, matching this
39
+ * module's universal discipline: only a POSITIVE mismatch counts as reuse,
40
+ * never an absence of evidence). */
41
+ export declare function identitiesMatch(a: string, b: string): boolean;
42
+ /** Three-valued liveness verdict for a RECORDED node pid, identity-guarded
43
+ * against PID REUSE using the launch-time baseline (`pi_pid_identity`,
44
+ * captured by `recordPid`). On a heavily-forking host — esp. Linux, where
45
+ * low pids recycle fast — a broker's recorded pid can be reused by an
46
+ * unrelated process (or the `ps` probe confirming it can itself
47
+ * intermittently fail) within milliseconds of the broker dying; collapsing
48
+ * "confirmed alive", "confirmed dead/reused", and "couldn't confirm either
49
+ * way" into one boolean forces every caller to fail open to ALIVE on the
50
+ * third case — which, upstream in the daemon's grace-clock bookkeeping, is
51
+ * worse than misreporting one tick: a single such read WIPES the daemon's
52
+ * memory that the node had already been observed dead, restarting the whole
53
+ * grace window from scratch (the confirmed Round-2 crash-revive wedge — see
54
+ * `crtrd.ts`'s `handleNodeLiveness`). Exposing the third state lets callers
55
+ * decide for themselves whether "can't confirm" should act like alive or
56
+ * dead for their own direction of risk.
57
+ *
58
+ * Reuses the existing primitives — this is a classification layered on top
59
+ * of `isPidAlive` + `capturePidIdentities`, never a new `ps` parse:
60
+ * - `!isPidAlive(pid)` → `'dead'`. Load-bearing: this is what preserves
61
+ * zombie handling — `isPidAlive` already treats a gone pid AND a zombie
62
+ * (SIGKILLed detached broker awaiting reap) as dead, which is the entire
63
+ * reason a wedged zombie broker still gets revived. Do not bypass it.
64
+ * - `expectedIdentity == null` (legacy row / launch capture failed) →
65
+ * `'alive'` (preserve pre-identity behavior — no baseline to check
66
+ * against).
67
+ * - `ps` probe itself failed (`capturePidIdentities` → `null`) →
68
+ * `'indeterminate'`. THIS is the branch the Round-2 fix changes: it used
69
+ * to fail open to `true` (alive) here, which is the confirmed root cause
70
+ * of the wedge.
71
+ * - probe succeeded but has no row for this pid (alive per signal-0 a
72
+ * moment ago, gone now) → `'dead'`.
73
+ * - probe succeeded and the identity is present → `'alive'` iff
74
+ * `identitiesMatch` says so, else `'dead'` (positively reused). */
75
+ export type RecordedPidLiveness = 'alive' | 'dead' | 'indeterminate';
76
+ export declare function recordedPidLiveness(pid: number | null | undefined, expectedIdentity: string | null | undefined): RecordedPidLiveness;
77
+ /** Boolean teardown-direction adapter over `recordedPidLiveness`, kept so
78
+ * every pre-existing signalling caller stays behavior-identical: `dead` is
79
+ * the ONLY thing that reads false. `indeterminate` (can't confirm either
80
+ * way) reads true alongside `alive` — never signal a maybe-ours pid on mere
81
+ * uncertainty, matching this module's universal discipline (a POSITIVE
82
+ * mismatch is the only thing that changes behavior, never an absence of
83
+ * evidence). Callers that need to distinguish "confirmed alive" from
84
+ * "can't confirm" (e.g. a double-launch guard, where treating uncertainty as
85
+ * alive risks silently skipping a needed relaunch) must call
86
+ * `recordedPidLiveness` directly instead of this adapter. */
29
87
  export declare function isRecordedPidAlive(pid: number | null | undefined, expectedIdentity: string | null | undefined): boolean;
30
88
  /** SIGTERM a process GROUP by pid (the negative-pid convention) — best-effort,
31
89
  * swallowing ESRCH (already gone). Shared by triggers.ts (cancel-time reap of
@@ -60,31 +60,118 @@ export function isPidAlive(pid) {
60
60
  }
61
61
  return !isZombie(pid);
62
62
  }
63
- /** Like `isPidAlive`, but guards a RECORDED node pid against PID REUSE using the
64
- * launch-time identity baseline (`pi_pid_identity`, captured by `recordPid`).
65
- * On a heavily-forking host — esp. Linux, where low pids recycle fast a
66
- * broker's recorded pid can be reused by an unrelated process after the broker
67
- * dies; a bare `isPidAlive` then reports the STRANGER as "our broker alive",
68
- * wedging every daemon supervise/revive decision (the daemon never revives a
69
- * row that looks alive; `reviveNode`'s double-launch guard never relaunches).
70
- * Returns false the instant the pid's CURRENT identity is POSITIVELY known to
71
- * differ from `expectedIdentity` (the recycle case). Fail-open everywhere else
72
- * no baseline (legacy row / capture failed at launch), `ps` probe failure, or
73
- * a pid present-but-unreadable — exactly matching `captureTeardownSnapshot`'s
74
- * discipline: a MISMATCH is the ONLY thing that reads dead, never an absence of
75
- * evidence. */
76
- export function isRecordedPidAlive(pid, expectedIdentity) {
77
- if (!isPidAlive(pid))
78
- return false; // gone/zombie dead (unchanged path)
79
- if (expectedIdentity == null)
80
- return true; // no baseline trust isPidAlive (legacy behavior)
63
+ /** Split a `composeIdentity` fingerprint (`lstart` or `lstart#ticks`) into its
64
+ * coarse (`lstart`, second-granular, portable) and fine (`ticks`, Linux-only
65
+ * jiffies, best-effort) parts. `#` never appears in `lstart` itself (a `ps`
66
+ * timestamp), so the first `#` unambiguously separates them. */
67
+ function splitIdentity(identity) {
68
+ const i = identity.indexOf('#');
69
+ return i === -1 ? { base: identity, ticks: undefined } : { base: identity.slice(0, i), ticks: identity.slice(i + 1) };
70
+ }
71
+ /** THE single precision-aware identity compare, used EVERYWHERE two
72
+ * `composeIdentity` fingerprints are compared (this module's own
73
+ * `recordedPidLiveness`, `captureTeardownSnapshot`'s reuse check, and
74
+ * `killProcessTreePids`'s reuse check) centralized so no call site can
75
+ * drift from this rule (review Major #1, Round 2 of the pid-reuse-liveness
76
+ * fix).
77
+ *
78
+ * The hazard this closes: `composeIdentity` appends the `#<ticks>` Linux
79
+ * discriminator only when `/proc/<pid>/stat` happens to be readable at
80
+ * capture time so the SAME live process can be recorded plain `lstart` on
81
+ * one capture and `lstart#ticks` on another (a transient `/proc` read
82
+ * failure, or capturing on a platform/container without procfs). A naive
83
+ * `===` then reads that as a reuse (different string) and reports `dead` for
84
+ * a broker that never actually died — the false-DEAD direction, which can
85
+ * make `reviveNode`'s double-launch guard wrongly relaunch a SECOND broker
86
+ * onto a still-live session.
87
+ *
88
+ * Rule: split both on `#`. Bases (`lstart`) differ → definitely NOT a match
89
+ * (different process, or the same pid reused in a different second). Bases
90
+ * match and BOTH carry a `#ticks` suffix → match iff the suffixes are equal
91
+ * (this still catches genuine same-second reuse, the case `procStartTicks`
92
+ * exists for). Bases match and EITHER lacks the suffix → coarse-vs-fine,
93
+ * can't distinguish → MATCH (fail-open to same-process, matching this
94
+ * module's universal discipline: only a POSITIVE mismatch counts as reuse,
95
+ * never an absence of evidence). */
96
+ export function identitiesMatch(a, b) {
97
+ const ia = splitIdentity(a);
98
+ const ib = splitIdentity(b);
99
+ if (ia.base !== ib.base)
100
+ return false;
101
+ if (ia.ticks !== undefined && ib.ticks !== undefined)
102
+ return ia.ticks === ib.ticks;
103
+ return true;
104
+ }
105
+ export function recordedPidLiveness(pid, expectedIdentity) {
106
+ const debug = Boolean(process.env.CRTR_DEBUG_PID_LIVENESS);
107
+ const alive = isPidAlive(pid);
108
+ if (!alive) {
109
+ if (debug)
110
+ logPidLivenessDecision(pid, alive, expectedIdentity, 'no-probe', 'isPidAlive=false → dead', 'dead');
111
+ return 'dead'; // gone/zombie → dead (unchanged path)
112
+ }
113
+ if (expectedIdentity == null) {
114
+ if (debug)
115
+ logPidLivenessDecision(pid, alive, expectedIdentity, 'no-probe', 'no baseline → trust isPidAlive', 'alive');
116
+ return 'alive'; // no baseline → trust isPidAlive (legacy behavior)
117
+ }
81
118
  const current = capturePidIdentities([pid]);
82
- if (current == null)
83
- return true; // probe itself failed → fail-open
119
+ if (current == null) {
120
+ if (debug)
121
+ logPidLivenessDecision(pid, alive, expectedIdentity, 'probe-null', 'ps probe failed → indeterminate', 'indeterminate');
122
+ return 'indeterminate'; // probe itself failed → we don't know (THE fix: was fail-open alive)
123
+ }
84
124
  const actual = current.get(pid);
85
- if (actual === undefined)
86
- return true; // present but no identity row (race) → fail-open
87
- return actual === expectedIdentity; // different identity REUSED → dead
125
+ if (actual === undefined) {
126
+ if (debug)
127
+ logPidLivenessDecision(pid, alive, expectedIdentity, 'no-entry', 'no identity row for pid (race) → dead', 'dead');
128
+ return 'dead'; // isPidAlive said alive but it's gone now → dead
129
+ }
130
+ const matches = identitiesMatch(actual, expectedIdentity);
131
+ if (debug)
132
+ logPidLivenessDecision(pid, alive, expectedIdentity, 'value', matches ? 'identity matches expected → alive' : 'identity differs from expected → REUSED → dead', matches ? 'alive' : 'dead', actual);
133
+ return matches ? 'alive' : 'dead';
134
+ }
135
+ /** Boolean teardown-direction adapter over `recordedPidLiveness`, kept so
136
+ * every pre-existing signalling caller stays behavior-identical: `dead` is
137
+ * the ONLY thing that reads false. `indeterminate` (can't confirm either
138
+ * way) reads true alongside `alive` — never signal a maybe-ours pid on mere
139
+ * uncertainty, matching this module's universal discipline (a POSITIVE
140
+ * mismatch is the only thing that changes behavior, never an absence of
141
+ * evidence). Callers that need to distinguish "confirmed alive" from
142
+ * "can't confirm" (e.g. a double-launch guard, where treating uncertainty as
143
+ * alive risks silently skipping a needed relaunch) must call
144
+ * `recordedPidLiveness` directly instead of this adapter. */
145
+ export function isRecordedPidAlive(pid, expectedIdentity) {
146
+ return recordedPidLiveness(pid, expectedIdentity) !== 'dead';
147
+ }
148
+ /** Diagnostic-only decision trace for `recordedPidLiveness`, gated entirely
149
+ * behind `CRTR_DEBUG_PID_LIVENESS` — callers must check the env var
150
+ * themselves before calling this so the (tiny) formatting cost is paid ONLY
151
+ * when the env var is set; this function does no env check of its own.
152
+ * Written to trace the intermittent Linux-only CI crash-revive wedge
153
+ * (`broker-crash-teardown.test.ts`'s 'CRASH → grace-revive RESUME'): the
154
+ * goal is for a single stderr line per call to make BOTH a passing run ("pid
155
+ * read dead → revive") and a wedged run ("read alive via <branch>") fully
156
+ * self-explaining, without changing any return value. `capturePidIdentities`
157
+ * outcome is reported as one of three DISTINCT cases — `probe-null` (the
158
+ * `ps` probe itself failed), `no-entry` (probe succeeded but the map has no
159
+ * row for this pid), or `value` (probe succeeded and found an identity,
160
+ * printed verbatim) — plus `no-probe` for the two branches that return
161
+ * before `capturePidIdentities` is ever called. Kept as permanent
162
+ * observability for this primitive, not a throwaway debug print: pid-reuse
163
+ * liveness is subtle enough that the next person chasing a wedge here should
164
+ * find this already in place. */
165
+ function logPidLivenessDecision(pid, isPidAliveResult, expectedIdentity, probeOutcome, branch, returned, actualIdentity) {
166
+ const parts = [
167
+ `pid=${String(pid)}`,
168
+ `isPidAlive=${isPidAliveResult}`,
169
+ `expectedIdentity=${expectedIdentity === undefined ? 'undefined' : expectedIdentity === null ? 'null' : JSON.stringify(expectedIdentity)}`,
170
+ `capturePidIdentities=${probeOutcome}${probeOutcome === 'value' ? `(${JSON.stringify(actualIdentity)})` : ''}`,
171
+ `branch=${JSON.stringify(branch)}`,
172
+ `returned=${returned}`,
173
+ ];
174
+ process.stderr.write(`[recordedPidLiveness] ${parts.join(' ')}\n`);
88
175
  }
89
176
  /** SIGTERM a process GROUP by pid (the negative-pid convention) — best-effort,
90
177
  * swallowing ESRCH (already gone). Shared by triggers.ts (cancel-time reap of
@@ -294,7 +381,9 @@ export function captureTeardownSnapshot(rootPid, expectedIdentity) {
294
381
  if (table === null)
295
382
  return { tree: [rootPid], identities: null, reused: false };
296
383
  const currentRootIdentity = table.identities.get(rootPid);
297
- const reused = expectedIdentity != null && currentRootIdentity !== undefined && currentRootIdentity !== expectedIdentity;
384
+ const reused = expectedIdentity != null &&
385
+ currentRootIdentity !== undefined &&
386
+ !identitiesMatch(currentRootIdentity, expectedIdentity);
298
387
  if (reused)
299
388
  return { tree: [], identities: null, reused: true };
300
389
  const tree = [rootPid, ...descendantPidsFrom(rootPid, table.childrenOf)];
@@ -362,7 +451,7 @@ export function killProcessTreePids(pids, signal = 'SIGTERM', identities) {
362
451
  if (identities != null && current != null) {
363
452
  const expected = identities.get(pid);
364
453
  const actual = current.get(pid);
365
- const reused = expected !== undefined && actual !== undefined && actual !== expected;
454
+ const reused = expected !== undefined && actual !== undefined && !identitiesMatch(actual, expected);
366
455
  if (reused)
367
456
  continue;
368
457
  }
@@ -13,7 +13,7 @@
13
13
  // self-guards the double-spawn. The selection predicate (`isDisconnected`) is
14
14
  // pure so the scope rules are unit-testable without booting a single process.
15
15
  import { listNodes, getNode } from '../canvas/index.js';
16
- import { isPidAlive } from '../canvas/pid.js';
16
+ import { recordedPidLiveness } from '../canvas/pid.js';
17
17
  import { reviveNode } from './revive.js';
18
18
  /** Statuses a node lands in DELIBERATELY: a worker that finished its own work
19
19
  * (`done`) or a node a human closed/reaped (`canceled`). The daemon never
@@ -56,7 +56,13 @@ export function listDisconnected() {
56
56
  const out = [];
57
57
  for (const row of listNodes()) {
58
58
  const meta = getNode(row.node_id);
59
- if (meta !== null && isDisconnected(meta, isPidAlive))
59
+ // IDENTITY-aware AND requires CONFIRMED alive: a bare isPidAlive would
60
+ // misread a recycled pi_pid as "engine running" and SKIP the node, so it
61
+ // never reaches reviveNode's own identity-aware double-launch guard
62
+ // (Round 2 of the pid-reuse-liveness fix, fold-in Major). `dead` OR
63
+ // `indeterminate` both count as disconnected/eligible here — only a
64
+ // positively confirmed-alive read is treated as "still connected".
65
+ if (meta !== null && isDisconnected(meta, (pid) => recordedPidLiveness(pid, meta.pi_pid_identity) === 'alive'))
60
66
  out.push(meta);
61
67
  }
62
68
  return out;
@@ -29,7 +29,7 @@ import { fanDoctrineWake } from './close.js';
29
29
  import { buildPiArgv } from './launch.js';
30
30
  import { buildReviveKickoff, drainBearings } from './kickoff.js';
31
31
  import { headlessBrokerHost } from './host.js';
32
- import { isRecordedPidAlive } from '../canvas/pid.js';
32
+ import { recordedPidLiveness } from '../canvas/pid.js';
33
33
  import { reconcileBootLiveness } from '../canvas/boot.js';
34
34
  import { clearFault } from './fault.js';
35
35
  import { clearInjectedDocs } from '../substrate/injected-store.js';
@@ -89,14 +89,24 @@ export function reviveNode(nodeId, opts) {
89
89
  // actually matters, so it's safe to call on every revive regardless of
90
90
  // entry point (daemon tick, or a direct/manual `node lifecycle revive`).
91
91
  reconcileBootLiveness();
92
- // Double-revive guard: the broker's isAlive is identity-aware liveness on
93
- // pi_pid, so a node whose broker pid is still running (and CONFIRMED to
94
- // still be that same broker, not a reused pid) was already revived by
95
- // another path — re-launching would put a SECOND broker on the same
96
- // session file. No-op. A bare isPidAlive would misread a reused pid (heavy
97
- // forking, esp. Linux) as "still alive" and skip the relaunch forever.
92
+ // Double-revive guard: identity-aware liveness on pi_pid, requiring
93
+ // CONFIRMED alive not the fail-open-to-alive boolean adapter. A node
94
+ // whose broker pid is CONFIRMED still running as that same broker (not a
95
+ // reused pid) was already revived by another path — re-launching would put
96
+ // a SECOND broker on the same session file. No-op. A bare isPidAlive would
97
+ // misread a reused pid (heavy forking, esp. Linux) as "still alive" and
98
+ // skip the relaunch forever; conversely, treating an INDETERMINATE read
99
+ // (can't confirm either way, e.g. a transient `ps` probe failure) as
100
+ // "alive" here would risk the opposite failure — silently skipping a
101
+ // needed relaunch and stranding the node with no engine. So this guard
102
+ // blocks ONLY on a positively confirmed-alive read; `dead` AND
103
+ // `indeterminate` both proceed to relaunch. We accept that tradeoff: an
104
+ // `indeterminate` read CAN come from a genuinely live broker whose identity
105
+ // `ps` probe transiently failed, so proceeding risks a rare double-launch —
106
+ // but stranding a truly-dead node with no engine is the worse failure, and
107
+ // recovering from it is far harder, so we bias toward relaunching.
98
108
  const live = getNode(nodeId) ?? meta;
99
- if (isRecordedPidAlive(live.pi_pid, live.pi_pid_identity)) {
109
+ if (recordedPidLiveness(live.pi_pid, live.pi_pid_identity) === 'alive') {
100
110
  return {
101
111
  window: null,
102
112
  session: live.tmux_session ?? null,
@@ -58,6 +58,7 @@ import { pathExists, readText, walkFiles } from '../fs-utils.js';
58
58
  import { parseFrontmatterGeneric } from '../frontmatter.js';
59
59
  import { evalCondition } from '../predicate.js';
60
60
  import { listAllMemoryDocs } from '../memory-resolver.js';
61
+ import { userScopeRoot } from '../scope.js';
61
62
  import { assembleNodeSubject, buildCeilingIndex, displayName, effectiveRung, gatePasses, normalizeDocName, parseSubstrateDoc, parseSubstrateFrontmatter, previewLine, resolveDocName, } from './index.js';
62
63
  import { cachedSubstrateDocs } from './session-cache.js';
63
64
  // Ancestor dirs we never look inside for a `.crouter/memory/` store (the read
@@ -156,16 +157,26 @@ function hasExplicitTrigger(doc) {
156
157
  * `.crouter/memory/`, walking from the read file up to $HOME (or the
157
158
  * filesystem root for a read outside $HOME), skipping junk ancestors. The
158
159
  * user-global `~/.crouter/memory/` store is deliberately skipped: otherwise
159
- * every file read under $HOME would surface unrelated user-wide preferences. */
160
+ * every file read under $HOME would surface unrelated user-wide preferences.
161
+ *
162
+ * The crouter home (`~/.crouter`, the user scope root) is skipped for the same
163
+ * reason it is never a project scope root (scope.ts's `isProjectScopeDir`): it
164
+ * is the user scope, not a project ancestor. Without this fence, reading any
165
+ * canvas runtime artifact (`~/.crouter/canvas/nodes/.../reports|context/...`)
166
+ * walks up THROUGH `~/.crouter` and would inject whatever lives at
167
+ * `~/.crouter/.crouter/memory/` — e.g. a differently-scoped node whose cwd
168
+ * resolved its project store there — leaking one node's private memory into
169
+ * every unrelated node that reads a canvas file. */
160
170
  function positionalCandidates(absReadFile) {
161
171
  const out = [];
162
172
  const seenDocPaths = new Set();
163
173
  const home = realpathOrSelf(homedir());
174
+ const userRoot = realpathOrSelf(userScopeRoot());
164
175
  const fsRoot = parse(absReadFile).root;
165
176
  let dir = dirname(absReadFile);
166
177
  let depth = 0;
167
178
  while (true) {
168
- if (dir !== home && !isJunkAncestor(dir)) {
179
+ if (dir !== home && dir !== userRoot && !isJunkAncestor(dir)) {
169
180
  const memDir = join(dir, CRTR_DIR_NAME, 'memory');
170
181
  if (pathExists(memDir)) {
171
182
  for (const file of safeWalkMd(memDir)) {
@@ -64,7 +64,7 @@ import { FAULT_QUIET_MS, readFault, recordFault } from '../core/runtime/fault.js
64
64
  import { fanDoctrineWake } from '../core/runtime/close.js';
65
65
  import { logger } from '../core/log.js';
66
66
  export { FAULT_QUIET_MS };
67
- import { isPidAlive, isRecordedPidAlive } from '../core/canvas/pid.js';
67
+ import { isPidAlive, isRecordedPidAlive, recordedPidLiveness } from '../core/canvas/pid.js';
68
68
  import { reconcileBootLiveness } from '../core/canvas/boot.js';
69
69
  import { listLivePanes, tearDownNode } from '../core/runtime/placement.js';
70
70
  import { reviveNode } from '../core/runtime/revive.js';
@@ -705,14 +705,24 @@ async function handleNodeLiveness(row, now, revivedThisTick) {
705
705
  // its turn is over, and the engine never exited — past the grace it kills the
706
706
  // engine so the dead-pid refresh branch below revives it fresh.
707
707
  //
708
- // Liveness here is IDENTITY-aware, not a bare isPidAlive: on a heavily-
708
+ // Liveness here is IDENTITY-aware AND requires CONFIRMED alive, not a bare
709
+ // isPidAlive or the fail-open-to-alive boolean adapter: on a heavily-
709
710
  // forking host (esp. Linux, where low pids recycle fast) a just-dead
710
- // broker's pid can be reused by an unrelated process before this tick
711
- // runs, and a bare isPidAlive would report the STRANGER as "our broker
712
- // alive" clearing the grace clock forever and wedging revive (the
713
- // recorded-pid-reuse hang). isRecordedPidAlive guards against that with
714
- // the launch-time identity baseline (`row.pi_pid_identity`).
715
- if (pid != null && isRecordedPidAlive(pid, row.pi_pid_identity)) {
711
+ // broker's pid can be reused by an unrelated process within milliseconds,
712
+ // AND the `ps` probe confirming identity can itself intermittently fail.
713
+ // Either misread as "alive" here doesn't just misreport this one tick it
714
+ // clears `unhealthySince` (the grace clock), discarding the daemon's memory
715
+ // that the node had already been observed dead, so the VERY NEXT tick treats
716
+ // a dead-for-20s+ node as a brand-new first observation and restarts the
717
+ // whole grace window from scratch (the confirmed Round-2 crash-revive wedge:
718
+ // a single `probe-null` read on a Linux CI runner wedged
719
+ // broker-crash-teardown.test.ts's 'CRASH → grace-revive RESUME'). So only a
720
+ // POSITIVELY CONFIRMED-alive read (`recordedPidLiveness === 'alive'`) may
721
+ // clear the clock; `'dead'` AND `'indeterminate'` both fall through to the
722
+ // ordinary not-alive path below, which now CONTRIBUTES to the grace window
723
+ // instead of zeroing it — robust to a flicker, and still revives promptly
724
+ // once a node is truly gone for the full grace duration.
725
+ if (pid != null && recordedPidLiveness(pid, row.pi_pid_identity) === 'alive') {
716
726
  unhealthySince.delete(id);
717
727
  clearStrandedRelaunchState(id); // a live engine means the stranding resolved
718
728
  handleYieldStall(row, pid, now);
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@crouton-kit/crouter",
3
- "version": "0.3.64",
3
+ "version": "0.3.66",
4
4
  "description": "crtr — agent runtime with memory, plugins, and marketplaces",
5
5
  "type": "module",
6
6
  "main": "dist/index.js",