@crouton-kit/crouter 0.3.65 → 0.3.66
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/__tests__/provider-rotation.test.ts +24 -12
- package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/provider-rotation.ts +41 -16
- package/dist/clients/attach/attach-cmd.js +329 -329
- package/dist/clients/attach/graph-overlay.js +25 -4
- package/dist/commands/node-lifecycle-revive.js +9 -2
- package/dist/core/__tests__/on-read-crouter-home-fence.test.d.ts +1 -0
- package/dist/core/__tests__/on-read-crouter-home-fence.test.js +67 -0
- package/dist/core/canvas/pid.d.ts +71 -13
- package/dist/core/canvas/pid.js +73 -29
- package/dist/core/runtime/revive-all.js +8 -2
- package/dist/core/runtime/revive.js +18 -8
- package/dist/core/substrate/on-read.js +13 -2
- package/dist/daemon/crtrd.js +18 -8
- package/package.json +1 -1
|
@@ -384,25 +384,46 @@ export class GraphOverlay {
|
|
|
384
384
|
if (idx < 0)
|
|
385
385
|
idx = Math.max(0, rows.findIndex((r) => r.id === this.self));
|
|
386
386
|
const cur = rows[idx];
|
|
387
|
+
// Cycle rows (↺ back-refs) carry the SAME id as the real node they echo, so a
|
|
388
|
+
// cursor landing on one resolves back to the earlier real row on the next
|
|
389
|
+
// findIndex — the cursor snaps up and can't pass the echo. They're non-
|
|
390
|
+
// actionable (focusing re-targets the real node you can already reach), so
|
|
391
|
+
// navigation skips them: the cursor only ever rests on a unique, real row.
|
|
392
|
+
const isSelectable = (r) => r !== undefined && !r.cycle;
|
|
393
|
+
const step = (from, dir) => {
|
|
394
|
+
for (let i = from + dir; i >= 0 && i < rows.length; i += dir) {
|
|
395
|
+
if (isSelectable(rows[i]))
|
|
396
|
+
return i;
|
|
397
|
+
}
|
|
398
|
+
return from; // no selectable row that way — hold position
|
|
399
|
+
};
|
|
400
|
+
const edge = (dir) => {
|
|
401
|
+
const start = dir === 1 ? 0 : rows.length - 1;
|
|
402
|
+
for (let i = start; i >= 0 && i < rows.length; i += dir) {
|
|
403
|
+
if (isSelectable(rows[i]))
|
|
404
|
+
return i;
|
|
405
|
+
}
|
|
406
|
+
return idx;
|
|
407
|
+
};
|
|
387
408
|
if (matchesKey(data, 'j')) {
|
|
388
|
-
idx =
|
|
409
|
+
idx = step(idx, 1);
|
|
389
410
|
this.cursorId = rows[idx]?.id ?? this.cursorId;
|
|
390
411
|
this.tui.requestRender();
|
|
391
412
|
return;
|
|
392
413
|
}
|
|
393
414
|
if (matchesKey(data, 'k')) {
|
|
394
|
-
idx =
|
|
415
|
+
idx = step(idx, -1);
|
|
395
416
|
this.cursorId = rows[idx]?.id ?? this.cursorId;
|
|
396
417
|
this.tui.requestRender();
|
|
397
418
|
return;
|
|
398
419
|
}
|
|
399
420
|
if (matchesKey(data, 'g')) {
|
|
400
|
-
this.cursorId = rows[
|
|
421
|
+
this.cursorId = rows[edge(1)]?.id ?? this.cursorId;
|
|
401
422
|
this.tui.requestRender();
|
|
402
423
|
return;
|
|
403
424
|
}
|
|
404
425
|
if (matchesKey(data, 'shift+g')) {
|
|
405
|
-
this.cursorId = rows[
|
|
426
|
+
this.cursorId = rows[edge(-1)]?.id ?? this.cursorId;
|
|
406
427
|
this.tui.requestRender();
|
|
407
428
|
return;
|
|
408
429
|
}
|
|
@@ -20,7 +20,7 @@ import { InputError } from '../core/io.js';
|
|
|
20
20
|
import { reviveNode } from '../core/runtime/revive.js';
|
|
21
21
|
import { waitForBrokerViewSocket } from '../core/runtime/placement.js';
|
|
22
22
|
import { getNode, fullName } from '../core/canvas/index.js';
|
|
23
|
-
import {
|
|
23
|
+
import { recordedPidLiveness } from '../core/canvas/pid.js';
|
|
24
24
|
import { readFault } from '../core/runtime/fault.js';
|
|
25
25
|
import { isBusy } from '../core/runtime/busy.js';
|
|
26
26
|
import { faultNeedsYou } from '../core/canvas/status-glyph.js';
|
|
@@ -114,7 +114,14 @@ export const nodeReviveLeaf = defineLeaf({
|
|
|
114
114
|
if (now) {
|
|
115
115
|
const pid = meta.pi_pid;
|
|
116
116
|
const stall = readFault(nodeId);
|
|
117
|
-
|
|
117
|
+
// Identity-aware AND requires CONFIRMED alive — --now's whole job is to
|
|
118
|
+
// SIGTERM this pid, so a bare isPidAlive risks signalling a STRANGER
|
|
119
|
+
// process that reused a dead broker's recycled pid. `dead` (including a
|
|
120
|
+
// positively confirmed reuse) correctly refuses here. `indeterminate`
|
|
121
|
+
// (can't confirm identity either way, e.g. a transient `ps` probe
|
|
122
|
+
// failure) ALSO refuses — only a positively confirmed-alive-and-ours pid
|
|
123
|
+
// is ever signaled by this command.
|
|
124
|
+
if (pid == null || recordedPidLiveness(pid, meta.pi_pid_identity) !== 'alive') {
|
|
118
125
|
throw new InputError({
|
|
119
126
|
error: 'not_hanging',
|
|
120
127
|
message: `${nodeId} has no live broker — nothing to kick. --now is for a HANGING node (live broker parked on an active fault).`,
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
// The on-read POSITIONAL walk must never treat the crouter home (`~/.crouter`,
|
|
2
|
+
// the user scope root) as a project ancestor. Canvas runtime artifacts live at
|
|
3
|
+
// `~/.crouter/canvas/nodes/<id>/{reports,context}/...`, so a naive ancestor
|
|
4
|
+
// walk from such a read climbs THROUGH `~/.crouter` and would inject whatever
|
|
5
|
+
// lives at `~/.crouter/.crouter/memory/` — e.g. a differently-scoped node whose
|
|
6
|
+
// cwd resolved its private project store there. That leaks one node's private
|
|
7
|
+
// memory into every unrelated node that reads a canvas file. This test pins the
|
|
8
|
+
// fence: reading a canvas artifact must not surface a store nested in the
|
|
9
|
+
// crouter home.
|
|
10
|
+
//
|
|
11
|
+
// Run: node --import tsx/esm --test src/core/__tests__/on-read-crouter-home-fence.test.ts
|
|
12
|
+
import { test, before, beforeEach, after } from 'node:test';
|
|
13
|
+
import assert from 'node:assert/strict';
|
|
14
|
+
import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from 'node:fs';
|
|
15
|
+
import { tmpdir } from 'node:os';
|
|
16
|
+
import { join } from 'node:path';
|
|
17
|
+
import { closeDb } from '../canvas/db.js';
|
|
18
|
+
import { resetScopeCache } from '../scope.js';
|
|
19
|
+
import { spawnNode } from '../runtime/nodes.js';
|
|
20
|
+
import { renderOnReadDocs } from '../substrate/on-read.js';
|
|
21
|
+
let canvasHome;
|
|
22
|
+
let fakeHome;
|
|
23
|
+
let prevHomeEnv;
|
|
24
|
+
before(() => {
|
|
25
|
+
prevHomeEnv = process.env['HOME'];
|
|
26
|
+
});
|
|
27
|
+
beforeEach(() => {
|
|
28
|
+
closeDb();
|
|
29
|
+
resetScopeCache();
|
|
30
|
+
// A private user HOME so `homedir()`/`userScopeRoot()` resolve into our
|
|
31
|
+
// sandbox, not the real machine.
|
|
32
|
+
fakeHome = mkdtempSync(join(tmpdir(), 'crtr-fence-home-'));
|
|
33
|
+
process.env['HOME'] = fakeHome;
|
|
34
|
+
// The node graph (canvas.db) lives in its own CRTR_HOME so spawnNode does not
|
|
35
|
+
// touch the fake user home; the read path we test is constructed by hand.
|
|
36
|
+
canvasHome = mkdtempSync(join(tmpdir(), 'crtr-fence-canvas-'));
|
|
37
|
+
process.env['CRTR_HOME'] = canvasHome;
|
|
38
|
+
});
|
|
39
|
+
after(() => {
|
|
40
|
+
closeDb();
|
|
41
|
+
resetScopeCache();
|
|
42
|
+
rmSync(canvasHome, { recursive: true, force: true });
|
|
43
|
+
rmSync(fakeHome, { recursive: true, force: true });
|
|
44
|
+
delete process.env['CRTR_HOME'];
|
|
45
|
+
if (prevHomeEnv === undefined)
|
|
46
|
+
delete process.env['HOME'];
|
|
47
|
+
else
|
|
48
|
+
process.env['HOME'] = prevHomeEnv;
|
|
49
|
+
});
|
|
50
|
+
test('a store nested in the crouter home does not leak into a canvas-artifact read', () => {
|
|
51
|
+
const node = spawnNode({ kind: 'general', cwd: fakeHome, parent: null }).node_id;
|
|
52
|
+
// A private store misresolved into the crouter home: `~/.crouter/.crouter/memory/`.
|
|
53
|
+
const leakedStore = join(fakeHome, '.crouter', '.crouter', 'memory');
|
|
54
|
+
mkdirSync(leakedStore, { recursive: true });
|
|
55
|
+
writeFileSync(join(leakedStore, 'private-thing.md'), '---\nkind: knowledge\n' +
|
|
56
|
+
"when-and-why-to-read: When X, read this because Y.\n" +
|
|
57
|
+
'file-read-visibility: content\n---\n' +
|
|
58
|
+
'PRIVATE ASSISTANT MEMORY MUST NOT LEAK\n');
|
|
59
|
+
// A canvas runtime artifact under the crouter home — the shape of every node
|
|
60
|
+
// report/context file the harness auto-reads.
|
|
61
|
+
const reportFile = join(fakeHome, '.crouter', 'canvas', 'nodes', 'somenode', 'reports', 'r.md');
|
|
62
|
+
mkdirSync(join(fakeHome, '.crouter', 'canvas', 'nodes', 'somenode', 'reports'), { recursive: true });
|
|
63
|
+
writeFileSync(reportFile, '# a node report\n');
|
|
64
|
+
const rendered = renderOnReadDocs(node, reportFile, new Set());
|
|
65
|
+
assert.ok(!rendered.includes('PRIVATE ASSISTANT MEMORY MUST NOT LEAK'), `crouter-home-nested store must not surface on a canvas read; got: ${rendered}`);
|
|
66
|
+
assert.ok(!rendered.includes('private-thing'), `crouter-home-nested store name must not surface on a canvas read; got: ${rendered}`);
|
|
67
|
+
});
|
|
@@ -13,19 +13,77 @@
|
|
|
13
13
|
* out under a long-lived spawning process (see `isZombie`'s doc) never comes
|
|
14
14
|
* back on its own. */
|
|
15
15
|
export declare function isPidAlive(pid: number | null | undefined): boolean;
|
|
16
|
-
/**
|
|
17
|
-
*
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
*
|
|
21
|
-
*
|
|
22
|
-
*
|
|
23
|
-
*
|
|
24
|
-
*
|
|
25
|
-
* —
|
|
26
|
-
*
|
|
27
|
-
*
|
|
28
|
-
*
|
|
16
|
+
/** THE single precision-aware identity compare, used EVERYWHERE two
|
|
17
|
+
* `composeIdentity` fingerprints are compared (this module's own
|
|
18
|
+
* `recordedPidLiveness`, `captureTeardownSnapshot`'s reuse check, and
|
|
19
|
+
* `killProcessTreePids`'s reuse check) — centralized so no call site can
|
|
20
|
+
* drift from this rule (review Major #1, Round 2 of the pid-reuse-liveness
|
|
21
|
+
* fix).
|
|
22
|
+
*
|
|
23
|
+
* The hazard this closes: `composeIdentity` appends the `#<ticks>` Linux
|
|
24
|
+
* discriminator only when `/proc/<pid>/stat` happens to be readable at
|
|
25
|
+
* capture time — so the SAME live process can be recorded plain `lstart` on
|
|
26
|
+
* one capture and `lstart#ticks` on another (a transient `/proc` read
|
|
27
|
+
* failure, or capturing on a platform/container without procfs). A naive
|
|
28
|
+
* `===` then reads that as a reuse (different string) and reports `dead` for
|
|
29
|
+
* a broker that never actually died — the false-DEAD direction, which can
|
|
30
|
+
* make `reviveNode`'s double-launch guard wrongly relaunch a SECOND broker
|
|
31
|
+
* onto a still-live session.
|
|
32
|
+
*
|
|
33
|
+
* Rule: split both on `#`. Bases (`lstart`) differ → definitely NOT a match
|
|
34
|
+
* (different process, or the same pid reused in a different second). Bases
|
|
35
|
+
* match and BOTH carry a `#ticks` suffix → match iff the suffixes are equal
|
|
36
|
+
* (this still catches genuine same-second reuse, the case `procStartTicks`
|
|
37
|
+
* exists for). Bases match and EITHER lacks the suffix → coarse-vs-fine,
|
|
38
|
+
* can't distinguish → MATCH (fail-open to same-process, matching this
|
|
39
|
+
* module's universal discipline: only a POSITIVE mismatch counts as reuse,
|
|
40
|
+
* never an absence of evidence). */
|
|
41
|
+
export declare function identitiesMatch(a: string, b: string): boolean;
|
|
42
|
+
/** Three-valued liveness verdict for a RECORDED node pid, identity-guarded
|
|
43
|
+
* against PID REUSE using the launch-time baseline (`pi_pid_identity`,
|
|
44
|
+
* captured by `recordPid`). On a heavily-forking host — esp. Linux, where
|
|
45
|
+
* low pids recycle fast — a broker's recorded pid can be reused by an
|
|
46
|
+
* unrelated process (or the `ps` probe confirming it can itself
|
|
47
|
+
* intermittently fail) within milliseconds of the broker dying; collapsing
|
|
48
|
+
* "confirmed alive", "confirmed dead/reused", and "couldn't confirm either
|
|
49
|
+
* way" into one boolean forces every caller to fail open to ALIVE on the
|
|
50
|
+
* third case — which, upstream in the daemon's grace-clock bookkeeping, is
|
|
51
|
+
* worse than misreporting one tick: a single such read WIPES the daemon's
|
|
52
|
+
* memory that the node had already been observed dead, restarting the whole
|
|
53
|
+
* grace window from scratch (the confirmed Round-2 crash-revive wedge — see
|
|
54
|
+
* `crtrd.ts`'s `handleNodeLiveness`). Exposing the third state lets callers
|
|
55
|
+
* decide for themselves whether "can't confirm" should act like alive or
|
|
56
|
+
* dead for their own direction of risk.
|
|
57
|
+
*
|
|
58
|
+
* Reuses the existing primitives — this is a classification layered on top
|
|
59
|
+
* of `isPidAlive` + `capturePidIdentities`, never a new `ps` parse:
|
|
60
|
+
* - `!isPidAlive(pid)` → `'dead'`. Load-bearing: this is what preserves
|
|
61
|
+
* zombie handling — `isPidAlive` already treats a gone pid AND a zombie
|
|
62
|
+
* (SIGKILLed detached broker awaiting reap) as dead, which is the entire
|
|
63
|
+
* reason a wedged zombie broker still gets revived. Do not bypass it.
|
|
64
|
+
* - `expectedIdentity == null` (legacy row / launch capture failed) →
|
|
65
|
+
* `'alive'` (preserve pre-identity behavior — no baseline to check
|
|
66
|
+
* against).
|
|
67
|
+
* - `ps` probe itself failed (`capturePidIdentities` → `null`) →
|
|
68
|
+
* `'indeterminate'`. THIS is the branch the Round-2 fix changes: it used
|
|
69
|
+
* to fail open to `true` (alive) here, which is the confirmed root cause
|
|
70
|
+
* of the wedge.
|
|
71
|
+
* - probe succeeded but has no row for this pid (alive per signal-0 a
|
|
72
|
+
* moment ago, gone now) → `'dead'`.
|
|
73
|
+
* - probe succeeded and the identity is present → `'alive'` iff
|
|
74
|
+
* `identitiesMatch` says so, else `'dead'` (positively reused). */
|
|
75
|
+
export type RecordedPidLiveness = 'alive' | 'dead' | 'indeterminate';
|
|
76
|
+
export declare function recordedPidLiveness(pid: number | null | undefined, expectedIdentity: string | null | undefined): RecordedPidLiveness;
|
|
77
|
+
/** Boolean teardown-direction adapter over `recordedPidLiveness`, kept so
|
|
78
|
+
* every pre-existing signalling caller stays behavior-identical: `dead` is
|
|
79
|
+
* the ONLY thing that reads false. `indeterminate` (can't confirm either
|
|
80
|
+
* way) reads true alongside `alive` — never signal a maybe-ours pid on mere
|
|
81
|
+
* uncertainty, matching this module's universal discipline (a POSITIVE
|
|
82
|
+
* mismatch is the only thing that changes behavior, never an absence of
|
|
83
|
+
* evidence). Callers that need to distinguish "confirmed alive" from
|
|
84
|
+
* "can't confirm" (e.g. a double-launch guard, where treating uncertainty as
|
|
85
|
+
* alive risks silently skipping a needed relaunch) must call
|
|
86
|
+
* `recordedPidLiveness` directly instead of this adapter. */
|
|
29
87
|
export declare function isRecordedPidAlive(pid: number | null | undefined, expectedIdentity: string | null | undefined): boolean;
|
|
30
88
|
/** SIGTERM a process GROUP by pid (the negative-pid convention) — best-effort,
|
|
31
89
|
* swallowing ESRCH (already gone). Shared by triggers.ts (cancel-time reap of
|
package/dist/core/canvas/pid.js
CHANGED
|
@@ -60,50 +60,92 @@ export function isPidAlive(pid) {
|
|
|
60
60
|
}
|
|
61
61
|
return !isZombie(pid);
|
|
62
62
|
}
|
|
63
|
-
/**
|
|
64
|
-
*
|
|
65
|
-
*
|
|
66
|
-
*
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
*
|
|
73
|
-
*
|
|
74
|
-
*
|
|
75
|
-
*
|
|
76
|
-
|
|
63
|
+
/** Split a `composeIdentity` fingerprint (`lstart` or `lstart#ticks`) into its
|
|
64
|
+
* coarse (`lstart`, second-granular, portable) and fine (`ticks`, Linux-only
|
|
65
|
+
* jiffies, best-effort) parts. `#` never appears in `lstart` itself (a `ps`
|
|
66
|
+
* timestamp), so the first `#` unambiguously separates them. */
|
|
67
|
+
function splitIdentity(identity) {
|
|
68
|
+
const i = identity.indexOf('#');
|
|
69
|
+
return i === -1 ? { base: identity, ticks: undefined } : { base: identity.slice(0, i), ticks: identity.slice(i + 1) };
|
|
70
|
+
}
|
|
71
|
+
/** THE single precision-aware identity compare, used EVERYWHERE two
|
|
72
|
+
* `composeIdentity` fingerprints are compared (this module's own
|
|
73
|
+
* `recordedPidLiveness`, `captureTeardownSnapshot`'s reuse check, and
|
|
74
|
+
* `killProcessTreePids`'s reuse check) — centralized so no call site can
|
|
75
|
+
* drift from this rule (review Major #1, Round 2 of the pid-reuse-liveness
|
|
76
|
+
* fix).
|
|
77
|
+
*
|
|
78
|
+
* The hazard this closes: `composeIdentity` appends the `#<ticks>` Linux
|
|
79
|
+
* discriminator only when `/proc/<pid>/stat` happens to be readable at
|
|
80
|
+
* capture time — so the SAME live process can be recorded plain `lstart` on
|
|
81
|
+
* one capture and `lstart#ticks` on another (a transient `/proc` read
|
|
82
|
+
* failure, or capturing on a platform/container without procfs). A naive
|
|
83
|
+
* `===` then reads that as a reuse (different string) and reports `dead` for
|
|
84
|
+
* a broker that never actually died — the false-DEAD direction, which can
|
|
85
|
+
* make `reviveNode`'s double-launch guard wrongly relaunch a SECOND broker
|
|
86
|
+
* onto a still-live session.
|
|
87
|
+
*
|
|
88
|
+
* Rule: split both on `#`. Bases (`lstart`) differ → definitely NOT a match
|
|
89
|
+
* (different process, or the same pid reused in a different second). Bases
|
|
90
|
+
* match and BOTH carry a `#ticks` suffix → match iff the suffixes are equal
|
|
91
|
+
* (this still catches genuine same-second reuse, the case `procStartTicks`
|
|
92
|
+
* exists for). Bases match and EITHER lacks the suffix → coarse-vs-fine,
|
|
93
|
+
* can't distinguish → MATCH (fail-open to same-process, matching this
|
|
94
|
+
* module's universal discipline: only a POSITIVE mismatch counts as reuse,
|
|
95
|
+
* never an absence of evidence). */
|
|
96
|
+
export function identitiesMatch(a, b) {
|
|
97
|
+
const ia = splitIdentity(a);
|
|
98
|
+
const ib = splitIdentity(b);
|
|
99
|
+
if (ia.base !== ib.base)
|
|
100
|
+
return false;
|
|
101
|
+
if (ia.ticks !== undefined && ib.ticks !== undefined)
|
|
102
|
+
return ia.ticks === ib.ticks;
|
|
103
|
+
return true;
|
|
104
|
+
}
|
|
105
|
+
export function recordedPidLiveness(pid, expectedIdentity) {
|
|
77
106
|
const debug = Boolean(process.env.CRTR_DEBUG_PID_LIVENESS);
|
|
78
107
|
const alive = isPidAlive(pid);
|
|
79
108
|
if (!alive) {
|
|
80
109
|
if (debug)
|
|
81
|
-
logPidLivenessDecision(pid, alive, expectedIdentity, 'no-probe', 'isPidAlive=false → dead',
|
|
82
|
-
return
|
|
110
|
+
logPidLivenessDecision(pid, alive, expectedIdentity, 'no-probe', 'isPidAlive=false → dead', 'dead');
|
|
111
|
+
return 'dead'; // gone/zombie → dead (unchanged path)
|
|
83
112
|
}
|
|
84
113
|
if (expectedIdentity == null) {
|
|
85
114
|
if (debug)
|
|
86
|
-
logPidLivenessDecision(pid, alive, expectedIdentity, 'no-probe', 'no baseline → trust isPidAlive',
|
|
87
|
-
return
|
|
115
|
+
logPidLivenessDecision(pid, alive, expectedIdentity, 'no-probe', 'no baseline → trust isPidAlive', 'alive');
|
|
116
|
+
return 'alive'; // no baseline → trust isPidAlive (legacy behavior)
|
|
88
117
|
}
|
|
89
118
|
const current = capturePidIdentities([pid]);
|
|
90
119
|
if (current == null) {
|
|
91
120
|
if (debug)
|
|
92
|
-
logPidLivenessDecision(pid, alive, expectedIdentity, 'probe-null', 'ps probe failed →
|
|
93
|
-
return
|
|
121
|
+
logPidLivenessDecision(pid, alive, expectedIdentity, 'probe-null', 'ps probe failed → indeterminate', 'indeterminate');
|
|
122
|
+
return 'indeterminate'; // probe itself failed → we don't know (THE fix: was fail-open alive)
|
|
94
123
|
}
|
|
95
124
|
const actual = current.get(pid);
|
|
96
125
|
if (actual === undefined) {
|
|
97
126
|
if (debug)
|
|
98
|
-
logPidLivenessDecision(pid, alive, expectedIdentity, 'no-entry', 'no identity row for pid (race) →
|
|
99
|
-
return
|
|
127
|
+
logPidLivenessDecision(pid, alive, expectedIdentity, 'no-entry', 'no identity row for pid (race) → dead', 'dead');
|
|
128
|
+
return 'dead'; // isPidAlive said alive but it's gone now → dead
|
|
100
129
|
}
|
|
101
|
-
const
|
|
130
|
+
const matches = identitiesMatch(actual, expectedIdentity);
|
|
102
131
|
if (debug)
|
|
103
|
-
logPidLivenessDecision(pid, alive, expectedIdentity, 'value',
|
|
104
|
-
return
|
|
132
|
+
logPidLivenessDecision(pid, alive, expectedIdentity, 'value', matches ? 'identity matches expected → alive' : 'identity differs from expected → REUSED → dead', matches ? 'alive' : 'dead', actual);
|
|
133
|
+
return matches ? 'alive' : 'dead';
|
|
134
|
+
}
|
|
135
|
+
/** Boolean teardown-direction adapter over `recordedPidLiveness`, kept so
|
|
136
|
+
* every pre-existing signalling caller stays behavior-identical: `dead` is
|
|
137
|
+
* the ONLY thing that reads false. `indeterminate` (can't confirm either
|
|
138
|
+
* way) reads true alongside `alive` — never signal a maybe-ours pid on mere
|
|
139
|
+
* uncertainty, matching this module's universal discipline (a POSITIVE
|
|
140
|
+
* mismatch is the only thing that changes behavior, never an absence of
|
|
141
|
+
* evidence). Callers that need to distinguish "confirmed alive" from
|
|
142
|
+
* "can't confirm" (e.g. a double-launch guard, where treating uncertainty as
|
|
143
|
+
* alive risks silently skipping a needed relaunch) must call
|
|
144
|
+
* `recordedPidLiveness` directly instead of this adapter. */
|
|
145
|
+
export function isRecordedPidAlive(pid, expectedIdentity) {
|
|
146
|
+
return recordedPidLiveness(pid, expectedIdentity) !== 'dead';
|
|
105
147
|
}
|
|
106
|
-
/** Diagnostic-only decision trace for `
|
|
148
|
+
/** Diagnostic-only decision trace for `recordedPidLiveness`, gated entirely
|
|
107
149
|
* behind `CRTR_DEBUG_PID_LIVENESS` — callers must check the env var
|
|
108
150
|
* themselves before calling this so the (tiny) formatting cost is paid ONLY
|
|
109
151
|
* when the env var is set; this function does no env check of its own.
|
|
@@ -129,7 +171,7 @@ function logPidLivenessDecision(pid, isPidAliveResult, expectedIdentity, probeOu
|
|
|
129
171
|
`branch=${JSON.stringify(branch)}`,
|
|
130
172
|
`returned=${returned}`,
|
|
131
173
|
];
|
|
132
|
-
process.stderr.write(`[
|
|
174
|
+
process.stderr.write(`[recordedPidLiveness] ${parts.join(' ')}\n`);
|
|
133
175
|
}
|
|
134
176
|
/** SIGTERM a process GROUP by pid (the negative-pid convention) — best-effort,
|
|
135
177
|
* swallowing ESRCH (already gone). Shared by triggers.ts (cancel-time reap of
|
|
@@ -339,7 +381,9 @@ export function captureTeardownSnapshot(rootPid, expectedIdentity) {
|
|
|
339
381
|
if (table === null)
|
|
340
382
|
return { tree: [rootPid], identities: null, reused: false };
|
|
341
383
|
const currentRootIdentity = table.identities.get(rootPid);
|
|
342
|
-
const reused = expectedIdentity != null &&
|
|
384
|
+
const reused = expectedIdentity != null &&
|
|
385
|
+
currentRootIdentity !== undefined &&
|
|
386
|
+
!identitiesMatch(currentRootIdentity, expectedIdentity);
|
|
343
387
|
if (reused)
|
|
344
388
|
return { tree: [], identities: null, reused: true };
|
|
345
389
|
const tree = [rootPid, ...descendantPidsFrom(rootPid, table.childrenOf)];
|
|
@@ -407,7 +451,7 @@ export function killProcessTreePids(pids, signal = 'SIGTERM', identities) {
|
|
|
407
451
|
if (identities != null && current != null) {
|
|
408
452
|
const expected = identities.get(pid);
|
|
409
453
|
const actual = current.get(pid);
|
|
410
|
-
const reused = expected !== undefined && actual !== undefined && actual
|
|
454
|
+
const reused = expected !== undefined && actual !== undefined && !identitiesMatch(actual, expected);
|
|
411
455
|
if (reused)
|
|
412
456
|
continue;
|
|
413
457
|
}
|
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
// self-guards the double-spawn. The selection predicate (`isDisconnected`) is
|
|
14
14
|
// pure so the scope rules are unit-testable without booting a single process.
|
|
15
15
|
import { listNodes, getNode } from '../canvas/index.js';
|
|
16
|
-
import {
|
|
16
|
+
import { recordedPidLiveness } from '../canvas/pid.js';
|
|
17
17
|
import { reviveNode } from './revive.js';
|
|
18
18
|
/** Statuses a node lands in DELIBERATELY: a worker that finished its own work
|
|
19
19
|
* (`done`) or a node a human closed/reaped (`canceled`). The daemon never
|
|
@@ -56,7 +56,13 @@ export function listDisconnected() {
|
|
|
56
56
|
const out = [];
|
|
57
57
|
for (const row of listNodes()) {
|
|
58
58
|
const meta = getNode(row.node_id);
|
|
59
|
-
|
|
59
|
+
// IDENTITY-aware AND requires CONFIRMED alive: a bare isPidAlive would
|
|
60
|
+
// misread a recycled pi_pid as "engine running" and SKIP the node, so it
|
|
61
|
+
// never reaches reviveNode's own identity-aware double-launch guard
|
|
62
|
+
// (Round 2 of the pid-reuse-liveness fix, fold-in Major). `dead` OR
|
|
63
|
+
// `indeterminate` both count as disconnected/eligible here — only a
|
|
64
|
+
// positively confirmed-alive read is treated as "still connected".
|
|
65
|
+
if (meta !== null && isDisconnected(meta, (pid) => recordedPidLiveness(pid, meta.pi_pid_identity) === 'alive'))
|
|
60
66
|
out.push(meta);
|
|
61
67
|
}
|
|
62
68
|
return out;
|
|
@@ -29,7 +29,7 @@ import { fanDoctrineWake } from './close.js';
|
|
|
29
29
|
import { buildPiArgv } from './launch.js';
|
|
30
30
|
import { buildReviveKickoff, drainBearings } from './kickoff.js';
|
|
31
31
|
import { headlessBrokerHost } from './host.js';
|
|
32
|
-
import {
|
|
32
|
+
import { recordedPidLiveness } from '../canvas/pid.js';
|
|
33
33
|
import { reconcileBootLiveness } from '../canvas/boot.js';
|
|
34
34
|
import { clearFault } from './fault.js';
|
|
35
35
|
import { clearInjectedDocs } from '../substrate/injected-store.js';
|
|
@@ -89,14 +89,24 @@ export function reviveNode(nodeId, opts) {
|
|
|
89
89
|
// actually matters, so it's safe to call on every revive regardless of
|
|
90
90
|
// entry point (daemon tick, or a direct/manual `node lifecycle revive`).
|
|
91
91
|
reconcileBootLiveness();
|
|
92
|
-
// Double-revive guard:
|
|
93
|
-
//
|
|
94
|
-
// still
|
|
95
|
-
// another path — re-launching would put
|
|
96
|
-
// session file. No-op. A bare isPidAlive would
|
|
97
|
-
// forking, esp. Linux) as "still alive" and
|
|
92
|
+
// Double-revive guard: identity-aware liveness on pi_pid, requiring
|
|
93
|
+
// CONFIRMED alive — not the fail-open-to-alive boolean adapter. A node
|
|
94
|
+
// whose broker pid is CONFIRMED still running as that same broker (not a
|
|
95
|
+
// reused pid) was already revived by another path — re-launching would put
|
|
96
|
+
// a SECOND broker on the same session file. No-op. A bare isPidAlive would
|
|
97
|
+
// misread a reused pid (heavy forking, esp. Linux) as "still alive" and
|
|
98
|
+
// skip the relaunch forever; conversely, treating an INDETERMINATE read
|
|
99
|
+
// (can't confirm either way, e.g. a transient `ps` probe failure) as
|
|
100
|
+
// "alive" here would risk the opposite failure — silently skipping a
|
|
101
|
+
// needed relaunch and stranding the node with no engine. So this guard
|
|
102
|
+
// blocks ONLY on a positively confirmed-alive read; `dead` AND
|
|
103
|
+
// `indeterminate` both proceed to relaunch. We accept that tradeoff: an
|
|
104
|
+
// `indeterminate` read CAN come from a genuinely live broker whose identity
|
|
105
|
+
// `ps` probe transiently failed, so proceeding risks a rare double-launch —
|
|
106
|
+
// but stranding a truly-dead node with no engine is the worse failure, and
|
|
107
|
+
// recovering from it is far harder, so we bias toward relaunching.
|
|
98
108
|
const live = getNode(nodeId) ?? meta;
|
|
99
|
-
if (
|
|
109
|
+
if (recordedPidLiveness(live.pi_pid, live.pi_pid_identity) === 'alive') {
|
|
100
110
|
return {
|
|
101
111
|
window: null,
|
|
102
112
|
session: live.tmux_session ?? null,
|
|
@@ -58,6 +58,7 @@ import { pathExists, readText, walkFiles } from '../fs-utils.js';
|
|
|
58
58
|
import { parseFrontmatterGeneric } from '../frontmatter.js';
|
|
59
59
|
import { evalCondition } from '../predicate.js';
|
|
60
60
|
import { listAllMemoryDocs } from '../memory-resolver.js';
|
|
61
|
+
import { userScopeRoot } from '../scope.js';
|
|
61
62
|
import { assembleNodeSubject, buildCeilingIndex, displayName, effectiveRung, gatePasses, normalizeDocName, parseSubstrateDoc, parseSubstrateFrontmatter, previewLine, resolveDocName, } from './index.js';
|
|
62
63
|
import { cachedSubstrateDocs } from './session-cache.js';
|
|
63
64
|
// Ancestor dirs we never look inside for a `.crouter/memory/` store (the read
|
|
@@ -156,16 +157,26 @@ function hasExplicitTrigger(doc) {
|
|
|
156
157
|
* `.crouter/memory/`, walking from the read file up to $HOME (or the
|
|
157
158
|
* filesystem root for a read outside $HOME), skipping junk ancestors. The
|
|
158
159
|
* user-global `~/.crouter/memory/` store is deliberately skipped: otherwise
|
|
159
|
-
* every file read under $HOME would surface unrelated user-wide preferences.
|
|
160
|
+
* every file read under $HOME would surface unrelated user-wide preferences.
|
|
161
|
+
*
|
|
162
|
+
* The crouter home (`~/.crouter`, the user scope root) is skipped for the same
|
|
163
|
+
* reason it is never a project scope root (scope.ts's `isProjectScopeDir`): it
|
|
164
|
+
* is the user scope, not a project ancestor. Without this fence, reading any
|
|
165
|
+
* canvas runtime artifact (`~/.crouter/canvas/nodes/.../reports|context/...`)
|
|
166
|
+
* walks up THROUGH `~/.crouter` and would inject whatever lives at
|
|
167
|
+
* `~/.crouter/.crouter/memory/` — e.g. a differently-scoped node whose cwd
|
|
168
|
+
* resolved its project store there — leaking one node's private memory into
|
|
169
|
+
* every unrelated node that reads a canvas file. */
|
|
160
170
|
function positionalCandidates(absReadFile) {
|
|
161
171
|
const out = [];
|
|
162
172
|
const seenDocPaths = new Set();
|
|
163
173
|
const home = realpathOrSelf(homedir());
|
|
174
|
+
const userRoot = realpathOrSelf(userScopeRoot());
|
|
164
175
|
const fsRoot = parse(absReadFile).root;
|
|
165
176
|
let dir = dirname(absReadFile);
|
|
166
177
|
let depth = 0;
|
|
167
178
|
while (true) {
|
|
168
|
-
if (dir !== home && !isJunkAncestor(dir)) {
|
|
179
|
+
if (dir !== home && dir !== userRoot && !isJunkAncestor(dir)) {
|
|
169
180
|
const memDir = join(dir, CRTR_DIR_NAME, 'memory');
|
|
170
181
|
if (pathExists(memDir)) {
|
|
171
182
|
for (const file of safeWalkMd(memDir)) {
|
package/dist/daemon/crtrd.js
CHANGED
|
@@ -64,7 +64,7 @@ import { FAULT_QUIET_MS, readFault, recordFault } from '../core/runtime/fault.js
|
|
|
64
64
|
import { fanDoctrineWake } from '../core/runtime/close.js';
|
|
65
65
|
import { logger } from '../core/log.js';
|
|
66
66
|
export { FAULT_QUIET_MS };
|
|
67
|
-
import { isPidAlive, isRecordedPidAlive } from '../core/canvas/pid.js';
|
|
67
|
+
import { isPidAlive, isRecordedPidAlive, recordedPidLiveness } from '../core/canvas/pid.js';
|
|
68
68
|
import { reconcileBootLiveness } from '../core/canvas/boot.js';
|
|
69
69
|
import { listLivePanes, tearDownNode } from '../core/runtime/placement.js';
|
|
70
70
|
import { reviveNode } from '../core/runtime/revive.js';
|
|
@@ -705,14 +705,24 @@ async function handleNodeLiveness(row, now, revivedThisTick) {
|
|
|
705
705
|
// its turn is over, and the engine never exited — past the grace it kills the
|
|
706
706
|
// engine so the dead-pid refresh branch below revives it fresh.
|
|
707
707
|
//
|
|
708
|
-
// Liveness here is IDENTITY-aware
|
|
708
|
+
// Liveness here is IDENTITY-aware AND requires CONFIRMED alive, not a bare
|
|
709
|
+
// isPidAlive or the fail-open-to-alive boolean adapter: on a heavily-
|
|
709
710
|
// forking host (esp. Linux, where low pids recycle fast) a just-dead
|
|
710
|
-
// broker's pid can be reused by an unrelated process
|
|
711
|
-
//
|
|
712
|
-
// alive"
|
|
713
|
-
//
|
|
714
|
-
// the
|
|
715
|
-
|
|
711
|
+
// broker's pid can be reused by an unrelated process within milliseconds,
|
|
712
|
+
// AND the `ps` probe confirming identity can itself intermittently fail.
|
|
713
|
+
// Either misread as "alive" here doesn't just misreport this one tick — it
|
|
714
|
+
// clears `unhealthySince` (the grace clock), discarding the daemon's memory
|
|
715
|
+
// that the node had already been observed dead, so the VERY NEXT tick treats
|
|
716
|
+
// a dead-for-20s+ node as a brand-new first observation and restarts the
|
|
717
|
+
// whole grace window from scratch (the confirmed Round-2 crash-revive wedge:
|
|
718
|
+
// a single `probe-null` read on a Linux CI runner wedged
|
|
719
|
+
// broker-crash-teardown.test.ts's 'CRASH → grace-revive RESUME'). So only a
|
|
720
|
+
// POSITIVELY CONFIRMED-alive read (`recordedPidLiveness === 'alive'`) may
|
|
721
|
+
// clear the clock; `'dead'` AND `'indeterminate'` both fall through to the
|
|
722
|
+
// ordinary not-alive path below, which now CONTRIBUTES to the grace window
|
|
723
|
+
// instead of zeroing it — robust to a flicker, and still revives promptly
|
|
724
|
+
// once a node is truly gone for the full grace duration.
|
|
725
|
+
if (pid != null && recordedPidLiveness(pid, row.pi_pid_identity) === 'alive') {
|
|
716
726
|
unhealthySince.delete(id);
|
|
717
727
|
clearStrandedRelaunchState(id); // a live engine means the stranding resolved
|
|
718
728
|
handleYieldStall(row, pid, now);
|