@crouton-kit/crouter 0.3.64 → 0.3.66
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/__tests__/provider-rotation.test.ts +24 -12
- package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/provider-rotation.ts +41 -16
- package/dist/clients/attach/attach-cmd.js +329 -329
- package/dist/clients/attach/graph-overlay.js +25 -4
- package/dist/commands/node-lifecycle-revive.js +9 -2
- package/dist/core/__tests__/full/broker-crash-teardown.test.js +17 -1
- package/dist/core/__tests__/on-read-crouter-home-fence.test.d.ts +1 -0
- package/dist/core/__tests__/on-read-crouter-home-fence.test.js +67 -0
- package/dist/core/canvas/pid.d.ts +71 -13
- package/dist/core/canvas/pid.js +114 -25
- package/dist/core/runtime/revive-all.js +8 -2
- package/dist/core/runtime/revive.js +18 -8
- package/dist/core/substrate/on-read.js +13 -2
- package/dist/daemon/crtrd.js +18 -8
- package/package.json +1 -1
|
@@ -384,25 +384,46 @@ export class GraphOverlay {
|
|
|
384
384
|
if (idx < 0)
|
|
385
385
|
idx = Math.max(0, rows.findIndex((r) => r.id === this.self));
|
|
386
386
|
const cur = rows[idx];
|
|
387
|
+
// Cycle rows (↺ back-refs) carry the SAME id as the real node they echo, so a
|
|
388
|
+
// cursor landing on one resolves back to the earlier real row on the next
|
|
389
|
+
// findIndex — the cursor snaps up and can't pass the echo. They're non-
|
|
390
|
+
// actionable (focusing re-targets the real node you can already reach), so
|
|
391
|
+
// navigation skips them: the cursor only ever rests on a unique, real row.
|
|
392
|
+
const isSelectable = (r) => r !== undefined && !r.cycle;
|
|
393
|
+
const step = (from, dir) => {
|
|
394
|
+
for (let i = from + dir; i >= 0 && i < rows.length; i += dir) {
|
|
395
|
+
if (isSelectable(rows[i]))
|
|
396
|
+
return i;
|
|
397
|
+
}
|
|
398
|
+
return from; // no selectable row that way — hold position
|
|
399
|
+
};
|
|
400
|
+
const edge = (dir) => {
|
|
401
|
+
const start = dir === 1 ? 0 : rows.length - 1;
|
|
402
|
+
for (let i = start; i >= 0 && i < rows.length; i += dir) {
|
|
403
|
+
if (isSelectable(rows[i]))
|
|
404
|
+
return i;
|
|
405
|
+
}
|
|
406
|
+
return idx;
|
|
407
|
+
};
|
|
387
408
|
if (matchesKey(data, 'j')) {
|
|
388
|
-
idx =
|
|
409
|
+
idx = step(idx, 1);
|
|
389
410
|
this.cursorId = rows[idx]?.id ?? this.cursorId;
|
|
390
411
|
this.tui.requestRender();
|
|
391
412
|
return;
|
|
392
413
|
}
|
|
393
414
|
if (matchesKey(data, 'k')) {
|
|
394
|
-
idx =
|
|
415
|
+
idx = step(idx, -1);
|
|
395
416
|
this.cursorId = rows[idx]?.id ?? this.cursorId;
|
|
396
417
|
this.tui.requestRender();
|
|
397
418
|
return;
|
|
398
419
|
}
|
|
399
420
|
if (matchesKey(data, 'g')) {
|
|
400
|
-
this.cursorId = rows[
|
|
421
|
+
this.cursorId = rows[edge(1)]?.id ?? this.cursorId;
|
|
401
422
|
this.tui.requestRender();
|
|
402
423
|
return;
|
|
403
424
|
}
|
|
404
425
|
if (matchesKey(data, 'shift+g')) {
|
|
405
|
-
this.cursorId = rows[
|
|
426
|
+
this.cursorId = rows[edge(-1)]?.id ?? this.cursorId;
|
|
406
427
|
this.tui.requestRender();
|
|
407
428
|
return;
|
|
408
429
|
}
|
|
@@ -20,7 +20,7 @@ import { InputError } from '../core/io.js';
|
|
|
20
20
|
import { reviveNode } from '../core/runtime/revive.js';
|
|
21
21
|
import { waitForBrokerViewSocket } from '../core/runtime/placement.js';
|
|
22
22
|
import { getNode, fullName } from '../core/canvas/index.js';
|
|
23
|
-
import {
|
|
23
|
+
import { recordedPidLiveness } from '../core/canvas/pid.js';
|
|
24
24
|
import { readFault } from '../core/runtime/fault.js';
|
|
25
25
|
import { isBusy } from '../core/runtime/busy.js';
|
|
26
26
|
import { faultNeedsYou } from '../core/canvas/status-glyph.js';
|
|
@@ -114,7 +114,14 @@ export const nodeReviveLeaf = defineLeaf({
|
|
|
114
114
|
if (now) {
|
|
115
115
|
const pid = meta.pi_pid;
|
|
116
116
|
const stall = readFault(nodeId);
|
|
117
|
-
|
|
117
|
+
// Identity-aware AND requires CONFIRMED alive — --now's whole job is to
|
|
118
|
+
// SIGTERM this pid, so a bare isPidAlive risks signalling a STRANGER
|
|
119
|
+
// process that reused a dead broker's recycled pid. `dead` (including a
|
|
120
|
+
// positively confirmed reuse) correctly refuses here. `indeterminate`
|
|
121
|
+
// (can't confirm identity either way, e.g. a transient `ps` probe
|
|
122
|
+
// failure) ALSO refuses — only a positively confirmed-alive-and-ours pid
|
|
123
|
+
// is ever signaled by this command.
|
|
124
|
+
if (pid == null || recordedPidLiveness(pid, meta.pi_pid_identity) !== 'alive') {
|
|
118
125
|
throw new InputError({
|
|
119
126
|
error: 'not_hanging',
|
|
120
127
|
message: `${nodeId} has no live broker — nothing to kick. --now is for a HANGING node (live broker parked on an active fault).`,
|
|
@@ -44,7 +44,21 @@ import { existsSync, mkdirSync, writeFileSync } from 'node:fs';
|
|
|
44
44
|
import { join } from 'node:path';
|
|
45
45
|
import { createHarness } from '../helpers/harness.js';
|
|
46
46
|
import { subscribe } from '../../canvas/canvas.js';
|
|
47
|
-
import { isPidAlive } from '../../canvas/pid.js';
|
|
47
|
+
import { isPidAlive, capturePidIdentities } from '../../canvas/pid.js';
|
|
48
|
+
// Diagnostic-only, gated behind CRTR_DEBUG_PID_LIVENESS (same env var as the
|
|
49
|
+
// isRecordedPidAlive trace in pid.ts): logs whether the SIGKILLed pid reads
|
|
50
|
+
// alive and what identity ps reports for it, right at the two moments that
|
|
51
|
+
// matter for the intermittent Linux-only crash-revive wedge — directly after
|
|
52
|
+
// confirming the kill and again just before the grace-elapsed tick — so a real
|
|
53
|
+
// run shows whether the pid was reused during the grace window. No effect on
|
|
54
|
+
// test behavior when the env var is unset.
|
|
55
|
+
function debugLogPidState(label, pid) {
|
|
56
|
+
if (!process.env.CRTR_DEBUG_PID_LIVENESS)
|
|
57
|
+
return;
|
|
58
|
+
const identities = capturePidIdentities([pid]);
|
|
59
|
+
const identity = identities === null ? 'probe-null' : identities.has(pid) ? JSON.stringify(identities.get(pid)) : 'no-entry';
|
|
60
|
+
process.stderr.write(`[broker-crash-teardown] ${label}: pid=${pid} isPidAlive=${isPidAlive(pid)} capturePidIdentities=${identity}\n`);
|
|
61
|
+
}
|
|
48
62
|
// crtrd.ts module const (not exported): the fresh-pi-boot grace window the daemon
|
|
49
63
|
// waits before grace-reviving / boot-failing a pi observed dead. Reference:
|
|
50
64
|
// crtrd.ts `REVIVE_GRACE_MS = 20_000`.
|
|
@@ -81,9 +95,11 @@ test('CRASH → grace-revive RESUME (one-writer), then clean teardown on close',
|
|
|
81
95
|
// Actions run: crouter-development/prove-ci-fixes-in-actions). The waitFor's
|
|
82
96
|
// successful resolution already IS the proof of deadness at this point.
|
|
83
97
|
await h.waitFor(() => !isPidAlive(oldPid), { label: 'crashed broker pid is dead' });
|
|
98
|
+
debugLogPidState('after waitFor(!isPidAlive(oldPid))', oldPid);
|
|
84
99
|
const NOW = 5_000_000;
|
|
85
100
|
await h.tick(NOW); // pid dead, intent null → handleBrokerLiveness → handleLiveWindow marks pending
|
|
86
101
|
assert.equal(h.bootCount(id), boots, 'inside the grace window → NOT yet revived');
|
|
102
|
+
debugLogPidState('before grace-elapsed tick', oldPid);
|
|
87
103
|
await h.tick(NOW + REVIVE_GRACE_MS + 1); // grace elapsed → reviveNode(resume:true)
|
|
88
104
|
// This is the file's SECOND real broker boot (crash-revive), on an
|
|
89
105
|
// oversubscribed CI runner sharing the host with the other full-tier files'
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
// The on-read POSITIONAL walk must never treat the crouter home (`~/.crouter`,
|
|
2
|
+
// the user scope root) as a project ancestor. Canvas runtime artifacts live at
|
|
3
|
+
// `~/.crouter/canvas/nodes/<id>/{reports,context}/...`, so a naive ancestor
|
|
4
|
+
// walk from such a read climbs THROUGH `~/.crouter` and would inject whatever
|
|
5
|
+
// lives at `~/.crouter/.crouter/memory/` — e.g. a differently-scoped node whose
|
|
6
|
+
// cwd resolved its private project store there. That leaks one node's private
|
|
7
|
+
// memory into every unrelated node that reads a canvas file. This test pins the
|
|
8
|
+
// fence: reading a canvas artifact must not surface a store nested in the
|
|
9
|
+
// crouter home.
|
|
10
|
+
//
|
|
11
|
+
// Run: node --import tsx/esm --test src/core/__tests__/on-read-crouter-home-fence.test.ts
|
|
12
|
+
import { test, before, beforeEach, after } from 'node:test';
|
|
13
|
+
import assert from 'node:assert/strict';
|
|
14
|
+
import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from 'node:fs';
|
|
15
|
+
import { tmpdir } from 'node:os';
|
|
16
|
+
import { join } from 'node:path';
|
|
17
|
+
import { closeDb } from '../canvas/db.js';
|
|
18
|
+
import { resetScopeCache } from '../scope.js';
|
|
19
|
+
import { spawnNode } from '../runtime/nodes.js';
|
|
20
|
+
import { renderOnReadDocs } from '../substrate/on-read.js';
|
|
21
|
+
let canvasHome;
|
|
22
|
+
let fakeHome;
|
|
23
|
+
let prevHomeEnv;
|
|
24
|
+
before(() => {
|
|
25
|
+
prevHomeEnv = process.env['HOME'];
|
|
26
|
+
});
|
|
27
|
+
beforeEach(() => {
|
|
28
|
+
closeDb();
|
|
29
|
+
resetScopeCache();
|
|
30
|
+
// A private user HOME so `homedir()`/`userScopeRoot()` resolve into our
|
|
31
|
+
// sandbox, not the real machine.
|
|
32
|
+
fakeHome = mkdtempSync(join(tmpdir(), 'crtr-fence-home-'));
|
|
33
|
+
process.env['HOME'] = fakeHome;
|
|
34
|
+
// The node graph (canvas.db) lives in its own CRTR_HOME so spawnNode does not
|
|
35
|
+
// touch the fake user home; the read path we test is constructed by hand.
|
|
36
|
+
canvasHome = mkdtempSync(join(tmpdir(), 'crtr-fence-canvas-'));
|
|
37
|
+
process.env['CRTR_HOME'] = canvasHome;
|
|
38
|
+
});
|
|
39
|
+
after(() => {
|
|
40
|
+
closeDb();
|
|
41
|
+
resetScopeCache();
|
|
42
|
+
rmSync(canvasHome, { recursive: true, force: true });
|
|
43
|
+
rmSync(fakeHome, { recursive: true, force: true });
|
|
44
|
+
delete process.env['CRTR_HOME'];
|
|
45
|
+
if (prevHomeEnv === undefined)
|
|
46
|
+
delete process.env['HOME'];
|
|
47
|
+
else
|
|
48
|
+
process.env['HOME'] = prevHomeEnv;
|
|
49
|
+
});
|
|
50
|
+
test('a store nested in the crouter home does not leak into a canvas-artifact read', () => {
|
|
51
|
+
const node = spawnNode({ kind: 'general', cwd: fakeHome, parent: null }).node_id;
|
|
52
|
+
// A private store misresolved into the crouter home: `~/.crouter/.crouter/memory/`.
|
|
53
|
+
const leakedStore = join(fakeHome, '.crouter', '.crouter', 'memory');
|
|
54
|
+
mkdirSync(leakedStore, { recursive: true });
|
|
55
|
+
writeFileSync(join(leakedStore, 'private-thing.md'), '---\nkind: knowledge\n' +
|
|
56
|
+
"when-and-why-to-read: When X, read this because Y.\n" +
|
|
57
|
+
'file-read-visibility: content\n---\n' +
|
|
58
|
+
'PRIVATE ASSISTANT MEMORY MUST NOT LEAK\n');
|
|
59
|
+
// A canvas runtime artifact under the crouter home — the shape of every node
|
|
60
|
+
// report/context file the harness auto-reads.
|
|
61
|
+
const reportFile = join(fakeHome, '.crouter', 'canvas', 'nodes', 'somenode', 'reports', 'r.md');
|
|
62
|
+
mkdirSync(join(fakeHome, '.crouter', 'canvas', 'nodes', 'somenode', 'reports'), { recursive: true });
|
|
63
|
+
writeFileSync(reportFile, '# a node report\n');
|
|
64
|
+
const rendered = renderOnReadDocs(node, reportFile, new Set());
|
|
65
|
+
assert.ok(!rendered.includes('PRIVATE ASSISTANT MEMORY MUST NOT LEAK'), `crouter-home-nested store must not surface on a canvas read; got: ${rendered}`);
|
|
66
|
+
assert.ok(!rendered.includes('private-thing'), `crouter-home-nested store name must not surface on a canvas read; got: ${rendered}`);
|
|
67
|
+
});
|
|
@@ -13,19 +13,77 @@
|
|
|
13
13
|
* out under a long-lived spawning process (see `isZombie`'s doc) never comes
|
|
14
14
|
* back on its own. */
|
|
15
15
|
export declare function isPidAlive(pid: number | null | undefined): boolean;
|
|
16
|
-
/**
|
|
17
|
-
*
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
*
|
|
21
|
-
*
|
|
22
|
-
*
|
|
23
|
-
*
|
|
24
|
-
*
|
|
25
|
-
* —
|
|
26
|
-
*
|
|
27
|
-
*
|
|
28
|
-
*
|
|
16
|
+
/** THE single precision-aware identity compare, used EVERYWHERE two
|
|
17
|
+
* `composeIdentity` fingerprints are compared (this module's own
|
|
18
|
+
* `recordedPidLiveness`, `captureTeardownSnapshot`'s reuse check, and
|
|
19
|
+
* `killProcessTreePids`'s reuse check) — centralized so no call site can
|
|
20
|
+
* drift from this rule (review Major #1, Round 2 of the pid-reuse-liveness
|
|
21
|
+
* fix).
|
|
22
|
+
*
|
|
23
|
+
* The hazard this closes: `composeIdentity` appends the `#<ticks>` Linux
|
|
24
|
+
* discriminator only when `/proc/<pid>/stat` happens to be readable at
|
|
25
|
+
* capture time — so the SAME live process can be recorded plain `lstart` on
|
|
26
|
+
* one capture and `lstart#ticks` on another (a transient `/proc` read
|
|
27
|
+
* failure, or capturing on a platform/container without procfs). A naive
|
|
28
|
+
* `===` then reads that as a reuse (different string) and reports `dead` for
|
|
29
|
+
* a broker that never actually died — the false-DEAD direction, which can
|
|
30
|
+
* make `reviveNode`'s double-launch guard wrongly relaunch a SECOND broker
|
|
31
|
+
* onto a still-live session.
|
|
32
|
+
*
|
|
33
|
+
* Rule: split both on `#`. Bases (`lstart`) differ → definitely NOT a match
|
|
34
|
+
* (different process, or the same pid reused in a different second). Bases
|
|
35
|
+
* match and BOTH carry a `#ticks` suffix → match iff the suffixes are equal
|
|
36
|
+
* (this still catches genuine same-second reuse, the case `procStartTicks`
|
|
37
|
+
* exists for). Bases match and EITHER lacks the suffix → coarse-vs-fine,
|
|
38
|
+
* can't distinguish → MATCH (fail-open to same-process, matching this
|
|
39
|
+
* module's universal discipline: only a POSITIVE mismatch counts as reuse,
|
|
40
|
+
* never an absence of evidence). */
|
|
41
|
+
export declare function identitiesMatch(a: string, b: string): boolean;
|
|
42
|
+
/** Three-valued liveness verdict for a RECORDED node pid, identity-guarded
|
|
43
|
+
* against PID REUSE using the launch-time baseline (`pi_pid_identity`,
|
|
44
|
+
* captured by `recordPid`). On a heavily-forking host — esp. Linux, where
|
|
45
|
+
* low pids recycle fast — a broker's recorded pid can be reused by an
|
|
46
|
+
* unrelated process (or the `ps` probe confirming it can itself
|
|
47
|
+
* intermittently fail) within milliseconds of the broker dying; collapsing
|
|
48
|
+
* "confirmed alive", "confirmed dead/reused", and "couldn't confirm either
|
|
49
|
+
* way" into one boolean forces every caller to fail open to ALIVE on the
|
|
50
|
+
* third case — which, upstream in the daemon's grace-clock bookkeeping, is
|
|
51
|
+
* worse than misreporting one tick: a single such read WIPES the daemon's
|
|
52
|
+
* memory that the node had already been observed dead, restarting the whole
|
|
53
|
+
* grace window from scratch (the confirmed Round-2 crash-revive wedge — see
|
|
54
|
+
* `crtrd.ts`'s `handleNodeLiveness`). Exposing the third state lets callers
|
|
55
|
+
* decide for themselves whether "can't confirm" should act like alive or
|
|
56
|
+
* dead for their own direction of risk.
|
|
57
|
+
*
|
|
58
|
+
* Reuses the existing primitives — this is a classification layered on top
|
|
59
|
+
* of `isPidAlive` + `capturePidIdentities`, never a new `ps` parse:
|
|
60
|
+
* - `!isPidAlive(pid)` → `'dead'`. Load-bearing: this is what preserves
|
|
61
|
+
* zombie handling — `isPidAlive` already treats a gone pid AND a zombie
|
|
62
|
+
* (SIGKILLed detached broker awaiting reap) as dead, which is the entire
|
|
63
|
+
* reason a wedged zombie broker still gets revived. Do not bypass it.
|
|
64
|
+
* - `expectedIdentity == null` (legacy row / launch capture failed) →
|
|
65
|
+
* `'alive'` (preserve pre-identity behavior — no baseline to check
|
|
66
|
+
* against).
|
|
67
|
+
* - `ps` probe itself failed (`capturePidIdentities` → `null`) →
|
|
68
|
+
* `'indeterminate'`. THIS is the branch the Round-2 fix changes: it used
|
|
69
|
+
* to fail open to `true` (alive) here, which is the confirmed root cause
|
|
70
|
+
* of the wedge.
|
|
71
|
+
* - probe succeeded but has no row for this pid (alive per signal-0 a
|
|
72
|
+
* moment ago, gone now) → `'dead'`.
|
|
73
|
+
* - probe succeeded and the identity is present → `'alive'` iff
|
|
74
|
+
* `identitiesMatch` says so, else `'dead'` (positively reused). */
|
|
75
|
+
export type RecordedPidLiveness = 'alive' | 'dead' | 'indeterminate';
|
|
76
|
+
export declare function recordedPidLiveness(pid: number | null | undefined, expectedIdentity: string | null | undefined): RecordedPidLiveness;
|
|
77
|
+
/** Boolean teardown-direction adapter over `recordedPidLiveness`, kept so
|
|
78
|
+
* every pre-existing signalling caller stays behavior-identical: `dead` is
|
|
79
|
+
* the ONLY thing that reads false. `indeterminate` (can't confirm either
|
|
80
|
+
* way) reads true alongside `alive` — never signal a maybe-ours pid on mere
|
|
81
|
+
* uncertainty, matching this module's universal discipline (a POSITIVE
|
|
82
|
+
* mismatch is the only thing that changes behavior, never an absence of
|
|
83
|
+
* evidence). Callers that need to distinguish "confirmed alive" from
|
|
84
|
+
* "can't confirm" (e.g. a double-launch guard, where treating uncertainty as
|
|
85
|
+
* alive risks silently skipping a needed relaunch) must call
|
|
86
|
+
* `recordedPidLiveness` directly instead of this adapter. */
|
|
29
87
|
export declare function isRecordedPidAlive(pid: number | null | undefined, expectedIdentity: string | null | undefined): boolean;
|
|
30
88
|
/** SIGTERM a process GROUP by pid (the negative-pid convention) — best-effort,
|
|
31
89
|
* swallowing ESRCH (already gone). Shared by triggers.ts (cancel-time reap of
|
package/dist/core/canvas/pid.js
CHANGED
|
@@ -60,31 +60,118 @@ export function isPidAlive(pid) {
|
|
|
60
60
|
}
|
|
61
61
|
return !isZombie(pid);
|
|
62
62
|
}
|
|
63
|
-
/**
|
|
64
|
-
*
|
|
65
|
-
*
|
|
66
|
-
*
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
*
|
|
73
|
-
*
|
|
74
|
-
*
|
|
75
|
-
*
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
63
|
+
/** Split a `composeIdentity` fingerprint (`lstart` or `lstart#ticks`) into its
|
|
64
|
+
* coarse (`lstart`, second-granular, portable) and fine (`ticks`, Linux-only
|
|
65
|
+
* jiffies, best-effort) parts. `#` never appears in `lstart` itself (a `ps`
|
|
66
|
+
* timestamp), so the first `#` unambiguously separates them. */
|
|
67
|
+
function splitIdentity(identity) {
|
|
68
|
+
const i = identity.indexOf('#');
|
|
69
|
+
return i === -1 ? { base: identity, ticks: undefined } : { base: identity.slice(0, i), ticks: identity.slice(i + 1) };
|
|
70
|
+
}
|
|
71
|
+
/** THE single precision-aware identity compare, used EVERYWHERE two
|
|
72
|
+
* `composeIdentity` fingerprints are compared (this module's own
|
|
73
|
+
* `recordedPidLiveness`, `captureTeardownSnapshot`'s reuse check, and
|
|
74
|
+
* `killProcessTreePids`'s reuse check) — centralized so no call site can
|
|
75
|
+
* drift from this rule (review Major #1, Round 2 of the pid-reuse-liveness
|
|
76
|
+
* fix).
|
|
77
|
+
*
|
|
78
|
+
* The hazard this closes: `composeIdentity` appends the `#<ticks>` Linux
|
|
79
|
+
* discriminator only when `/proc/<pid>/stat` happens to be readable at
|
|
80
|
+
* capture time — so the SAME live process can be recorded plain `lstart` on
|
|
81
|
+
* one capture and `lstart#ticks` on another (a transient `/proc` read
|
|
82
|
+
* failure, or capturing on a platform/container without procfs). A naive
|
|
83
|
+
* `===` then reads that as a reuse (different string) and reports `dead` for
|
|
84
|
+
* a broker that never actually died — the false-DEAD direction, which can
|
|
85
|
+
* make `reviveNode`'s double-launch guard wrongly relaunch a SECOND broker
|
|
86
|
+
* onto a still-live session.
|
|
87
|
+
*
|
|
88
|
+
* Rule: split both on `#`. Bases (`lstart`) differ → definitely NOT a match
|
|
89
|
+
* (different process, or the same pid reused in a different second). Bases
|
|
90
|
+
* match and BOTH carry a `#ticks` suffix → match iff the suffixes are equal
|
|
91
|
+
* (this still catches genuine same-second reuse, the case `procStartTicks`
|
|
92
|
+
* exists for). Bases match and EITHER lacks the suffix → coarse-vs-fine,
|
|
93
|
+
* can't distinguish → MATCH (fail-open to same-process, matching this
|
|
94
|
+
* module's universal discipline: only a POSITIVE mismatch counts as reuse,
|
|
95
|
+
* never an absence of evidence). */
|
|
96
|
+
export function identitiesMatch(a, b) {
|
|
97
|
+
const ia = splitIdentity(a);
|
|
98
|
+
const ib = splitIdentity(b);
|
|
99
|
+
if (ia.base !== ib.base)
|
|
100
|
+
return false;
|
|
101
|
+
if (ia.ticks !== undefined && ib.ticks !== undefined)
|
|
102
|
+
return ia.ticks === ib.ticks;
|
|
103
|
+
return true;
|
|
104
|
+
}
|
|
105
|
+
export function recordedPidLiveness(pid, expectedIdentity) {
|
|
106
|
+
const debug = Boolean(process.env.CRTR_DEBUG_PID_LIVENESS);
|
|
107
|
+
const alive = isPidAlive(pid);
|
|
108
|
+
if (!alive) {
|
|
109
|
+
if (debug)
|
|
110
|
+
logPidLivenessDecision(pid, alive, expectedIdentity, 'no-probe', 'isPidAlive=false → dead', 'dead');
|
|
111
|
+
return 'dead'; // gone/zombie → dead (unchanged path)
|
|
112
|
+
}
|
|
113
|
+
if (expectedIdentity == null) {
|
|
114
|
+
if (debug)
|
|
115
|
+
logPidLivenessDecision(pid, alive, expectedIdentity, 'no-probe', 'no baseline → trust isPidAlive', 'alive');
|
|
116
|
+
return 'alive'; // no baseline → trust isPidAlive (legacy behavior)
|
|
117
|
+
}
|
|
81
118
|
const current = capturePidIdentities([pid]);
|
|
82
|
-
if (current == null)
|
|
83
|
-
|
|
119
|
+
if (current == null) {
|
|
120
|
+
if (debug)
|
|
121
|
+
logPidLivenessDecision(pid, alive, expectedIdentity, 'probe-null', 'ps probe failed → indeterminate', 'indeterminate');
|
|
122
|
+
return 'indeterminate'; // probe itself failed → we don't know (THE fix: was fail-open alive)
|
|
123
|
+
}
|
|
84
124
|
const actual = current.get(pid);
|
|
85
|
-
if (actual === undefined)
|
|
86
|
-
|
|
87
|
-
|
|
125
|
+
if (actual === undefined) {
|
|
126
|
+
if (debug)
|
|
127
|
+
logPidLivenessDecision(pid, alive, expectedIdentity, 'no-entry', 'no identity row for pid (race) → dead', 'dead');
|
|
128
|
+
return 'dead'; // isPidAlive said alive but it's gone now → dead
|
|
129
|
+
}
|
|
130
|
+
const matches = identitiesMatch(actual, expectedIdentity);
|
|
131
|
+
if (debug)
|
|
132
|
+
logPidLivenessDecision(pid, alive, expectedIdentity, 'value', matches ? 'identity matches expected → alive' : 'identity differs from expected → REUSED → dead', matches ? 'alive' : 'dead', actual);
|
|
133
|
+
return matches ? 'alive' : 'dead';
|
|
134
|
+
}
|
|
135
|
+
/** Boolean teardown-direction adapter over `recordedPidLiveness`, kept so
|
|
136
|
+
* every pre-existing signalling caller stays behavior-identical: `dead` is
|
|
137
|
+
* the ONLY thing that reads false. `indeterminate` (can't confirm either
|
|
138
|
+
* way) reads true alongside `alive` — never signal a maybe-ours pid on mere
|
|
139
|
+
* uncertainty, matching this module's universal discipline (a POSITIVE
|
|
140
|
+
* mismatch is the only thing that changes behavior, never an absence of
|
|
141
|
+
* evidence). Callers that need to distinguish "confirmed alive" from
|
|
142
|
+
* "can't confirm" (e.g. a double-launch guard, where treating uncertainty as
|
|
143
|
+
* alive risks silently skipping a needed relaunch) must call
|
|
144
|
+
* `recordedPidLiveness` directly instead of this adapter. */
|
|
145
|
+
export function isRecordedPidAlive(pid, expectedIdentity) {
|
|
146
|
+
return recordedPidLiveness(pid, expectedIdentity) !== 'dead';
|
|
147
|
+
}
|
|
148
|
+
/** Diagnostic-only decision trace for `recordedPidLiveness`, gated entirely
|
|
149
|
+
* behind `CRTR_DEBUG_PID_LIVENESS` — callers must check the env var
|
|
150
|
+
* themselves before calling this so the (tiny) formatting cost is paid ONLY
|
|
151
|
+
* when the env var is set; this function does no env check of its own.
|
|
152
|
+
* Written to trace the intermittent Linux-only CI crash-revive wedge
|
|
153
|
+
* (`broker-crash-teardown.test.ts`'s 'CRASH → grace-revive RESUME'): the
|
|
154
|
+
* goal is for a single stderr line per call to make BOTH a passing run ("pid
|
|
155
|
+
* read dead → revive") and a wedged run ("read alive via <branch>") fully
|
|
156
|
+
* self-explaining, without changing any return value. `capturePidIdentities`
|
|
157
|
+
* outcome is reported as one of three DISTINCT cases — `probe-null` (the
|
|
158
|
+
* `ps` probe itself failed), `no-entry` (probe succeeded but the map has no
|
|
159
|
+
* row for this pid), or `value` (probe succeeded and found an identity,
|
|
160
|
+
* printed verbatim) — plus `no-probe` for the two branches that return
|
|
161
|
+
* before `capturePidIdentities` is ever called. Kept as permanent
|
|
162
|
+
* observability for this primitive, not a throwaway debug print: pid-reuse
|
|
163
|
+
* liveness is subtle enough that the next person chasing a wedge here should
|
|
164
|
+
* find this already in place. */
|
|
165
|
+
function logPidLivenessDecision(pid, isPidAliveResult, expectedIdentity, probeOutcome, branch, returned, actualIdentity) {
|
|
166
|
+
const parts = [
|
|
167
|
+
`pid=${String(pid)}`,
|
|
168
|
+
`isPidAlive=${isPidAliveResult}`,
|
|
169
|
+
`expectedIdentity=${expectedIdentity === undefined ? 'undefined' : expectedIdentity === null ? 'null' : JSON.stringify(expectedIdentity)}`,
|
|
170
|
+
`capturePidIdentities=${probeOutcome}${probeOutcome === 'value' ? `(${JSON.stringify(actualIdentity)})` : ''}`,
|
|
171
|
+
`branch=${JSON.stringify(branch)}`,
|
|
172
|
+
`returned=${returned}`,
|
|
173
|
+
];
|
|
174
|
+
process.stderr.write(`[recordedPidLiveness] ${parts.join(' ')}\n`);
|
|
88
175
|
}
|
|
89
176
|
/** SIGTERM a process GROUP by pid (the negative-pid convention) — best-effort,
|
|
90
177
|
* swallowing ESRCH (already gone). Shared by triggers.ts (cancel-time reap of
|
|
@@ -294,7 +381,9 @@ export function captureTeardownSnapshot(rootPid, expectedIdentity) {
|
|
|
294
381
|
if (table === null)
|
|
295
382
|
return { tree: [rootPid], identities: null, reused: false };
|
|
296
383
|
const currentRootIdentity = table.identities.get(rootPid);
|
|
297
|
-
const reused = expectedIdentity != null &&
|
|
384
|
+
const reused = expectedIdentity != null &&
|
|
385
|
+
currentRootIdentity !== undefined &&
|
|
386
|
+
!identitiesMatch(currentRootIdentity, expectedIdentity);
|
|
298
387
|
if (reused)
|
|
299
388
|
return { tree: [], identities: null, reused: true };
|
|
300
389
|
const tree = [rootPid, ...descendantPidsFrom(rootPid, table.childrenOf)];
|
|
@@ -362,7 +451,7 @@ export function killProcessTreePids(pids, signal = 'SIGTERM', identities) {
|
|
|
362
451
|
if (identities != null && current != null) {
|
|
363
452
|
const expected = identities.get(pid);
|
|
364
453
|
const actual = current.get(pid);
|
|
365
|
-
const reused = expected !== undefined && actual !== undefined && actual
|
|
454
|
+
const reused = expected !== undefined && actual !== undefined && !identitiesMatch(actual, expected);
|
|
366
455
|
if (reused)
|
|
367
456
|
continue;
|
|
368
457
|
}
|
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
// self-guards the double-spawn. The selection predicate (`isDisconnected`) is
|
|
14
14
|
// pure so the scope rules are unit-testable without booting a single process.
|
|
15
15
|
import { listNodes, getNode } from '../canvas/index.js';
|
|
16
|
-
import {
|
|
16
|
+
import { recordedPidLiveness } from '../canvas/pid.js';
|
|
17
17
|
import { reviveNode } from './revive.js';
|
|
18
18
|
/** Statuses a node lands in DELIBERATELY: a worker that finished its own work
|
|
19
19
|
* (`done`) or a node a human closed/reaped (`canceled`). The daemon never
|
|
@@ -56,7 +56,13 @@ export function listDisconnected() {
|
|
|
56
56
|
const out = [];
|
|
57
57
|
for (const row of listNodes()) {
|
|
58
58
|
const meta = getNode(row.node_id);
|
|
59
|
-
|
|
59
|
+
// IDENTITY-aware AND requires CONFIRMED alive: a bare isPidAlive would
|
|
60
|
+
// misread a recycled pi_pid as "engine running" and SKIP the node, so it
|
|
61
|
+
// never reaches reviveNode's own identity-aware double-launch guard
|
|
62
|
+
// (Round 2 of the pid-reuse-liveness fix, fold-in Major). `dead` OR
|
|
63
|
+
// `indeterminate` both count as disconnected/eligible here — only a
|
|
64
|
+
// positively confirmed-alive read is treated as "still connected".
|
|
65
|
+
if (meta !== null && isDisconnected(meta, (pid) => recordedPidLiveness(pid, meta.pi_pid_identity) === 'alive'))
|
|
60
66
|
out.push(meta);
|
|
61
67
|
}
|
|
62
68
|
return out;
|
|
@@ -29,7 +29,7 @@ import { fanDoctrineWake } from './close.js';
|
|
|
29
29
|
import { buildPiArgv } from './launch.js';
|
|
30
30
|
import { buildReviveKickoff, drainBearings } from './kickoff.js';
|
|
31
31
|
import { headlessBrokerHost } from './host.js';
|
|
32
|
-
import {
|
|
32
|
+
import { recordedPidLiveness } from '../canvas/pid.js';
|
|
33
33
|
import { reconcileBootLiveness } from '../canvas/boot.js';
|
|
34
34
|
import { clearFault } from './fault.js';
|
|
35
35
|
import { clearInjectedDocs } from '../substrate/injected-store.js';
|
|
@@ -89,14 +89,24 @@ export function reviveNode(nodeId, opts) {
|
|
|
89
89
|
// actually matters, so it's safe to call on every revive regardless of
|
|
90
90
|
// entry point (daemon tick, or a direct/manual `node lifecycle revive`).
|
|
91
91
|
reconcileBootLiveness();
|
|
92
|
-
// Double-revive guard:
|
|
93
|
-
//
|
|
94
|
-
// still
|
|
95
|
-
// another path — re-launching would put
|
|
96
|
-
// session file. No-op. A bare isPidAlive would
|
|
97
|
-
// forking, esp. Linux) as "still alive" and
|
|
92
|
+
// Double-revive guard: identity-aware liveness on pi_pid, requiring
|
|
93
|
+
// CONFIRMED alive — not the fail-open-to-alive boolean adapter. A node
|
|
94
|
+
// whose broker pid is CONFIRMED still running as that same broker (not a
|
|
95
|
+
// reused pid) was already revived by another path — re-launching would put
|
|
96
|
+
// a SECOND broker on the same session file. No-op. A bare isPidAlive would
|
|
97
|
+
// misread a reused pid (heavy forking, esp. Linux) as "still alive" and
|
|
98
|
+
// skip the relaunch forever; conversely, treating an INDETERMINATE read
|
|
99
|
+
// (can't confirm either way, e.g. a transient `ps` probe failure) as
|
|
100
|
+
// "alive" here would risk the opposite failure — silently skipping a
|
|
101
|
+
// needed relaunch and stranding the node with no engine. So this guard
|
|
102
|
+
// blocks ONLY on a positively confirmed-alive read; `dead` AND
|
|
103
|
+
// `indeterminate` both proceed to relaunch. We accept that tradeoff: an
|
|
104
|
+
// `indeterminate` read CAN come from a genuinely live broker whose identity
|
|
105
|
+
// `ps` probe transiently failed, so proceeding risks a rare double-launch —
|
|
106
|
+
// but stranding a truly-dead node with no engine is the worse failure, and
|
|
107
|
+
// recovering from it is far harder, so we bias toward relaunching.
|
|
98
108
|
const live = getNode(nodeId) ?? meta;
|
|
99
|
-
if (
|
|
109
|
+
if (recordedPidLiveness(live.pi_pid, live.pi_pid_identity) === 'alive') {
|
|
100
110
|
return {
|
|
101
111
|
window: null,
|
|
102
112
|
session: live.tmux_session ?? null,
|
|
@@ -58,6 +58,7 @@ import { pathExists, readText, walkFiles } from '../fs-utils.js';
|
|
|
58
58
|
import { parseFrontmatterGeneric } from '../frontmatter.js';
|
|
59
59
|
import { evalCondition } from '../predicate.js';
|
|
60
60
|
import { listAllMemoryDocs } from '../memory-resolver.js';
|
|
61
|
+
import { userScopeRoot } from '../scope.js';
|
|
61
62
|
import { assembleNodeSubject, buildCeilingIndex, displayName, effectiveRung, gatePasses, normalizeDocName, parseSubstrateDoc, parseSubstrateFrontmatter, previewLine, resolveDocName, } from './index.js';
|
|
62
63
|
import { cachedSubstrateDocs } from './session-cache.js';
|
|
63
64
|
// Ancestor dirs we never look inside for a `.crouter/memory/` store (the read
|
|
@@ -156,16 +157,26 @@ function hasExplicitTrigger(doc) {
|
|
|
156
157
|
* `.crouter/memory/`, walking from the read file up to $HOME (or the
|
|
157
158
|
* filesystem root for a read outside $HOME), skipping junk ancestors. The
|
|
158
159
|
* user-global `~/.crouter/memory/` store is deliberately skipped: otherwise
|
|
159
|
-
* every file read under $HOME would surface unrelated user-wide preferences.
|
|
160
|
+
* every file read under $HOME would surface unrelated user-wide preferences.
|
|
161
|
+
*
|
|
162
|
+
* The crouter home (`~/.crouter`, the user scope root) is skipped for the same
|
|
163
|
+
* reason it is never a project scope root (scope.ts's `isProjectScopeDir`): it
|
|
164
|
+
* is the user scope, not a project ancestor. Without this fence, reading any
|
|
165
|
+
* canvas runtime artifact (`~/.crouter/canvas/nodes/.../reports|context/...`)
|
|
166
|
+
* walks up THROUGH `~/.crouter` and would inject whatever lives at
|
|
167
|
+
* `~/.crouter/.crouter/memory/` — e.g. a differently-scoped node whose cwd
|
|
168
|
+
* resolved its project store there — leaking one node's private memory into
|
|
169
|
+
* every unrelated node that reads a canvas file. */
|
|
160
170
|
function positionalCandidates(absReadFile) {
|
|
161
171
|
const out = [];
|
|
162
172
|
const seenDocPaths = new Set();
|
|
163
173
|
const home = realpathOrSelf(homedir());
|
|
174
|
+
const userRoot = realpathOrSelf(userScopeRoot());
|
|
164
175
|
const fsRoot = parse(absReadFile).root;
|
|
165
176
|
let dir = dirname(absReadFile);
|
|
166
177
|
let depth = 0;
|
|
167
178
|
while (true) {
|
|
168
|
-
if (dir !== home && !isJunkAncestor(dir)) {
|
|
179
|
+
if (dir !== home && dir !== userRoot && !isJunkAncestor(dir)) {
|
|
169
180
|
const memDir = join(dir, CRTR_DIR_NAME, 'memory');
|
|
170
181
|
if (pathExists(memDir)) {
|
|
171
182
|
for (const file of safeWalkMd(memDir)) {
|
package/dist/daemon/crtrd.js
CHANGED
|
@@ -64,7 +64,7 @@ import { FAULT_QUIET_MS, readFault, recordFault } from '../core/runtime/fault.js
|
|
|
64
64
|
import { fanDoctrineWake } from '../core/runtime/close.js';
|
|
65
65
|
import { logger } from '../core/log.js';
|
|
66
66
|
export { FAULT_QUIET_MS };
|
|
67
|
-
import { isPidAlive, isRecordedPidAlive } from '../core/canvas/pid.js';
|
|
67
|
+
import { isPidAlive, isRecordedPidAlive, recordedPidLiveness } from '../core/canvas/pid.js';
|
|
68
68
|
import { reconcileBootLiveness } from '../core/canvas/boot.js';
|
|
69
69
|
import { listLivePanes, tearDownNode } from '../core/runtime/placement.js';
|
|
70
70
|
import { reviveNode } from '../core/runtime/revive.js';
|
|
@@ -705,14 +705,24 @@ async function handleNodeLiveness(row, now, revivedThisTick) {
|
|
|
705
705
|
// its turn is over, and the engine never exited — past the grace it kills the
|
|
706
706
|
// engine so the dead-pid refresh branch below revives it fresh.
|
|
707
707
|
//
|
|
708
|
-
// Liveness here is IDENTITY-aware
|
|
708
|
+
// Liveness here is IDENTITY-aware AND requires CONFIRMED alive, not a bare
|
|
709
|
+
// isPidAlive or the fail-open-to-alive boolean adapter: on a heavily-
|
|
709
710
|
// forking host (esp. Linux, where low pids recycle fast) a just-dead
|
|
710
|
-
// broker's pid can be reused by an unrelated process
|
|
711
|
-
//
|
|
712
|
-
// alive"
|
|
713
|
-
//
|
|
714
|
-
// the
|
|
715
|
-
|
|
711
|
+
// broker's pid can be reused by an unrelated process within milliseconds,
|
|
712
|
+
// AND the `ps` probe confirming identity can itself intermittently fail.
|
|
713
|
+
// Either misread as "alive" here doesn't just misreport this one tick — it
|
|
714
|
+
// clears `unhealthySince` (the grace clock), discarding the daemon's memory
|
|
715
|
+
// that the node had already been observed dead, so the VERY NEXT tick treats
|
|
716
|
+
// a dead-for-20s+ node as a brand-new first observation and restarts the
|
|
717
|
+
// whole grace window from scratch (the confirmed Round-2 crash-revive wedge:
|
|
718
|
+
// a single `probe-null` read on a Linux CI runner wedged
|
|
719
|
+
// broker-crash-teardown.test.ts's 'CRASH → grace-revive RESUME'). So only a
|
|
720
|
+
// POSITIVELY CONFIRMED-alive read (`recordedPidLiveness === 'alive'`) may
|
|
721
|
+
// clear the clock; `'dead'` AND `'indeterminate'` both fall through to the
|
|
722
|
+
// ordinary not-alive path below, which now CONTRIBUTES to the grace window
|
|
723
|
+
// instead of zeroing it — robust to a flicker, and still revives promptly
|
|
724
|
+
// once a node is truly gone for the full grace duration.
|
|
725
|
+
if (pid != null && recordedPidLiveness(pid, row.pi_pid_identity) === 'alive') {
|
|
716
726
|
unhealthySince.delete(id);
|
|
717
727
|
clearStrandedRelaunchState(id); // a live engine means the stranding resolved
|
|
718
728
|
handleYieldStall(row, pid, now);
|