@north-light/crouter 0.3.154

package/dist/core/__tests__/host-teardown-process-group.test.js

@@ -0,0 +1,476 @@
+// Run with: node --import tsx/esm --test src/core/__tests__/host-teardown-process-group.test.ts
+//
+// Regression cover for crouton-labs/crouter#98: a canceled/wedged child node
+// ORPHANS its subprocess tree (bash → test runner → spawned app) because
+// teardown sent a single SIGTERM to the broker pid ONLY — no group signal, no
+// SIGKILL escalation, and (the deeper gap) no awareness that the pi SDK's
+// bash tool spawns its shell child `detached: true`, which calls `setsid()`
+// and lands that child in a brand-new process GROUP of its own, distinct from
+// the broker's. A plain `kill(brokerPid, sig)` — and even a GROUP kill of
+// `-brokerPid` — never reaches a descendant shaped that way; only walking the
+// process TREE by ppid (which `setsid()` never changes) and signaling each
+// discovered descendant's OWN group actually reaps it.
+//
+// These tests drive `headlessBrokerHost.teardown()` directly (no real pi
+// engine, no tmux) against REAL fabricated process trees standing in for
+// "broker (session leader) → descendant", asserting against REAL pids read
+// back with `isPidAlive` — never a mock.
+//
+// (1) BUG LOCKED — four locks:
+//     • "dead broker orphans a same-group descendant" — the fake broker is
+//       SIGKILLed out from under us BEFORE teardown runs (so `getNode().pi_pid`
+//       is a pid that's already gone), leaving its non-detached child alive in
+//       the SAME process group. A plain `kill(deadPid, 'SIGTERM')` throws
+//       ESRCH and touches nothing else; the broker's own group is what must
+//       be reaped (its pgid is untouched by the kernel reparenting the child's
+//       ppid to init, which happens immediately on the broker's exit).
+//     • "wedged broker orphans a DETACHED descendant" (the exact GH repro
+//       shape — the current test's core addition): the broker is alive but
+//       unreachable (nothing listening on view.sock), and its descendant
+//       detached into its OWN process group exactly like the pi bash tool
+//       does. We assert structurally that the descendant's pgid differs from
+//       the broker's pid — proof a bare `kill(-brokerPid)` cannot reach it —
+//       then prove the real `teardown()` (ppid-tree-walk) reaps BOTH the
+//       broker and the detached descendant.
+//     • "detached descendant ignores SIGTERM" (a stubborn/wedged leaf): must
+//       survive the initial tree-wide SIGTERM and only die once the SIGKILL
+//       escalation fires after the grace window — proving the ladder actually
+//       reaches SIGKILL against a descendant's OWN group, not just the
+//       broker's.
+//     • "CONNECTED broker exits cleanly after the shutdown frame but leaves a
+//       DETACHED descendant alive" (review finding 1, the CONNECTED/shutdown
+//       path — the fake broker here is a real unix-socket server on the
+//       node's view.sock that ACCEPTS the connection + shutdown frame, then
+//       exits WITHOUT reaping its own detached descendant): proves teardown's
+//       post-shutdown-frame liveness/escalation check reuses the ONE tree
+//       snapshot taken before the broker could exit, rather than re-walking
+//       `ps` by ppid after the broker is gone — a re-walk finds nothing the
+//       instant the kernel reparents the surviving descendant away from the
+//       now-dead broker's pid, silently skipping escalation forever.
+//
+// (2) HOW THIS FAILS IF THE BUG REGRESSES — reverting to a single
+//     `process.kill(pid, 'SIGTERM')` (or even `killProcessGroup(pid, sig)`
+//     alone, with no ppid-tree walk) of the broker pid leaves the detached-
+//     descendant assertions in this file red: those pids stay alive
+//     indefinitely (the `waitFor` polls time out), and the SIGTERM-ignoring
+//     detached leaf never dies at all. Reverting ONLY the connected-path fix
+//     (having `teardown()` re-derive the tree via a fresh `isProcessTreeAlive`/
+//     `descendantPids` walk after the shutdown frame, instead of reusing the
+//     pre-signal snapshot) leaves just the fourth test above red: the CONNECTED
+//     broker's detached descendant survives forever because the re-walk loses
+//     it the instant the broker exits.
+import { test, before, after } from 'node:test';
+import assert from 'node:assert/strict';
+import { spawn } from 'node:child_process';
+import { mkdtempSync, readFileSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { createHarness } from './helpers/harness.js';
+import { isPidAlive, capturePidIdentities, killProcessTreePids, captureTeardownSnapshot } from '../canvas/pid.js';
+import { recordPid } from '../canvas/canvas.js';
+import { openDb } from '../canvas/db.js';
+import { headlessBrokerHost } from '../runtime/host.js';
+let h;
+let root;
+let scratch;
+// Every pid this suite spawns as a fixture (leader OR descendant), tracked the
+// INSTANT it is known — review finding 2: an assertion or setup failure
+// partway through a test must not leak a `sleep 300` (or a leader waiting on
+// one) past this file's own `after()`, which otherwise only disposes the
+// harness/scratch dir and never touches these fabricated OS processes. Swept
+// best-effort in `after()` below, in ADDITION to (never instead of) each
+// test's real assertion that `headlessBrokerHost.teardown()` itself reaped them.
+const spawnedPids = new Set();
+function trackPid(pid) {
+    if (pid != null)
+        spawnedPids.add(pid);
+}
+before(async () => {
+    h = await createHarness({ headless: true, sessionPrefix: 'crtr-hostpg' });
+    root = h.spawnRoot('host-teardown-process-group suite root');
+    scratch = mkdtempSync(join(tmpdir(), 'crtr-hostpg-'));
+});
+after(async () => {
+    if (h !== undefined)
+        await h.dispose();
+    rmSync(scratch, { recursive: true, force: true });
+    // Best-effort final sweep (review finding 2): a test that asserted before
+    // teardown finished reaping, or threw mid-fixture-setup, can leave a tracked
+    // pid (or its whole process GROUP, for the detached-leader fixtures) alive.
+    // Every signal here is best-effort — ESRCH (already gone) is swallowed — so
+    // this is a harmless no-op against a suite that reaped everything cleanly.
+    for (const pid of spawnedPids) {
+        try {
+            process.kill(pid, 'SIGKILL');
+        }
+        catch { /* already gone */ }
+        try {
+            process.kill(-pid, 'SIGKILL');
+        }
+        catch { /* already gone, or never its own group leader */ }
+    }
+});
+/** Wait for `leader` to write a pid (as a newline-terminated decimal number) to
+ *  its stdout — with an actual FAILURE path (review finding 2): a bare
+ *  `stdout.on('data', ...)` promise as used here previously had no `exit`/
+ *  `close`/timeout rejection, so a leader that crashed or exited before ever
+ *  writing its line left the `await` hanging FOREVER (no output, no timeout,
+ *  the whole suite process wedged). This rejects on the leader exiting first,
+ *  a stream error, OR a `timeoutMs` ceiling — whichever comes first — and
+ *  always tears down its own listeners so it settles exactly once. */
+async function readPidFromStdout(leader, label, timeoutMs = 5_000) {
+    return new Promise((resolvePid, reject) => {
+        let buf = '';
+        let settled = false;
+        const cleanup = () => {
+            clearTimeout(timer);
+            leader.stdout?.off('data', onData);
+            leader.off('error', onError);
+            leader.off('exit', onExit);
+        };
+        const onData = (chunk) => {
+            if (settled)
+                return;
+            buf += chunk.toString('utf8');
+            const nl = buf.indexOf('\n');
+            if (nl === -1)
+                return;
+            settled = true;
+            cleanup();
+            resolvePid(Number.parseInt(buf.slice(0, nl), 10));
+        };
+        const onError = (err) => {
+            if (settled)
+                return;
+            settled = true;
+            cleanup();
+            reject(err);
+        };
+        const onExit = (code, signal) => {
+            if (settled)
+                return;
+            settled = true;
+            cleanup();
+            reject(new Error(`${label}: leader exited (code=${String(code)}, signal=${String(signal)}) before writing its pid`));
+        };
+        const timer = setTimeout(() => {
+            if (settled)
+                return;
+            settled = true;
+            cleanup();
+            reject(new Error(`${label}: timed out after ${timeoutMs}ms waiting for the leader to write its pid`));
+        }, timeoutMs);
+        leader.stdout?.on('data', onData);
+        leader.once('error', onError);
+        leader.once('exit', onExit);
+    });
+}
+/** Spawn a fake "broker" (its own session/process group, like the real
+ *  `launch()`) that in turn backgrounds a plain (non-detached, SAME-group)
+ *  descendant — used only for the "dead broker, shared-group descendant"
+ *  case, where the broker's own pgid (untouched by kernel reparenting) is
+ *  what teardown must still reach. Hands back both pids once the descendant
+ *  has actually started. */
+async function spawnBrokerWithSharedGroupDescendant(tag, descendantScript = 'sleep 300') {
+    const pidFile = join(scratch, `${tag}.pid`);
+    const leader = spawn('bash', ['-c', `(${descendantScript}) & echo $! > ${pidFile}; wait`], {
+        detached: true,
+        stdio: 'ignore',
+    });
+    if (leader.pid == null)
+        throw new Error('failed to spawn fake broker leader');
+    trackPid(leader.pid);
+    const descendantPid = await h.waitFor(() => {
+        try {
+            const record = readFileSync(pidFile, 'utf8');
+            if (!/^\d+\n$/.test(record))
+                return false;
+            const pid = Number(record.slice(0, -1));
+            return Number.isSafeInteger(pid) && pid > 0 ? pid : false;
+        }
+        catch (error) {
+            if (error.code === 'ENOENT')
+                return false;
+            throw error;
+        }
+    }, { label: `${tag}: valid descendant pid written` });
+    trackPid(descendantPid);
+    await h.waitFor(() => isPidAlive(descendantPid), { label: `${tag}: descendant alive before teardown` });
+    return { leaderPid: leader.pid, descendantPid };
+}
+/** Spawn a fake "broker" that is ITSELF `detached: true` (own session/process
+ *  group, exactly like the real `launch()`), which in turn spawns `descendantCmd`
+ *  with `detached: true` TOO — mirroring the real pi bash tool's shell child
+ *  (verified in `@earendil-works/pi-agent-core`'s `harness/env/nodejs.js`),
+ *  which lands in a brand-new process group distinct from the broker's. The
+ *  leader is a tiny inline Node script (not bash — macOS ships no `setsid`
+ *  binary to force a real detach from a shell) that writes the descendant's
+ *  pid to stdout once spawned, then idles. Hands back both pids once the
+ *  descendant has actually started AND is confirmed to be in its OWN group. */
+async function spawnBrokerWithDetachedDescendant(descendantCmd) {
+    const leaderScript = [
+        "const { spawn } = require('node:child_process');",
+        'const [cmd, ...args] = process.argv.slice(1);',
+        "const child = spawn(cmd, args, { detached: true, stdio: 'ignore' });",
+        'child.unref();',
+        "process.stdout.write(String(child.pid) + '\\n');",
+        'setInterval(() => {}, 1000);',
+    ].join('\n');
+    const leader = spawn(process.execPath, ['-e', leaderScript, ...descendantCmd], {
+        detached: true,
+        stdio: ['ignore', 'pipe', 'ignore'],
+    });
+    if (leader.pid == null)
+        throw new Error('failed to spawn fake detached broker leader');
+    leader.unref();
+    const leaderPid = leader.pid;
+    trackPid(leaderPid);
+    const descendantPid = await readPidFromStdout(leader, 'spawnBrokerWithDetachedDescendant');
+    trackPid(descendantPid);
+    assert.ok(Number.isInteger(descendantPid) && descendantPid > 0, 'parsed a real detached descendant pid');
+    await h.waitFor(() => isPidAlive(descendantPid), { label: 'detached descendant alive before teardown' });
+    return { leaderPid, descendantPid };
+}
+/** `ps -o pgid= -p <pid>` as a number, or `null` if the pid has no row (already
+ *  gone). Used only to prove structurally that a detached descendant's pgid
+ *  differs from its broker's pid — never used to signal anything. */
+async function readPgid(pid) {
+    const { execFileSync } = await import('node:child_process');
+    try {
+        const out = execFileSync('ps', ['-o', 'pgid=', '-p', String(pid)], { encoding: 'utf8' });
+        const parsed = Number.parseInt(out.trim(), 10);
+        return Number.isInteger(parsed) ? parsed : null;
+    }
+    catch {
+        return null;
+    }
+}
+/** Spawn a fake "broker" that is ITSELF `detached: true` (own session/process
+ *  group, mirroring `launch()`) AND listens as a REAL unix-socket server on
+ *  `sockPath` (the exact path `headlessBrokerHost.teardown()` connects to). It
+ *  also spawns `descendantCmd` `detached: true` (own group, mirroring the pi
+ *  bash tool's shell child) before it starts listening, so the descendant is
+ *  discoverable via ppid at the moment teardown snapshots the tree. On the
+ *  FIRST byte received over an accepted connection (i.e. the real `shutdown`
+ *  frame `teardown()` writes) it calls `process.exit(0)` IMMEDIATELY —
+ *  modeling a `dispose()` that exits without ever reaping its own detached
+ *  descendant. Used only by the "review finding 1" CONNECTED-path regression
+ *  test below. Hands back both pids once the descendant is confirmed alive. */
+async function spawnBrokerWithDetachedDescendantAndSocket(sockPath, descendantCmd) {
+    const leaderScript = [
+        "const net = require('node:net');",
+        "const { spawn } = require('node:child_process');",
+        'const [sockPath, cmd, ...args] = process.argv.slice(1);',
+        "const child = spawn(cmd, args, { detached: true, stdio: 'ignore' });",
+        'child.unref();',
+        'const server = net.createServer((conn) => {',
+        "  conn.once('data', () => { process.exit(0); });",
+        "  conn.once('end', () => { process.exit(0); });",
+        '});',
+        'server.listen(sockPath, () => {',
+        "  process.stdout.write(String(child.pid) + '\\n');",
+        '});',
+    ].join('\n');
+    const leader = spawn(process.execPath, ['-e', leaderScript, sockPath, ...descendantCmd], {
+        detached: true,
+        stdio: ['ignore', 'pipe', 'ignore'],
+    });
+    if (leader.pid == null)
+        throw new Error('failed to spawn fake connected-broker leader');
+    leader.unref();
+    const leaderPid = leader.pid;
+    trackPid(leaderPid);
+    const descendantPid = await readPidFromStdout(leader, 'spawnBrokerWithDetachedDescendantAndSocket');
+    trackPid(descendantPid);
+    assert.ok(Number.isInteger(descendantPid) && descendantPid > 0, 'parsed a real detached descendant pid (connected-broker case)');
+    await h.waitFor(() => isPidAlive(descendantPid), {
+        label: 'detached descendant alive before teardown (connected-broker case)',
+    });
+    return { leaderPid, descendantPid };
+}
+test('dead broker orphans a SAME-group descendant — teardown reaps it via the broker\'s own group signal', async () => {
+    const { leaderPid, descendantPid } = await spawnBrokerWithSharedGroupDescendant('dead-broker');
+    // Simulate the exact GH repro: the broker itself is already gone (crashed /
+    // force-killed / wedged-then-reaped) BEFORE teardown ever runs — a plain
+    // `kill(leaderPid, 'SIGTERM')` would ESRCH here and touch nothing else.
+    process.kill(leaderPid, 'SIGKILL');
+    await h.waitFor(() => !isPidAlive(leaderPid), { label: 'fake broker leader is dead' });
+    assert.equal(isPidAlive(descendantPid), true, 'descendant survives its leader (orphaned, same pgid)');
+    const id = h.fabricateBrokerNode({ parent: root, pi_pid: leaderPid, status: 'active' });
+    headlessBrokerHost.teardown(id);
+    await h.waitFor(() => !isPidAlive(descendantPid), {
+        timeoutMs: 6_000,
+        label: 'orphaned same-group descendant reaped by the broker group kill',
+    });
+});
+test('wedged (alive, unreachable) broker with a DETACHED descendant — process-group kill alone cannot reach it; ' +
+    'the ppid-tree-walk teardown does', async () => {
+    const { leaderPid, descendantPid } = await spawnBrokerWithDetachedDescendant(['sleep', '300']);
+    assert.equal(isPidAlive(leaderPid), true, 'fake broker leader alive before teardown');
+    assert.equal(isPidAlive(descendantPid), true, 'detached descendant alive before teardown');
+    // Structural proof that a bare process-GROUP kill of the broker can never
+    // reach this descendant: `process.kill(-pid, sig)` only reaches processes
+    // whose pgid === pid. The detached descendant called its own setsid(), so
+    // its pgid is its OWN pid, not the broker's — confirmed here without
+    // signaling anything.
+    const leaderPgid = await readPgid(leaderPid);
+    const descendantPgid = await readPgid(descendantPid);
+    assert.equal(leaderPgid, leaderPid, 'broker leader is its own group leader (detached launch)');
+    assert.equal(descendantPgid, descendantPid, 'descendant is ALSO its own group leader (detached spawn)');
+    assert.notEqual(descendantPgid, leaderPid, 'descendant pgid differs from the broker pid — a group kill of ' +
+        '-brokerPid structurally cannot reach it');
+    // No real broker is listening on this node's view.sock, so teardown's
+    // connect() always fails here — exactly the "wedged, unreachable" path.
+    // The broker is still alive at the OS level (not yet exited), so the
+    // ppid link from broker → descendant is intact for the tree walk.
+    const id = h.fabricateBrokerNode({ parent: root, pi_pid: leaderPid, status: 'active' });
+    headlessBrokerHost.teardown(id);
+    await h.waitFor(() => !isPidAlive(leaderPid), { timeoutMs: 6_000, label: 'fake broker leader reaped' });
+    await h.waitFor(() => !isPidAlive(descendantPid), {
+        timeoutMs: 6_000,
+        label: 'DETACHED descendant reaped by the ppid-tree-walk kill',
+    });
+});
+test('a detached descendant that ignores SIGTERM only dies once the SIGKILL escalation fires', async () => {
+    // The leaf traps SIGTERM (ignores it) but cannot trap SIGKILL — proves the
+    // ladder genuinely reaches the final SIGKILL rung against a descendant's
+    // OWN group, not just the broker's.
+    const { leaderPid, descendantPid } = await spawnBrokerWithDetachedDescendant([
+        'bash',
+        '-c',
+        'trap "" TERM; sleep 300',
+    ]);
+    const id = h.fabricateBrokerNode({ parent: root, pi_pid: leaderPid, status: 'active' });
+    const start = Date.now();
+    headlessBrokerHost.teardown(id);
+    // Give the initial tree-wide SIGTERM a moment to land — the stubborn leaf
+    // must still be alive (it ignores TERM), proving SIGTERM alone doesn't reap it.
+    await new Promise((r) => setTimeout(r, 300));
+    assert.equal(isPidAlive(descendantPid), true, 'SIGTERM-ignoring descendant survives the first rung');
+    await h.waitFor(() => !isPidAlive(descendantPid), {
+        timeoutMs: 8_000,
+        label: 'stubborn detached descendant reaped once the SIGKILL escalation fires',
+    });
+    assert.ok(Date.now() - start >= 1_500, 'the kill only lands after the SIGKILL grace window, not instantly');
+});
+test('CONNECTED broker accepts the shutdown frame and exits cleanly, but leaves a DETACHED descendant alive — ' +
+    'teardown must still reap it (review finding 1: post-shutdown liveness/escalation must reuse the ONE fixed ' +
+    "tree snapshot taken before the broker could exit, never re-walk `ps` by ppid afterward, since the exiting " +
+    "broker severs the descendant's ppid link the SAME tick)", async () => {
+    // Fabricate the node FIRST (placeholder pid) so we know its view.sock path
+    // before spawning the fake broker that must listen on that exact path.
+    const id = h.fabricateBrokerNode({ parent: root, pi_pid: null, status: 'active' });
+    const sockPath = h.brokerSock(id);
+    const { leaderPid, descendantPid } = await spawnBrokerWithDetachedDescendantAndSocket(sockPath, ['sleep', '300']);
+    recordPid(id, leaderPid);
+    assert.equal(isPidAlive(leaderPid), true, 'fake connected broker leader alive before teardown');
+    assert.equal(isPidAlive(descendantPid), true, 'detached descendant alive before teardown');
+    // teardown() connects, writes the shutdown frame, and the fake broker
+    // exits the INSTANT it receives that first byte — WITHOUT reaping its own
+    // detached descendant (modeling a dispose() that never reaches the pi bash
+    // tool's shell child). This is the exact connected/shutdown path finding 1
+    // covers: a re-walk-by-ppid AFTER that exit would find the descendant
+    // reparented away from `leaderPid` and conclude the tree is dead, silently
+    // skipping escalation — this descendant would then survive FOREVER.
+    headlessBrokerHost.teardown(id);
+    await h.waitFor(() => !isPidAlive(leaderPid), {
+        timeoutMs: 6_000,
+        label: 'fake connected broker leader exited (accepted the shutdown frame)',
+    });
+    await h.waitFor(() => !isPidAlive(descendantPid), {
+        timeoutMs: 8_000,
+        label: 'descendant of a CLEANLY-EXITED connected broker reaped by the fixed-snapshot escalation',
+    });
+});
+test('a FAILED initial identity probe must not be mistaken for "pid already gone" — killProcessTreePids still ' +
+    'signals every real live pid in the snapshot (final review, crouter#98 signal-safety finding: ' +
+    '`capturePidIdentities` collapsed "ps succeeded, no row" and "ps probe itself failed" into the same empty ' +
+    'map, so a transient failure of the INITIAL capture made every real live pid look "reused" against a ' +
+    'later-successful current probe, and `killProcessTreePids` skipped signaling all of them — silently ' +
+    "disabling the entire #98 reap. This must fail against that collapse-bug behavior and pass once a failed " +
+    'probe is reported distinctly (`null`) instead of an empty map.)', async () => {
+    const { leaderPid, descendantPid } = await spawnBrokerWithSharedGroupDescendant('probe-failure');
+    const tree = [leaderPid, descendantPid];
+    // Force `capturePidIdentities`'s `ps` spawn to fail deterministically —
+    // no binary named `ps` is resolvable on this PATH — standing in for a
+    // transient probe failure (timeout, spawn error) at the exact moment
+    // `teardown()` takes its INITIAL identity snapshot. Mutating
+    // `process.env.PATH` is safe here only because `capturePidIdentities` is
+    // fully synchronous (`spawnSync`) and this file's tests run sequentially,
+    // so the broken PATH is never visible to any other in-flight test.
+    const origPath = process.env.PATH;
+    let identities;
+    try {
+        process.env.PATH = '/nonexistent-crtr-test-path';
+        identities = capturePidIdentities(tree);
+    }
+    finally {
+        process.env.PATH = origPath;
+    }
+    assert.equal(identities, null, 'a failed ps probe reports null, never an empty-but-valid map');
+    assert.equal(isPidAlive(leaderPid), true, 'fake broker leader alive before the guarded signal');
+    assert.equal(isPidAlive(descendantPid), true, 'shared-group descendant alive before the guarded signal');
+    // The current probe inside killProcessTreePids now runs with PATH already
+    // restored, so it succeeds — this is the exact shape of the bug: a failed
+    // INITIAL capture followed by a successful current one. A pre-fix
+    // `identities` here would be an empty (not null) map, which the collapse
+    // bug reads as "both pids were already dead at snapshot time," and then
+    // reads the current probe's real (alive) identities as proof of reuse —
+    // skipping every signal below and leaving both pids alive forever.
+    killProcessTreePids(tree, 'SIGTERM', identities);
+    await h.waitFor(() => !isPidAlive(leaderPid), {
+        timeoutMs: 6_000,
+        label: 'broker leader signaled despite the failed initial identity probe',
+    });
+    await h.waitFor(() => !isPidAlive(descendantPid), {
+        timeoutMs: 6_000,
+        label: 'shared-group descendant signaled despite the failed initial identity probe',
+    });
+});
+test('captureTeardownSnapshot refuses a pid whose CURRENT identity differs from its recorded LAUNCH-time baseline ' +
+    '(final review, Fix 1: the guard must catch a pid ALREADY reused before teardown ever starts, not just reuse ' +
+    'across the escalation window) — tree comes back EMPTY, so nothing downstream ever signals the stranger', async () => {
+    const { leaderPid, descendantPid } = await spawnBrokerWithSharedGroupDescendant('snapshot-mismatch');
+    // A deliberately WRONG "launch-time" identity — no real `ps lstart`/
+    // `/proc` starttime pair for a genuinely alive process can ever equal
+    // this string, so this stands in for "the OS recycled this pid for an
+    // unrelated process since launch" without needing to actually wait out a
+    // real pid-recycle.
+    const snapshot = captureTeardownSnapshot(leaderPid, 'deliberately-mismatched-launch-identity');
+    assert.equal(snapshot.reused, true, 'a real, alive, differently-identified pid reports as reused');
+    assert.deepEqual(snapshot.tree, [], 'a reused root yields an EMPTY tree — no descendant walk from a stranger');
+    assert.equal(snapshot.identities, null, 'no identity map for a poisoned/reused snapshot');
+    // Sanity: the SAME pid with its OWN real current identity as the baseline
+    // is never flagged reused, and its descendant tree resolves normally.
+    const realIdentity = capturePidIdentities([leaderPid])?.get(leaderPid) ?? null;
+    assert.notEqual(realIdentity, null, 'the real launch identity capture succeeds for a genuinely alive pid');
+    const matched = captureTeardownSnapshot(leaderPid, realIdentity);
+    assert.equal(matched.reused, false, 'a matching baseline is never flagged reused');
+    assert.ok(matched.tree.includes(leaderPid) && matched.tree.includes(descendantPid), 'the real tree is walked normally when the baseline matches');
+    // Never signaled by this test — the `after()` sweep is the safety net.
+    assert.equal(isPidAlive(leaderPid), true, 'unsignaled leader is still alive');
+    assert.equal(isPidAlive(descendantPid), true, 'unsignaled descendant is still alive');
+});
+test('end-to-end: headlessBrokerHost.teardown() honors a launch-time identity mismatch and never signals the ' +
+    'stand-in "reused pid" tree at all, even after both the SIGTERM and SIGKILL escalation grace windows elapse', async () => {
+    const { leaderPid, descendantPid } = await spawnBrokerWithSharedGroupDescendant('teardown-mismatch');
+    const id = h.fabricateBrokerNode({ parent: root, pi_pid: leaderPid, status: 'active' });
+    // recordPid captures the REAL current identity for leaderPid (genuinely
+    // alive right now, so this succeeds) — then this corrupts JUST the stored
+    // identity column to simulate "the recorded launch-time baseline no
+    // longer matches this pid's CURRENT owner", standing in for an actual OS
+    // pid-recycle between this node's launch and this teardown() call.
+    recordPid(id, leaderPid);
+    openDb()
+        .prepare('UPDATE nodes SET pi_pid_identity = ? WHERE node_id = ?')
+        .run('deliberately-mismatched-launch-identity', id);
+    headlessBrokerHost.teardown(id);
+    // No real broker listens on this fabricated node's view.sock, so
+    // teardown's connect-error escalation path is the one in play. Wait past
+    // BOTH the SIGTERM and SIGKILL grace windows (2s + 2s, host.ts) it would
+    // use if it were (wrongly) signaling this tree — if the guard failed, this
+    // fixture would be dead well before this wait elapses.
+    await new Promise((r) => setTimeout(r, 5_000));
+    assert.equal(isPidAlive(leaderPid), true, 'mismatched-identity leader was NEVER signaled — still alive');
+    assert.equal(isPidAlive(descendantPid), true, 'its descendant was NEVER signaled either — still alive');
+});

package/dist/core/__tests__/human-cancel-guard.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/core/__tests__/human-cancel-guard.test.js

@@ -0,0 +1,71 @@
+// Run with: node --import tsx/esm --test src/core/__tests__/human-cancel-guard.test.ts
+//
+// N2 — humanCancel on an already-canceled (but unresolved) interaction node must
+// short-circuit to {canceled:false, reason:'already_resolved'} instead of falling
+// through to transition('finalize'), which is illegal from status='canceled' and
+// would throw. Guards the one-line hardening in queue.ts's status guard.
+import { test, before, after, beforeEach, afterEach } from 'node:test';
+import assert from 'node:assert/strict';
+import { mkdtempSync, mkdirSync, rmSync, existsSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { createNode, setStatus } from '../canvas/canvas.js';
+import { closeDb } from '../canvas/db.js';
+import { humanCancel } from '../../commands/human/queue.js';
+import { createApiServer } from '../../daemon/api/server.js';
+import { apiSocketPath } from '../canvas/paths.js';
+// `humanCancel` is a thin CLI forwarder since the crtrd-API re-plumb: its
+// already-canceled short-circuit guard runs client-side after fetching the node
+// through `cliClient().getNode`. Stand up a real in-process `createApiServer()`
+// bound to the isolated CRTR_HOME's socket so that fetch resolves against the
+// genuine daemon seam (never an autostarted subprocess, never a mock client).
+let server;
+async function waitForSocket(path) {
+    for (let i = 0; i < 200; i++) {
+        if (existsSync(path))
+            return;
+        await new Promise((r) => setTimeout(r, 25));
+    }
+    throw new Error(`api socket never bound at ${path}`);
+}
+let home;
+function node(id) {
+    return {
+        node_id: id,
+        name: id,
+        created: new Date().toISOString(),
+        cwd: '/tmp/work',
+        kind: 'general',
+        mode: 'base',
+        lifecycle: 'terminal',
+        status: 'active',
+    };
+}
+before(() => {
+    home = mkdtempSync(join(tmpdir(), 'crtr-humancancel-'));
+    process.env['CRTR_HOME'] = home;
+});
+beforeEach(async () => {
+    closeDb();
+    rmSync(home, { recursive: true, force: true });
+    mkdirSync(home, { recursive: true });
+    server = createApiServer();
+    await waitForSocket(apiSocketPath());
+});
+afterEach(async () => {
+    await server?.close();
+});
+after(() => {
+    closeDb();
+    rmSync(home, { recursive: true, force: true });
+    delete process.env['CRTR_HOME'];
+});
+test('cancel on an already-canceled node is a no-op, never throws on the finalize', async () => {
+    const id = 'canceledJob';
+    createNode(node(id));
+    setStatus(id, 'canceled'); // canceled but no response.json written yet
+    const res = (await humanCancel.run({ job_id: id }));
+    assert.equal(res['canceled'], false);
+    assert.equal(res['reason'], 'already_resolved');
+    assert.equal(res['job_id'], id);
+});

package/dist/core/__tests__/human-deliver-e2e.test.d.ts

@@ -0,0 +1 @@
+export {};