@north-light/crouter 0.3.308 → 0.3.310

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. package/dist/clients/attach/viewer.js +394 -394
  2. package/dist/commands/__tests__/node-message.test.js +13 -0
  3. package/dist/commands/canvas-history/grep.js +1 -1
  4. package/dist/commands/canvas-history/read.js +1 -1
  5. package/dist/commands/canvas-history/search.js +1 -1
  6. package/dist/commands/memory/__tests__/command-selector-and-mutation-guards.test.js +20 -0
  7. package/dist/commands/memory/edit.js +8 -11
  8. package/dist/commands/node-inspect-artifacts.js +2 -2
  9. package/dist/commands/push.js +1 -1
  10. package/dist/core/__tests__/canvas-inbox-watcher.test.js +3 -3
  11. package/dist/core/__tests__/human-deliver.test.js +1 -1
  12. package/dist/core/__tests__/integration/broker-wedge-supervision.test.d.ts +1 -0
  13. package/dist/core/__tests__/integration/broker-wedge-supervision.test.js +102 -0
  14. package/dist/core/__tests__/integration/refresh-stall-recycle.test.js +68 -2
  15. package/dist/core/__tests__/integration/worktree-reap.test.js +31 -0
  16. package/dist/core/__tests__/seam/inbox-reference-guidance.test.d.ts +1 -0
  17. package/dist/core/__tests__/seam/inbox-reference-guidance.test.js +70 -0
  18. package/dist/core/canvas/__tests__/session-window.test.d.ts +1 -0
  19. package/dist/core/canvas/__tests__/session-window.test.js +70 -0
  20. package/dist/core/canvas/browse/app.js +52 -27
  21. package/dist/core/canvas/browse/render.js +11 -2
  22. package/dist/core/canvas/render-source.js +63 -25
  23. package/dist/core/command.js +15 -0
  24. package/dist/core/feed/inbox.js +6 -4
  25. package/dist/core/runtime/bearings-render.js +3 -3
  26. package/dist/core/runtime/broker/inbox.js +6 -4
  27. package/dist/core/runtime/fleet.js +1 -1
  28. package/dist/core/runtime/placement.d.ts +1 -1
  29. package/dist/core/runtime/placement.js +1 -1
  30. package/dist/core/runtime/tmux-driver.d.ts +7 -0
  31. package/dist/core/runtime/tmux-driver.js +20 -0
  32. package/dist/core/substrate/on-read.d.ts +6 -4
  33. package/dist/core/substrate/on-read.js +14 -6
  34. package/dist/core/substrate/surface-match.d.ts +12 -4
  35. package/dist/core/substrate/surface-match.js +19 -10
  36. package/dist/core/worktree-mutation-async.js +1 -2
  37. package/dist/core/worktree.d.ts +0 -1
  38. package/dist/core/worktree.js +0 -113
  39. package/dist/daemon/api/handlers/canvas.js +3 -3
  40. package/dist/daemon/reconcilers/broker-supervision.d.ts +14 -2
  41. package/dist/daemon/reconcilers/broker-supervision.js +109 -57
  42. package/dist/daemon/reconcilers/storage-maintenance.d.ts +5 -0
  43. package/dist/daemon/reconcilers/storage-maintenance.js +31 -8
  44. package/dist/pi-extensions/__tests__/pre-command-gate.test.js +10 -10
  45. package/dist/pi-extensions/canvas-doc-substrate.js +11 -5
  46. package/dist/shared/generated-context.d.ts +2 -0
  47. package/dist/shared/generated-context.js +6 -0
  48. package/package.json +1 -1
  49. package/runtime.lock.json +5 -5
@@ -3,7 +3,6 @@ import { createHash } from 'node:crypto';
3
3
  import { dirname, isAbsolute, join, relative, resolve, sep } from 'node:path';
4
4
  import { gitAsync, gitSync } from './git.js';
5
5
  import { crtrHome } from './canvas/paths.js';
6
- import { isPidAlive } from './canvas/pid.js';
7
6
  import { getNode, updateNode } from './canvas/index.js';
8
7
  import { CONFIG_FILE, CRTR_DIR_NAME, STATE_FILE } from '../types.js';
9
8
  import { withExclusiveDirectoryLock } from './exclusive-lock.js';
@@ -658,118 +657,6 @@ export function closeManagedWorktree(nodeId) {
658
657
  // the child checkout instead of the parent that created it.
659
658
  return withRepositoryWorktreeLock(existsSync(wt.path) ? wt.path : wt.repo_root, () => closeManagedWorktreeLocked(nodeId));
660
659
  }
661
- const ABANDON_AUTHOR_NAME = 'crtr-abandon';
662
- const ABANDON_AUTHOR_EMAIL = 'crtr-abandon@localhost';
663
- const ABANDON_SUBJECT = 'crtr-abandon';
664
- function persistAbandonedWorktreeState(nodeId, wt, tipSha, at, by, branch = wt.branch) {
665
- updateNode(nodeId, {
666
- cwd: wt.repo_root,
667
- managed_worktree: {
668
- ...wt,
669
- state: 'abandoned',
670
- abandoned: { branch, tip_sha: tipSha, at, by },
671
- },
672
- });
673
- }
674
- /** The disposal path for `abandon` when the recorded `crtr/*` branch is gone:
675
- * ordinary PR work moved this checkout onto a real branch. Abandon never
676
- * lands that branch and — unlike the ordinary path above — never invents a
677
- * commit on it: a synthetic `crtr-abandon` marker on someone's real PR
678
- * branch is not a trade this code may make on their behalf. So this
679
- * requires the checkout already clean and not mid-operation, then removes
680
- * only the checkout and preserves the actual branch and its commits exactly
681
- * as they stand. The caller already confirmed the node is not running. */
682
- function abandonBranchMovedWorktree(nodeId, wt, actualBranch, by) {
683
- if (isRebaseInProgress(wt.path)) {
684
- throw new WorktreeError('rebase_in_progress', 'this managed worktree is mid-rebase', `Resolve conflicts in ${wt.path}, run \`git rebase --continue\` there, then rerun \`crtr node worktree abandon\`.`);
685
- }
686
- const dirty = statusPorcelain(wt.path);
687
- if (dirty !== '') {
688
- throw new WorktreeError('dirty_worktree', 'managed worktree has uncommitted changes', `The recorded branch ${wt.branch} no longer exists; its checkout is now on ${actualBranch} with uncommitted changes crtr will not commit on your behalf. Commit or stash them in ${wt.path}, then rerun \`crtr node worktree abandon\`.`);
689
- }
690
- if (registeredWorktree(wt.repo_root, wt.path, actualBranch) !== true) {
691
- throw new WorktreeError('wrong_branch', `managed worktree's recorded branch ${wt.branch} no longer exists, and its checkout at ${wt.path} is not registered to ${actualBranch} either`, `Inspect ${wt.path} and its git worktree registration, then ask the user with \`crtr human send\` if you are unsure how to proceed.`);
692
- }
693
- const actualSha = runGit(wt.path, ['rev-parse', 'HEAD'], 'abandon_tip_failed', 'Inspect the worktree manually, then retry.');
694
- // Recheck identity immediately before the destructive step.
695
- if (registeredWorktree(wt.repo_root, wt.path, actualBranch) !== true || currentBranch(wt.path) !== actualBranch) {
696
- throw new WorktreeError('wrong_branch', `managed worktree at ${wt.path} changed while abandoning`, `Inspect ${wt.path}, then rerun \`crtr node worktree abandon\`.`);
697
- }
698
- runGit(wt.repo_root, ['worktree', 'remove', wt.path], 'worktree_remove_failed', 'Inspect the worktree manually, then retry.');
699
- const at = new Date().toISOString();
700
- persistAbandonedWorktreeState(nodeId, wt, actualSha, at, by, actualBranch);
701
- return { node_id: nodeId, branch: actualBranch, tip_sha: actualSha, worktree_path: wt.path, branch_deleted: false };
702
- }
703
- function createAbandonMarker(wt, parentSha, nodeId, at) {
704
- const treeSha = runGit(wt.repo_root, ['rev-parse', `${parentSha}^{tree}`], 'abandon_tree_failed', 'Inspect the branch manually, then retry.');
705
- const markerSha = runGit(wt.repo_root, ['-c', `user.name=${ABANDON_AUTHOR_NAME}`, '-c', `user.email=${ABANDON_AUTHOR_EMAIL}`, 'commit-tree', treeSha, '-p', parentSha, '-m', ABANDON_SUBJECT, '-m', `Crtr-Abandon: ${nodeId} ${at}`], 'abandon_commit_failed', 'Inspect the branch manually, then retry.');
706
- runGit(wt.repo_root, ['update-ref', `refs/heads/${wt.branch}`, markerSha, parentSha], 'abandon_marker_failed', 'The branch changed while abandon was starting; inspect the worktree, then retry.');
707
- return markerSha;
708
- }
709
- function abandonManagedWorktreeLocked(nodeId, by) {
710
- const node = getNode(nodeId);
711
- if (node === null) {
712
- throw new WorktreeError('unknown_node', `unknown node: ${nodeId}`, 'Pass a real node id.');
713
- }
714
- const wt = openManagedWorktreeForNode(nodeId);
715
- if (wt === null) {
716
- const { message, next } = noManagedWorktreeMessage(nodeId, 'abandon');
717
- throw new WorktreeError('no_managed_worktree', message, next);
718
- }
719
- if (node.status === 'active' || node.status === 'idle' || isPidAlive(node.pi_pid)) {
720
- throw new WorktreeError('node_running', `node ${nodeId} is still running`, 'Stop the node before abandoning its managed worktree.');
721
- }
722
- if (!existsSync(wt.path)) {
723
- throw new WorktreeError('worktree_missing', `managed worktree path is missing: ${wt.path}`, missingWorktreeRecoveryHint(wt));
724
- }
725
- const actualBranch = currentBranch(wt.path);
726
- if (actualBranch !== wt.branch) {
727
- const recordedRefExists = recordedBranchExists(wt.path, wt.branch);
728
- if (recordedRefExists || actualBranch === 'HEAD') {
729
- throw new WorktreeError('wrong_branch', `managed worktree is not on its recorded branch (expected ${wt.branch}${actualBranch === 'HEAD' ? ', found detached HEAD' : ''})`, recordedRefExists
730
- ? `Check out ${wt.branch} in ${wt.path}, then rerun \`crtr node worktree abandon\`.`
731
- : `The recorded branch ${wt.branch} no longer exists, and ${wt.path} is in detached HEAD, so there is no branch to preserve. Check out or create a branch there, then rerun \`crtr node worktree abandon\`.`);
732
- }
733
- // The recorded crtr/* branch is gone: ordinary PR work moved this checkout
734
- // onto a real branch.
735
- return abandonBranchMovedWorktree(nodeId, wt, actualBranch, by);
736
- }
737
- const at = new Date().toISOString();
738
- const clean = proveCleanEmptyManagedWorktree(wt);
739
- if (clean !== null && clean.branchSha === wt.base_sha) {
740
- const markerSha = createAbandonMarker(wt, clean.branchSha, nodeId, at);
741
- runGit(wt.repo_root, ['worktree', 'remove', wt.path], 'worktree_remove_failed', 'Inspect the worktree manually, then retry.');
742
- const deleted = bestEffortDeleteBranchWithContainmentProof(wt, markerSha, `refs/heads/${wt.base_ref}`, clean.baseSha);
743
- if (deleted.branchDeleted) {
744
- persistClosedWorktreeState(nodeId, wt, 'complete');
745
- return { node_id: nodeId, branch: wt.branch, tip_sha: markerSha, worktree_path: wt.path, branch_deleted: true };
746
- }
747
- const tipSha = runGit(wt.repo_root, ['rev-parse', `refs/heads/${wt.branch}`], 'abandon_tip_failed', 'Inspect the branch manually, then retry.');
748
- persistAbandonedWorktreeState(nodeId, wt, tipSha, at, by);
749
- return { node_id: nodeId, branch: wt.branch, tip_sha: tipSha, worktree_path: wt.path, branch_deleted: false };
750
- }
751
- // A clean branch with committed work already has its complete history. The
752
- // abandon record supplies the disposition; an empty provenance commit would
753
- // only move the local branch away from an already-pushed tip.
754
- if (!isRebaseInProgress(wt.path) && statusPorcelain(wt.path) === '') {
755
- runGit(wt.repo_root, ['worktree', 'remove', wt.path], 'worktree_remove_failed', 'Inspect the worktree manually, then retry.');
756
- const tipSha = runGit(wt.repo_root, ['rev-parse', `refs/heads/${wt.branch}`], 'abandon_tip_failed', 'Inspect the branch manually, then retry.');
757
- persistAbandonedWorktreeState(nodeId, wt, tipSha, at, by);
758
- return { node_id: nodeId, branch: wt.branch, tip_sha: tipSha, worktree_path: wt.path, branch_deleted: false };
759
- }
760
- runGit(wt.path, ['add', '--all'], 'abandon_stage_failed', 'Inspect the worktree manually, then retry.');
761
- runGit(wt.path, ['-c', `user.name=${ABANDON_AUTHOR_NAME}`, '-c', `user.email=${ABANDON_AUTHOR_EMAIL}`, 'commit', '--allow-empty', '-m', ABANDON_SUBJECT, '-m', `Crtr-Abandon: ${nodeId} ${at}`], 'abandon_commit_failed', 'Inspect the worktree manually, then retry.');
762
- runGit(wt.repo_root, ['worktree', 'remove', wt.path], 'worktree_remove_failed', 'Inspect the worktree manually, then retry.');
763
- const tipSha = runGit(wt.repo_root, ['rev-parse', `refs/heads/${wt.branch}`], 'abandon_tip_failed', 'Inspect the branch manually, then retry.');
764
- persistAbandonedWorktreeState(nodeId, wt, tipSha, at, by);
765
- return { node_id: nodeId, branch: wt.branch, tip_sha: tipSha, worktree_path: wt.path, branch_deleted: false };
766
- }
767
- export function abandonManagedWorktree(nodeId, by) {
768
- const wt = openManagedWorktreeForNode(nodeId);
769
- if (wt === null)
770
- return abandonManagedWorktreeLocked(nodeId, by);
771
- return withRepositoryWorktreeLock(wt.repo_root, () => abandonManagedWorktreeLocked(nodeId, by));
772
- }
773
660
  // Push-final auto-drop of a clean, zero-commit managed worktree
774
661
  /** Delete a source branch only while the ref that proved it already delivered
775
662
  * is still at the exact inspected SHA. `containingRef` can be the recorded
@@ -261,7 +261,7 @@ function handleHistoryRead(ctx) {
261
261
  if (!ref.includes(':')) {
262
262
  throw usage(`ref must be <node-id>:<relpath>; received: ${ref}`, {
263
263
  received: ref,
264
- next: 'Get a valid ref from `canvas history search` or `node inspect artifacts <node-id>`.',
264
+ next: 'Get a valid ref from `crtr canvas history search` or `crtr node inspect artifacts <node-id>`.',
265
265
  });
266
266
  }
267
267
  let resolved;
@@ -271,13 +271,13 @@ function handleHistoryRead(ctx) {
271
271
  catch (e) {
272
272
  throw usage(`ref rejected: ${e.message}`, {
273
273
  received: ref,
274
- next: 'Refs come verbatim from `canvas history search`; do not hand-construct relpaths.',
274
+ next: 'Refs come verbatim from `crtr canvas history search`; do not hand-construct relpaths.',
275
275
  });
276
276
  }
277
277
  if (resolved === null) {
278
278
  throw notFound(`no artifact for ref: ${ref}`, {
279
279
  ref,
280
- next: 'The node or file may not exist. Re-run `canvas history search` for a current ref, or `node inspect artifacts <node-id>` to list a node\'s artifacts.',
280
+ next: 'The node or file may not exist. Re-run `crtr canvas history search` for a current ref, or `crtr node inspect artifacts <node-id>` to list a node\'s artifacts.',
281
281
  });
282
282
  }
283
283
  const source = resolved.source === 'report' && resolved.reportKind ? `report:${resolved.reportKind}` : resolved.source;
@@ -11,6 +11,14 @@ export interface BrokerSupervisionContext {
11
11
  fleet: FleetRegistry;
12
12
  lifecycle: DetachedWorkLifecycle;
13
13
  }
14
+ type TreeSample = {
15
+ cpuPercent: number;
16
+ hasDescendants: boolean;
17
+ } | null;
18
+ export interface BrokerSupervisionOptions {
19
+ readTreeSample?: (pid: number) => Promise<TreeSample>;
20
+ signal?: (pid: number, signal: NodeJS.Signals) => void;
21
+ }
14
22
  export interface SaturationContext {
15
23
  fleet: FleetRegistry;
16
24
  lifecycle: DetachedWorkLifecycle;
@@ -41,13 +49,15 @@ export declare function wedgeVerdict(input: {
41
49
  export declare function sumTreeCpu(psOutput: string, rootPid: number): number | null;
42
50
  export declare function treeHasDescendants(psOutput: string, rootPid: number): boolean | null;
43
51
  export declare class BrokerSupervisionReconciler {
44
- private readonly yieldStallSince;
45
- private readonly yieldTermAt;
52
+ private readonly yieldStalls;
46
53
  private readonly wedgeNotifiedAt;
47
54
  private readonly fatalFaultNotifiedAt;
48
55
  private readonly unattendedSince;
56
+ private readonly sampleTree;
57
+ private readonly signal;
49
58
  private knownAuthMtimeMs;
50
59
  private lastOverCapWarnAt;
60
+ constructor(options?: BrokerSupervisionOptions);
51
61
  /** The fleet-wide preamble, run BEFORE the tick's row-list guard: an auth
52
62
  * reload does not depend on a successful canvas scan, and the thresholds it
53
63
  * returns are read once for the whole tick. */
@@ -70,8 +80,10 @@ export declare class BrokerSupervisionReconciler {
70
80
  private requestParkSummary;
71
81
  private warnOverCapInTmux;
72
82
  private handleYieldStall;
83
+ private signalYieldStall;
73
84
  private readTreeSample;
74
85
  private handleWedgeDetection;
75
86
  private handleFatalFault;
76
87
  private handleAuthReload;
77
88
  }
89
+ export {};
@@ -30,6 +30,16 @@ import { shouldParkAfterSummary } from '../park-activity.js';
30
30
  import { applyParkEvent, parkEvent } from './live-obligation.js';
31
31
  import { envNoDaemonAutostart, envTestUnattendedParkMs, parkSummaryGraceMs, envUnattendedParkMs, } from '../../shared/env.js';
32
32
  const execFileAsync = promisify(execFile);
33
+ function incarnationOf(pid, identity) {
34
+ return identity === null ? null : `${pid}:${identity}`;
35
+ }
36
+ function busySinceForIncarnation(row) {
37
+ const since = busySince(row.node_id);
38
+ const launchedAt = row.launched_at === null || row.launched_at === undefined ? Number.NaN : Date.parse(row.launched_at);
39
+ if (since === null || !Number.isFinite(launchedAt) || since < launchedAt)
40
+ return null;
41
+ return since;
42
+ }
33
43
  export const YIELD_STALL_GRACE_MS = 3 * 60_000;
34
44
  const KILL_ESCALATE_MS = 20_000;
35
45
  export const WEDGE_QUIET_MS = 20 * 60_000;
@@ -131,13 +141,18 @@ function piCredentialPaths() {
131
141
  ];
132
142
  }
133
143
  export class BrokerSupervisionReconciler {
134
- yieldStallSince = new Map();
135
- yieldTermAt = new Map();
144
+ yieldStalls = new Map();
136
145
  wedgeNotifiedAt = new Map();
137
146
  fatalFaultNotifiedAt = new Map();
138
147
  unattendedSince = new Map();
148
+ sampleTree;
149
+ signal;
139
150
  knownAuthMtimeMs = null;
140
151
  lastOverCapWarnAt = Number.NEGATIVE_INFINITY;
152
+ constructor(options = {}) {
153
+ this.sampleTree = options.readTreeSample ?? ((pid) => this.readTreeSample(pid));
154
+ this.signal = options.signal ?? ((pid, signal) => process.kill(pid, signal));
155
+ }
141
156
  /** The fleet-wide preamble, run BEFORE the tick's row-list guard: an auth
142
157
  * reload does not depend on a successful canvas scan, and the thresholds it
143
158
  * returns are read once for the whole tick. */
@@ -168,8 +183,8 @@ export class BrokerSupervisionReconciler {
168
183
  }
169
184
  await operationIdContext.fresh(async () => {
170
185
  try {
171
- this.handleYieldStall(row, entry.pid, now);
172
- await this.handleWedgeDetection(id, entry.pid, row.pi_pid_identity, now);
186
+ this.handleYieldStall(row, entry, ctx.fleet, now);
187
+ await this.handleWedgeDetection(row, entry, ctx.fleet, now);
173
188
  this.handleFatalFault(id);
174
189
  this.handleUnattendedParking(id, getNode(id), entry.pid, now, ctx, unattendedParkMs);
175
190
  }
@@ -391,62 +406,74 @@ export class BrokerSupervisionReconciler {
391
406
  emitEvent({ level: 'error', event: 'broker.capacity.tmux_list_clients_failed', error: err });
392
407
  }
393
408
  }
394
- handleYieldStall(row, pid, now) {
409
+ handleYieldStall(row, entry, fleet, now) {
395
410
  const id = row.node_id;
396
- if (row.intent !== 'refresh' || isBusy(id)) {
397
- this.yieldStallSince.delete(id);
398
- this.yieldTermAt.delete(id);
411
+ const incarnation = incarnationOf(entry.pid, row.pi_pid_identity);
412
+ if (row.intent !== 'refresh' || busySinceForIncarnation(row) !== null || incarnation === null) {
413
+ if (incarnation !== null)
414
+ this.yieldStalls.delete(incarnation);
399
415
  return;
400
416
  }
401
- const since = this.yieldStallSince.get(id);
402
- if (since === undefined) {
403
- this.yieldStallSince.set(id, now);
417
+ let state = this.yieldStalls.get(incarnation);
418
+ if (state === undefined) {
419
+ state = { since: now };
420
+ this.yieldStalls.set(incarnation, state);
404
421
  return;
405
422
  }
406
- if (yieldStallVerdict(true, row.intent, false, now - since) !== 'kill')
423
+ if (yieldStallVerdict(true, row.intent, false, now - state.since) !== 'kill')
407
424
  return;
408
- if (row.pi_pid_identity == null || recordedPidLiveness(pid, row.pi_pid_identity) !== 'alive')
425
+ const current = getNode(id);
426
+ if (fleet.get(id) !== entry || current?.pi_pid !== entry.pid || current.pi_pid_identity !== row.pi_pid_identity || recordedPidLiveness(entry.pid, row.pi_pid_identity) !== 'alive')
409
427
  return;
410
- const termed = this.yieldTermAt.get(id);
411
- if (termed === undefined) {
428
+ if (state.termedAt === undefined) {
429
+ if (!this.signalYieldStall(id, entry.pid, 'SIGTERM'))
430
+ return;
431
+ state.termedAt = now;
412
432
  emitEvent({
413
433
  level: 'warn',
414
434
  event: 'broker.yield_stall.sigterm',
415
435
  ...(isSafeNodeId(id) ? { node_id: id } : {}),
416
436
  fields: {
417
437
  ...(isSafeNodeId(id) ? {} : { affected_node_id: id }),
418
- pid,
419
- elapsed_ms: now - since,
438
+ pid: entry.pid,
439
+ elapsed_ms: now - state.since,
420
440
  grace_ms: YIELD_STALL_GRACE_MS,
421
441
  },
422
442
  });
423
- try {
424
- process.kill(pid, 'SIGTERM');
425
- }
426
- catch {
427
- /* already gone */
428
- }
429
- this.yieldTermAt.set(id, now);
430
443
  }
431
- else if (now - termed >= KILL_ESCALATE_MS) {
444
+ else if (now - state.termedAt >= KILL_ESCALATE_MS && this.signalYieldStall(id, entry.pid, 'SIGKILL')) {
432
445
  emitEvent({
433
446
  level: 'warn',
434
447
  event: 'broker.yield_stall.sigkill',
435
448
  ...(isSafeNodeId(id) ? { node_id: id } : {}),
436
449
  fields: {
437
450
  ...(isSafeNodeId(id) ? {} : { affected_node_id: id }),
438
- pid,
439
- elapsed_ms: now - termed,
451
+ pid: entry.pid,
452
+ elapsed_ms: now - state.termedAt,
440
453
  grace_ms: KILL_ESCALATE_MS,
441
- stall_elapsed_ms: now - since,
454
+ stall_elapsed_ms: now - state.since,
442
455
  },
443
456
  });
444
- try {
445
- process.kill(pid, 'SIGKILL');
446
- }
447
- catch {
448
- /* already gone */
449
- }
457
+ }
458
+ }
459
+ signalYieldStall(id, pid, signal) {
460
+ try {
461
+ this.signal(pid, signal);
462
+ return true;
463
+ }
464
+ catch (err) {
465
+ emitEvent({
466
+ level: 'error',
467
+ event: 'broker.yield_stall.signal_failed',
468
+ ...(isSafeNodeId(id) ? { node_id: id } : {}),
469
+ error: err,
470
+ fields: {
471
+ ...(isSafeNodeId(id) ? {} : { affected_node_id: id }),
472
+ pid,
473
+ signal,
474
+ },
475
+ });
476
+ return false;
450
477
  }
451
478
  }
452
479
  async readTreeSample(pid) {
@@ -467,29 +494,36 @@ export class BrokerSupervisionReconciler {
467
494
  return null;
468
495
  }
469
496
  }
470
- async handleWedgeDetection(id, pid, expectedIdentity, now) {
471
- if (!isBusy(id)) {
472
- this.wedgeNotifiedAt.delete(id);
497
+ async handleWedgeDetection(row, entry, fleet, now) {
498
+ const id = row.node_id;
499
+ const incarnation = incarnationOf(entry.pid, row.pi_pid_identity);
500
+ const since = busySinceForIncarnation(row);
501
+ if (since === null || incarnation === null) {
502
+ if (incarnation !== null)
503
+ this.wedgeNotifiedAt.delete(incarnation);
473
504
  return;
474
505
  }
475
- const since = busySince(id);
476
- const quietForMs = since === null ? null : now - since;
477
- const sample = quietForMs !== null && quietForMs >= WEDGE_QUIET_MS ? await this.readTreeSample(pid) : null;
506
+ const quietForMs = now - since;
507
+ const sample = quietForMs >= WEDGE_QUIET_MS ? await this.sampleTree(entry.pid) : null;
508
+ const current = getNode(id);
509
+ const currentSince = current === null ? null : busySinceForIncarnation(current);
510
+ if (fleet.get(id) !== entry || current === null || current.pi_pid !== entry.pid || current.pi_pid_identity !== row.pi_pid_identity || currentSince !== since)
511
+ return;
478
512
  const cpuPercent = sample?.cpuPercent ?? null;
479
513
  const sampleHasDescendants = sample?.hasDescendants ?? null;
480
514
  if (wedgeVerdict({ busy: true, quietForMs, cpuPercent, hasDescendants: sampleHasDescendants }) !== 'wedged')
481
515
  return;
482
- if (since !== null && this.wedgeNotifiedAt.get(id) === since)
516
+ if (this.wedgeNotifiedAt.get(incarnation) === since)
483
517
  return;
484
- if (since !== null)
485
- this.wedgeNotifiedAt.set(id, since);
486
- const meta = getNode(id);
487
- const name = meta !== null ? fullName(meta) : id;
518
+ this.wedgeNotifiedAt.set(incarnation, since);
519
+ const name = fullName(current);
488
520
  const minutes = Math.round((quietForMs ?? WEDGE_QUIET_MS) / 60_000);
489
521
  const hasDescendants = sampleHasDescendants ?? true;
490
- const kickSuppressionReason = !hasDescendants && (expectedIdentity == null || recordedPidLiveness(pid, expectedIdentity) !== 'alive')
522
+ const kickSuppressionReason = !hasDescendants && recordedPidLiveness(entry.pid, row.pi_pid_identity) !== 'alive'
491
523
  ? 'pid_identity_unproven'
492
524
  : null;
525
+ let kickAttempted = false;
526
+ let kicked = false;
493
527
  let label;
494
528
  if (hasDescendants) {
495
529
  label =
@@ -506,16 +540,32 @@ export class BrokerSupervisionReconciler {
506
540
  `confirmed alive.`;
507
541
  }
508
542
  else {
509
- label =
510
- `Child wedged — ${name} (${id}) is alive and mid-turn but has shown NO engine progress for ` +
511
- `${minutes}+ minutes and it has no subprocess to kill (the engine itself stalled). Kicking it ` +
512
- `now: SIGTERM to the broker — the daemon will resume a cleanly ` +
513
- `aborted session with a continuation, or start a recovery cycle if the abort cannot settle.`;
543
+ kickAttempted = true;
514
544
  try {
515
- process.kill(pid, 'SIGTERM');
545
+ this.signal(entry.pid, 'SIGTERM');
546
+ kicked = true;
547
+ label =
548
+ `Child wedged — ${name} (${id}) is alive and mid-turn but has shown NO engine progress for ` +
549
+ `${minutes}+ minutes and it has no subprocess to kill (the engine itself stalled). The daemon sent ` +
550
+ `SIGTERM to the broker and will resume a cleanly aborted session with a continuation, or start a ` +
551
+ `recovery cycle if the abort cannot settle.`;
516
552
  }
517
- catch {
518
- /* already gone — its ChildProcess exit event still owns recovery */
553
+ catch (err) {
554
+ emitEvent({
555
+ level: 'error',
556
+ event: 'broker.wedge.signal_failed',
557
+ ...(isSafeNodeId(id) ? { node_id: id } : {}),
558
+ error: err,
559
+ fields: {
560
+ ...(isSafeNodeId(id) ? {} : { affected_node_id: id }),
561
+ pid: entry.pid,
562
+ signal: 'SIGTERM',
563
+ },
564
+ });
565
+ label =
566
+ `Child wedged — ${name} (${id}) is alive and mid-turn but has shown NO engine progress for ` +
567
+ `${minutes}+ minutes and it has no subprocess to kill (the engine itself stalled). The daemon ` +
568
+ `attempted SIGTERM to the broker, but the signal call failed.`;
519
569
  }
520
570
  }
521
571
  emitEvent({
@@ -524,11 +574,12 @@ export class BrokerSupervisionReconciler {
524
574
  ...(isSafeNodeId(id) ? { node_id: id } : {}),
525
575
  fields: {
526
576
  ...(isSafeNodeId(id) ? {} : { affected_node_id: id }),
527
- pid,
577
+ pid: entry.pid,
528
578
  quiet_ms: quietForMs,
529
579
  cpu_percent: cpuPercent,
530
580
  has_descendants: hasDescendants,
531
- kicked: !hasDescendants && kickSuppressionReason === null,
581
+ kick_attempted: kickAttempted,
582
+ kicked,
532
583
  ...(kickSuppressionReason === null ? {} : { kick_suppression_reason: kickSuppressionReason }),
533
584
  },
534
585
  });
@@ -554,7 +605,8 @@ export class BrokerSupervisionReconciler {
554
605
  child: id,
555
606
  quiet_ms: quietForMs,
556
607
  cpu_percent: cpuPercent,
557
- kicked: !hasDescendants && kickSuppressionReason === null,
608
+ kick_attempted: kickAttempted,
609
+ kicked,
558
610
  ...(kickSuppressionReason === null ? {} : { kick_suppression_reason: kickSuppressionReason }),
559
611
  });
560
612
  }
@@ -13,6 +13,7 @@ export interface StorageMaintenanceContext {
13
13
  export declare class StorageMaintenanceReconciler {
14
14
  private lastSpareSweepAt;
15
15
  private lastGhostSweepAt;
16
+ private focusGcInFlight;
16
17
  private readonly worktreeSweep;
17
18
  run(now: number, ctx: StorageMaintenanceContext): void;
18
19
  /** Drop rows whose on-disk node dir is gone. Such a row can never be revived,
@@ -25,6 +26,10 @@ export declare class StorageMaintenanceReconciler {
25
26
  /** Reap only physical placement residue of terminal nodes. The durable row,
26
27
  * graph, and on-disk state remain available for a later manual revive. */
27
28
  private reapDeadResidue;
29
+ /** Register one detached focus-GC pass, never overlapping another. The pane
30
+ * probe is a subprocess, and running it on the tick blocks the API socket for
31
+ * its full duration on every beat. */
32
+ private scheduleFocusGc;
28
33
  /** Delete focus rows whose recorded tmux pane no longer exists. A failed tmux
29
34
  * probe is "can't tell" and never authorizes deleting every viewport. */
30
35
  private gcStaleFocuses;
@@ -1,10 +1,10 @@
1
1
  import { statSync } from 'node:fs';
2
2
  import { join } from 'node:path';
3
- import { closeFocusRow, listFocuses, listNodes, reapGhostRows, } from '../../core/canvas/index.js';
3
+ import { closeFocusRow, getFocusById, listFocuses, listNodes, reapGhostRows, } from '../../core/canvas/index.js';
4
4
  import { isSafeNodeId, jobDir, nodeDir } from '../../core/canvas/paths.js';
5
5
  import { emitEvent } from '../../core/events/emit.js';
6
6
  import { operationIdContext } from '../../core/events/operation-id.js';
7
- import { listLivePanes, tearDownNode } from '../../core/runtime/placement.js';
7
+ import { listLivePanesAsync, tearDownNode } from '../../core/runtime/placement.js';
8
8
  import { reapStaleSpares } from '../../core/runtime/warm-pool.js';
9
9
  import { ManagedWorktreeSweepReconciler } from './managed-worktree-sweep.js';
10
10
  // How long a dead node's on-disk record must be quiet before its leftover
@@ -18,13 +18,16 @@ const GHOST_SWEEP_INTERVAL_MS = 60 * 1000;
18
18
  export class StorageMaintenanceReconciler {
19
19
  lastSpareSweepAt = Number.NEGATIVE_INFINITY;
20
20
  lastGhostSweepAt = Number.NEGATIVE_INFINITY;
21
+ focusGcInFlight = false;
21
22
  worktreeSweep = new ManagedWorktreeSweepReconciler();
22
23
  run(now, ctx) {
23
24
  // Placement reap MUST precede focus GC, so GC catches a focus row the reap
24
25
  // strands. Both run after the lifecycle tick, so a same-tick revive has
25
- // already had the chance to re-anchor its focus.
26
+ // already had the chance to re-anchor its focus. The GC is detached, but it
27
+ // reads the focus table synchronously before its first await, so it still
28
+ // observes the post-reap state this ordering exists to give it.
26
29
  this.reapDeadResidue(now);
27
- this.gcStaleFocuses();
30
+ this.scheduleFocusGc(ctx);
28
31
  this.reapGhostRows(now);
29
32
  this.gcWarmPool(now);
30
33
  // Registers detached async Git work and returns immediately: this lane is
@@ -140,9 +143,23 @@ export class StorageMaintenanceReconciler {
140
143
  });
141
144
  }
142
145
  }
146
+ /** Register one detached focus-GC pass, never overlapping another. The pane
147
+ * probe is a subprocess, and running it on the tick blocks the API socket for
148
+ * its full duration on every beat. */
149
+ scheduleFocusGc(ctx) {
150
+ if (this.focusGcInFlight)
151
+ return;
152
+ if (!ctx.lifecycle.acceptsDetachedWork())
153
+ return;
154
+ this.focusGcInFlight = true;
155
+ const pass = this.gcStaleFocuses().finally(() => {
156
+ this.focusGcInFlight = false;
157
+ });
158
+ ctx.lifecycle.registerDetached(pass);
159
+ }
143
160
  /** Delete focus rows whose recorded tmux pane no longer exists. A failed tmux
144
161
  * probe is "can't tell" and never authorizes deleting every viewport. */
145
- gcStaleFocuses() {
162
+ async gcStaleFocuses() {
146
163
  let rows;
147
164
  try {
148
165
  rows = listFocuses();
@@ -157,7 +174,7 @@ export class StorageMaintenanceReconciler {
157
174
  return;
158
175
  let live;
159
176
  try {
160
- live = listLivePanes();
177
+ live = await listLivePanesAsync();
161
178
  }
162
179
  catch (err) {
163
180
  operationIdContext.fresh(() => {
@@ -170,8 +187,14 @@ export class StorageMaintenanceReconciler {
170
187
  for (const focus of rows) {
171
188
  if (focus.pane === null || live.has(focus.pane))
172
189
  continue;
173
- const removed = operationIdContext.fresh(() => {
190
+ const proceed = operationIdContext.fresh(() => {
174
191
  try {
192
+ // The probe awaited above let other event-loop work run, so a viewer
193
+ // may have re-pointed this row at a pane the probe never saw. Re-read
194
+ // and match the pane before deleting; the read and the delete are both
195
+ // synchronous, so nothing can interleave between them.
196
+ if (getFocusById(focus.focus_id)?.pane !== focus.pane)
197
+ return true;
175
198
  emitEvent({
176
199
  level: 'info',
177
200
  event: 'focus.gc.removed',
@@ -190,7 +213,7 @@ export class StorageMaintenanceReconciler {
190
213
  return false;
191
214
  }
192
215
  });
193
- if (!removed)
216
+ if (!proceed)
194
217
  return;
195
218
  }
196
219
  }
@@ -12,9 +12,9 @@
12
12
  // executes them concurrently, so without a per-turn flag the second sibling
13
13
  // would find the doc already exposed and slip past the hold its sibling just
14
14
  // raised.
15
- // • A session whose corpus carries no `pre-command` doc at all is the common
16
- // case and must never pay a daemon round-trip. Proven by closing the daemon:
17
- // if the short-circuit stopped working the call would fail loud.
15
+ // • A command that matches no `pre-command` doc must never pay a daemon
16
+ // round-trip, even when another document gates a different command. Proven
17
+ // against an unavailable test API socket: a lookup fails loudly.
18
18
  //
19
19
  // The gate runs against the real canvas api server, so the subject the matcher
20
20
  // gates on is the one crtrd actually serves.
@@ -104,6 +104,7 @@ function registerSubstrate() {
104
104
  bash: (command) => fire('tool_call', { toolName: 'bash', input: { command } }),
105
105
  endTurn: async () => void (await fire('turn_end', {})),
106
106
  compact: async () => void (await fire('session_compact', {})),
107
+ toolResult: (toolName, input) => fire('tool_result', { toolName, input, content: [] }),
107
108
  };
108
109
  }
109
110
  function assertHeld(result, what) {
@@ -207,14 +208,13 @@ test('compaction re-arms the hold and leaves an exposure ledger that reloads', a
207
208
  const reheld = assertHeld(await pi.bash('deploy prod'), 'the same command after compaction');
208
209
  assert.ok(reheld.reason.includes(DOCTRINE_BODY), 'guidance the transcript no longer carries is delivered again');
209
210
  });
210
- test('a corpus with no pre-command doc never reaches the daemon', async () => {
211
- writeDoctrineDoc('command');
211
+ test('a command that matches no gate never reaches the daemon', async () => {
212
+ writeDoctrineDoc('pre-command');
212
213
  seedNode('gate-quiet');
213
214
  const pi = registerSubstrate();
214
- // No daemon. Every path below must return before the subject lookup; one that
215
- // does not fails loud with `daemon_unavailable` instead of passing quietly.
215
+ // The test API socket is unavailable. Both hooks must return before their
216
+ // subject lookup; one that does not fails loud with `daemon_unavailable`.
216
217
  await closeServer();
217
- assert.equal(await pi.toolCall('read', { file: '/tmp/x' }), undefined, 'a non-bash call is not the gate’s business');
218
- assert.equal(await pi.toolCall('bash', { command: ' ' }), undefined, 'a bash call carrying no command is not held');
219
- assert.equal(await pi.bash('deploy prod'), undefined, 'a post-execution `command` doc never holds anything');
218
+ assert.equal(await pi.bash('git status'), undefined, 'the pre-call hook ignores an unrelated pre-command document');
219
+ assert.equal(await pi.toolResult('bash', { command: 'git status' }), undefined, 'the post-result hook ignores an unrelated pre-command document');
220
220
  });