@north-light/crouter 0.3.308 → 0.3.310
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/clients/attach/viewer.js +394 -394
- package/dist/commands/__tests__/node-message.test.js +13 -0
- package/dist/commands/canvas-history/grep.js +1 -1
- package/dist/commands/canvas-history/read.js +1 -1
- package/dist/commands/canvas-history/search.js +1 -1
- package/dist/commands/memory/__tests__/command-selector-and-mutation-guards.test.js +20 -0
- package/dist/commands/memory/edit.js +8 -11
- package/dist/commands/node-inspect-artifacts.js +2 -2
- package/dist/commands/push.js +1 -1
- package/dist/core/__tests__/canvas-inbox-watcher.test.js +3 -3
- package/dist/core/__tests__/human-deliver.test.js +1 -1
- package/dist/core/__tests__/integration/broker-wedge-supervision.test.d.ts +1 -0
- package/dist/core/__tests__/integration/broker-wedge-supervision.test.js +102 -0
- package/dist/core/__tests__/integration/refresh-stall-recycle.test.js +68 -2
- package/dist/core/__tests__/integration/worktree-reap.test.js +31 -0
- package/dist/core/__tests__/seam/inbox-reference-guidance.test.d.ts +1 -0
- package/dist/core/__tests__/seam/inbox-reference-guidance.test.js +70 -0
- package/dist/core/canvas/__tests__/session-window.test.d.ts +1 -0
- package/dist/core/canvas/__tests__/session-window.test.js +70 -0
- package/dist/core/canvas/browse/app.js +52 -27
- package/dist/core/canvas/browse/render.js +11 -2
- package/dist/core/canvas/render-source.js +63 -25
- package/dist/core/command.js +15 -0
- package/dist/core/feed/inbox.js +6 -4
- package/dist/core/runtime/bearings-render.js +3 -3
- package/dist/core/runtime/broker/inbox.js +6 -4
- package/dist/core/runtime/fleet.js +1 -1
- package/dist/core/runtime/placement.d.ts +1 -1
- package/dist/core/runtime/placement.js +1 -1
- package/dist/core/runtime/tmux-driver.d.ts +7 -0
- package/dist/core/runtime/tmux-driver.js +20 -0
- package/dist/core/substrate/on-read.d.ts +6 -4
- package/dist/core/substrate/on-read.js +14 -6
- package/dist/core/substrate/surface-match.d.ts +12 -4
- package/dist/core/substrate/surface-match.js +19 -10
- package/dist/core/worktree-mutation-async.js +1 -2
- package/dist/core/worktree.d.ts +0 -1
- package/dist/core/worktree.js +0 -113
- package/dist/daemon/api/handlers/canvas.js +3 -3
- package/dist/daemon/reconcilers/broker-supervision.d.ts +14 -2
- package/dist/daemon/reconcilers/broker-supervision.js +109 -57
- package/dist/daemon/reconcilers/storage-maintenance.d.ts +5 -0
- package/dist/daemon/reconcilers/storage-maintenance.js +31 -8
- package/dist/pi-extensions/__tests__/pre-command-gate.test.js +10 -10
- package/dist/pi-extensions/canvas-doc-substrate.js +11 -5
- package/dist/shared/generated-context.d.ts +2 -0
- package/dist/shared/generated-context.js +6 -0
- package/package.json +1 -1
- package/runtime.lock.json +5 -5
package/dist/core/worktree.js
CHANGED
|
@@ -3,7 +3,6 @@ import { createHash } from 'node:crypto';
|
|
|
3
3
|
import { dirname, isAbsolute, join, relative, resolve, sep } from 'node:path';
|
|
4
4
|
import { gitAsync, gitSync } from './git.js';
|
|
5
5
|
import { crtrHome } from './canvas/paths.js';
|
|
6
|
-
import { isPidAlive } from './canvas/pid.js';
|
|
7
6
|
import { getNode, updateNode } from './canvas/index.js';
|
|
8
7
|
import { CONFIG_FILE, CRTR_DIR_NAME, STATE_FILE } from '../types.js';
|
|
9
8
|
import { withExclusiveDirectoryLock } from './exclusive-lock.js';
|
|
@@ -658,118 +657,6 @@ export function closeManagedWorktree(nodeId) {
|
|
|
658
657
|
// the child checkout instead of the parent that created it.
|
|
659
658
|
return withRepositoryWorktreeLock(existsSync(wt.path) ? wt.path : wt.repo_root, () => closeManagedWorktreeLocked(nodeId));
|
|
660
659
|
}
|
|
661
|
-
const ABANDON_AUTHOR_NAME = 'crtr-abandon';
|
|
662
|
-
const ABANDON_AUTHOR_EMAIL = 'crtr-abandon@localhost';
|
|
663
|
-
const ABANDON_SUBJECT = 'crtr-abandon';
|
|
664
|
-
function persistAbandonedWorktreeState(nodeId, wt, tipSha, at, by, branch = wt.branch) {
|
|
665
|
-
updateNode(nodeId, {
|
|
666
|
-
cwd: wt.repo_root,
|
|
667
|
-
managed_worktree: {
|
|
668
|
-
...wt,
|
|
669
|
-
state: 'abandoned',
|
|
670
|
-
abandoned: { branch, tip_sha: tipSha, at, by },
|
|
671
|
-
},
|
|
672
|
-
});
|
|
673
|
-
}
|
|
674
|
-
/** The disposal path for `abandon` when the recorded `crtr/*` branch is gone:
|
|
675
|
-
* ordinary PR work moved this checkout onto a real branch. Abandon never
|
|
676
|
-
* lands that branch and — unlike the ordinary path above — never invents a
|
|
677
|
-
* commit on it: a synthetic `crtr-abandon` marker on someone's real PR
|
|
678
|
-
* branch is not a trade this code may make on their behalf. So this
|
|
679
|
-
* requires the checkout already clean and not mid-operation, then removes
|
|
680
|
-
* only the checkout and preserves the actual branch and its commits exactly
|
|
681
|
-
* as they stand. The caller already confirmed the node is not running. */
|
|
682
|
-
function abandonBranchMovedWorktree(nodeId, wt, actualBranch, by) {
|
|
683
|
-
if (isRebaseInProgress(wt.path)) {
|
|
684
|
-
throw new WorktreeError('rebase_in_progress', 'this managed worktree is mid-rebase', `Resolve conflicts in ${wt.path}, run \`git rebase --continue\` there, then rerun \`crtr node worktree abandon\`.`);
|
|
685
|
-
}
|
|
686
|
-
const dirty = statusPorcelain(wt.path);
|
|
687
|
-
if (dirty !== '') {
|
|
688
|
-
throw new WorktreeError('dirty_worktree', 'managed worktree has uncommitted changes', `The recorded branch ${wt.branch} no longer exists; its checkout is now on ${actualBranch} with uncommitted changes crtr will not commit on your behalf. Commit or stash them in ${wt.path}, then rerun \`crtr node worktree abandon\`.`);
|
|
689
|
-
}
|
|
690
|
-
if (registeredWorktree(wt.repo_root, wt.path, actualBranch) !== true) {
|
|
691
|
-
throw new WorktreeError('wrong_branch', `managed worktree's recorded branch ${wt.branch} no longer exists, and its checkout at ${wt.path} is not registered to ${actualBranch} either`, `Inspect ${wt.path} and its git worktree registration, then ask the user with \`crtr human send\` if you are unsure how to proceed.`);
|
|
692
|
-
}
|
|
693
|
-
const actualSha = runGit(wt.path, ['rev-parse', 'HEAD'], 'abandon_tip_failed', 'Inspect the worktree manually, then retry.');
|
|
694
|
-
// Recheck identity immediately before the destructive step.
|
|
695
|
-
if (registeredWorktree(wt.repo_root, wt.path, actualBranch) !== true || currentBranch(wt.path) !== actualBranch) {
|
|
696
|
-
throw new WorktreeError('wrong_branch', `managed worktree at ${wt.path} changed while abandoning`, `Inspect ${wt.path}, then rerun \`crtr node worktree abandon\`.`);
|
|
697
|
-
}
|
|
698
|
-
runGit(wt.repo_root, ['worktree', 'remove', wt.path], 'worktree_remove_failed', 'Inspect the worktree manually, then retry.');
|
|
699
|
-
const at = new Date().toISOString();
|
|
700
|
-
persistAbandonedWorktreeState(nodeId, wt, actualSha, at, by, actualBranch);
|
|
701
|
-
return { node_id: nodeId, branch: actualBranch, tip_sha: actualSha, worktree_path: wt.path, branch_deleted: false };
|
|
702
|
-
}
|
|
703
|
-
function createAbandonMarker(wt, parentSha, nodeId, at) {
|
|
704
|
-
const treeSha = runGit(wt.repo_root, ['rev-parse', `${parentSha}^{tree}`], 'abandon_tree_failed', 'Inspect the branch manually, then retry.');
|
|
705
|
-
const markerSha = runGit(wt.repo_root, ['-c', `user.name=${ABANDON_AUTHOR_NAME}`, '-c', `user.email=${ABANDON_AUTHOR_EMAIL}`, 'commit-tree', treeSha, '-p', parentSha, '-m', ABANDON_SUBJECT, '-m', `Crtr-Abandon: ${nodeId} ${at}`], 'abandon_commit_failed', 'Inspect the branch manually, then retry.');
|
|
706
|
-
runGit(wt.repo_root, ['update-ref', `refs/heads/${wt.branch}`, markerSha, parentSha], 'abandon_marker_failed', 'The branch changed while abandon was starting; inspect the worktree, then retry.');
|
|
707
|
-
return markerSha;
|
|
708
|
-
}
|
|
709
|
-
function abandonManagedWorktreeLocked(nodeId, by) {
|
|
710
|
-
const node = getNode(nodeId);
|
|
711
|
-
if (node === null) {
|
|
712
|
-
throw new WorktreeError('unknown_node', `unknown node: ${nodeId}`, 'Pass a real node id.');
|
|
713
|
-
}
|
|
714
|
-
const wt = openManagedWorktreeForNode(nodeId);
|
|
715
|
-
if (wt === null) {
|
|
716
|
-
const { message, next } = noManagedWorktreeMessage(nodeId, 'abandon');
|
|
717
|
-
throw new WorktreeError('no_managed_worktree', message, next);
|
|
718
|
-
}
|
|
719
|
-
if (node.status === 'active' || node.status === 'idle' || isPidAlive(node.pi_pid)) {
|
|
720
|
-
throw new WorktreeError('node_running', `node ${nodeId} is still running`, 'Stop the node before abandoning its managed worktree.');
|
|
721
|
-
}
|
|
722
|
-
if (!existsSync(wt.path)) {
|
|
723
|
-
throw new WorktreeError('worktree_missing', `managed worktree path is missing: ${wt.path}`, missingWorktreeRecoveryHint(wt));
|
|
724
|
-
}
|
|
725
|
-
const actualBranch = currentBranch(wt.path);
|
|
726
|
-
if (actualBranch !== wt.branch) {
|
|
727
|
-
const recordedRefExists = recordedBranchExists(wt.path, wt.branch);
|
|
728
|
-
if (recordedRefExists || actualBranch === 'HEAD') {
|
|
729
|
-
throw new WorktreeError('wrong_branch', `managed worktree is not on its recorded branch (expected ${wt.branch}${actualBranch === 'HEAD' ? ', found detached HEAD' : ''})`, recordedRefExists
|
|
730
|
-
? `Check out ${wt.branch} in ${wt.path}, then rerun \`crtr node worktree abandon\`.`
|
|
731
|
-
: `The recorded branch ${wt.branch} no longer exists, and ${wt.path} is in detached HEAD, so there is no branch to preserve. Check out or create a branch there, then rerun \`crtr node worktree abandon\`.`);
|
|
732
|
-
}
|
|
733
|
-
// The recorded crtr/* branch is gone: ordinary PR work moved this checkout
|
|
734
|
-
// onto a real branch.
|
|
735
|
-
return abandonBranchMovedWorktree(nodeId, wt, actualBranch, by);
|
|
736
|
-
}
|
|
737
|
-
const at = new Date().toISOString();
|
|
738
|
-
const clean = proveCleanEmptyManagedWorktree(wt);
|
|
739
|
-
if (clean !== null && clean.branchSha === wt.base_sha) {
|
|
740
|
-
const markerSha = createAbandonMarker(wt, clean.branchSha, nodeId, at);
|
|
741
|
-
runGit(wt.repo_root, ['worktree', 'remove', wt.path], 'worktree_remove_failed', 'Inspect the worktree manually, then retry.');
|
|
742
|
-
const deleted = bestEffortDeleteBranchWithContainmentProof(wt, markerSha, `refs/heads/${wt.base_ref}`, clean.baseSha);
|
|
743
|
-
if (deleted.branchDeleted) {
|
|
744
|
-
persistClosedWorktreeState(nodeId, wt, 'complete');
|
|
745
|
-
return { node_id: nodeId, branch: wt.branch, tip_sha: markerSha, worktree_path: wt.path, branch_deleted: true };
|
|
746
|
-
}
|
|
747
|
-
const tipSha = runGit(wt.repo_root, ['rev-parse', `refs/heads/${wt.branch}`], 'abandon_tip_failed', 'Inspect the branch manually, then retry.');
|
|
748
|
-
persistAbandonedWorktreeState(nodeId, wt, tipSha, at, by);
|
|
749
|
-
return { node_id: nodeId, branch: wt.branch, tip_sha: tipSha, worktree_path: wt.path, branch_deleted: false };
|
|
750
|
-
}
|
|
751
|
-
// A clean branch with committed work already has its complete history. The
|
|
752
|
-
// abandon record supplies the disposition; an empty provenance commit would
|
|
753
|
-
// only move the local branch away from an already-pushed tip.
|
|
754
|
-
if (!isRebaseInProgress(wt.path) && statusPorcelain(wt.path) === '') {
|
|
755
|
-
runGit(wt.repo_root, ['worktree', 'remove', wt.path], 'worktree_remove_failed', 'Inspect the worktree manually, then retry.');
|
|
756
|
-
const tipSha = runGit(wt.repo_root, ['rev-parse', `refs/heads/${wt.branch}`], 'abandon_tip_failed', 'Inspect the branch manually, then retry.');
|
|
757
|
-
persistAbandonedWorktreeState(nodeId, wt, tipSha, at, by);
|
|
758
|
-
return { node_id: nodeId, branch: wt.branch, tip_sha: tipSha, worktree_path: wt.path, branch_deleted: false };
|
|
759
|
-
}
|
|
760
|
-
runGit(wt.path, ['add', '--all'], 'abandon_stage_failed', 'Inspect the worktree manually, then retry.');
|
|
761
|
-
runGit(wt.path, ['-c', `user.name=${ABANDON_AUTHOR_NAME}`, '-c', `user.email=${ABANDON_AUTHOR_EMAIL}`, 'commit', '--allow-empty', '-m', ABANDON_SUBJECT, '-m', `Crtr-Abandon: ${nodeId} ${at}`], 'abandon_commit_failed', 'Inspect the worktree manually, then retry.');
|
|
762
|
-
runGit(wt.repo_root, ['worktree', 'remove', wt.path], 'worktree_remove_failed', 'Inspect the worktree manually, then retry.');
|
|
763
|
-
const tipSha = runGit(wt.repo_root, ['rev-parse', `refs/heads/${wt.branch}`], 'abandon_tip_failed', 'Inspect the branch manually, then retry.');
|
|
764
|
-
persistAbandonedWorktreeState(nodeId, wt, tipSha, at, by);
|
|
765
|
-
return { node_id: nodeId, branch: wt.branch, tip_sha: tipSha, worktree_path: wt.path, branch_deleted: false };
|
|
766
|
-
}
|
|
767
|
-
export function abandonManagedWorktree(nodeId, by) {
|
|
768
|
-
const wt = openManagedWorktreeForNode(nodeId);
|
|
769
|
-
if (wt === null)
|
|
770
|
-
return abandonManagedWorktreeLocked(nodeId, by);
|
|
771
|
-
return withRepositoryWorktreeLock(wt.repo_root, () => abandonManagedWorktreeLocked(nodeId, by));
|
|
772
|
-
}
|
|
773
660
|
// Push-final auto-drop of a clean, zero-commit managed worktree
|
|
774
661
|
/** Delete a source branch only while the ref that proved it already delivered
|
|
775
662
|
* is still at the exact inspected SHA. `containingRef` can be the recorded
|
|
@@ -261,7 +261,7 @@ function handleHistoryRead(ctx) {
|
|
|
261
261
|
if (!ref.includes(':')) {
|
|
262
262
|
throw usage(`ref must be <node-id>:<relpath>; received: ${ref}`, {
|
|
263
263
|
received: ref,
|
|
264
|
-
next: 'Get a valid ref from `canvas history search` or `node inspect artifacts <node-id>`.',
|
|
264
|
+
next: 'Get a valid ref from `crtr canvas history search` or `crtr node inspect artifacts <node-id>`.',
|
|
265
265
|
});
|
|
266
266
|
}
|
|
267
267
|
let resolved;
|
|
@@ -271,13 +271,13 @@ function handleHistoryRead(ctx) {
|
|
|
271
271
|
catch (e) {
|
|
272
272
|
throw usage(`ref rejected: ${e.message}`, {
|
|
273
273
|
received: ref,
|
|
274
|
-
next: 'Refs come verbatim from `canvas history search`; do not hand-construct relpaths.',
|
|
274
|
+
next: 'Refs come verbatim from `crtr canvas history search`; do not hand-construct relpaths.',
|
|
275
275
|
});
|
|
276
276
|
}
|
|
277
277
|
if (resolved === null) {
|
|
278
278
|
throw notFound(`no artifact for ref: ${ref}`, {
|
|
279
279
|
ref,
|
|
280
|
-
next: 'The node or file may not exist. Re-run `canvas history search` for a current ref, or `node inspect artifacts <node-id>` to list a node\'s artifacts.',
|
|
280
|
+
next: 'The node or file may not exist. Re-run `crtr canvas history search` for a current ref, or `crtr node inspect artifacts <node-id>` to list a node\'s artifacts.',
|
|
281
281
|
});
|
|
282
282
|
}
|
|
283
283
|
const source = resolved.source === 'report' && resolved.reportKind ? `report:${resolved.reportKind}` : resolved.source;
|
|
@@ -11,6 +11,14 @@ export interface BrokerSupervisionContext {
|
|
|
11
11
|
fleet: FleetRegistry;
|
|
12
12
|
lifecycle: DetachedWorkLifecycle;
|
|
13
13
|
}
|
|
14
|
+
type TreeSample = {
|
|
15
|
+
cpuPercent: number;
|
|
16
|
+
hasDescendants: boolean;
|
|
17
|
+
} | null;
|
|
18
|
+
export interface BrokerSupervisionOptions {
|
|
19
|
+
readTreeSample?: (pid: number) => Promise<TreeSample>;
|
|
20
|
+
signal?: (pid: number, signal: NodeJS.Signals) => void;
|
|
21
|
+
}
|
|
14
22
|
export interface SaturationContext {
|
|
15
23
|
fleet: FleetRegistry;
|
|
16
24
|
lifecycle: DetachedWorkLifecycle;
|
|
@@ -41,13 +49,15 @@ export declare function wedgeVerdict(input: {
|
|
|
41
49
|
export declare function sumTreeCpu(psOutput: string, rootPid: number): number | null;
|
|
42
50
|
export declare function treeHasDescendants(psOutput: string, rootPid: number): boolean | null;
|
|
43
51
|
export declare class BrokerSupervisionReconciler {
|
|
44
|
-
private readonly
|
|
45
|
-
private readonly yieldTermAt;
|
|
52
|
+
private readonly yieldStalls;
|
|
46
53
|
private readonly wedgeNotifiedAt;
|
|
47
54
|
private readonly fatalFaultNotifiedAt;
|
|
48
55
|
private readonly unattendedSince;
|
|
56
|
+
private readonly sampleTree;
|
|
57
|
+
private readonly signal;
|
|
49
58
|
private knownAuthMtimeMs;
|
|
50
59
|
private lastOverCapWarnAt;
|
|
60
|
+
constructor(options?: BrokerSupervisionOptions);
|
|
51
61
|
/** The fleet-wide preamble, run BEFORE the tick's row-list guard: an auth
|
|
52
62
|
* reload does not depend on a successful canvas scan, and the thresholds it
|
|
53
63
|
* returns are read once for the whole tick. */
|
|
@@ -70,8 +80,10 @@ export declare class BrokerSupervisionReconciler {
|
|
|
70
80
|
private requestParkSummary;
|
|
71
81
|
private warnOverCapInTmux;
|
|
72
82
|
private handleYieldStall;
|
|
83
|
+
private signalYieldStall;
|
|
73
84
|
private readTreeSample;
|
|
74
85
|
private handleWedgeDetection;
|
|
75
86
|
private handleFatalFault;
|
|
76
87
|
private handleAuthReload;
|
|
77
88
|
}
|
|
89
|
+
export {};
|
|
@@ -30,6 +30,16 @@ import { shouldParkAfterSummary } from '../park-activity.js';
|
|
|
30
30
|
import { applyParkEvent, parkEvent } from './live-obligation.js';
|
|
31
31
|
import { envNoDaemonAutostart, envTestUnattendedParkMs, parkSummaryGraceMs, envUnattendedParkMs, } from '../../shared/env.js';
|
|
32
32
|
const execFileAsync = promisify(execFile);
|
|
33
|
+
function incarnationOf(pid, identity) {
|
|
34
|
+
return identity === null ? null : `${pid}:${identity}`;
|
|
35
|
+
}
|
|
36
|
+
function busySinceForIncarnation(row) {
|
|
37
|
+
const since = busySince(row.node_id);
|
|
38
|
+
const launchedAt = row.launched_at === null || row.launched_at === undefined ? Number.NaN : Date.parse(row.launched_at);
|
|
39
|
+
if (since === null || !Number.isFinite(launchedAt) || since < launchedAt)
|
|
40
|
+
return null;
|
|
41
|
+
return since;
|
|
42
|
+
}
|
|
33
43
|
export const YIELD_STALL_GRACE_MS = 3 * 60_000;
|
|
34
44
|
const KILL_ESCALATE_MS = 20_000;
|
|
35
45
|
export const WEDGE_QUIET_MS = 20 * 60_000;
|
|
@@ -131,13 +141,18 @@ function piCredentialPaths() {
|
|
|
131
141
|
];
|
|
132
142
|
}
|
|
133
143
|
export class BrokerSupervisionReconciler {
|
|
134
|
-
|
|
135
|
-
yieldTermAt = new Map();
|
|
144
|
+
yieldStalls = new Map();
|
|
136
145
|
wedgeNotifiedAt = new Map();
|
|
137
146
|
fatalFaultNotifiedAt = new Map();
|
|
138
147
|
unattendedSince = new Map();
|
|
148
|
+
sampleTree;
|
|
149
|
+
signal;
|
|
139
150
|
knownAuthMtimeMs = null;
|
|
140
151
|
lastOverCapWarnAt = Number.NEGATIVE_INFINITY;
|
|
152
|
+
constructor(options = {}) {
|
|
153
|
+
this.sampleTree = options.readTreeSample ?? ((pid) => this.readTreeSample(pid));
|
|
154
|
+
this.signal = options.signal ?? ((pid, signal) => process.kill(pid, signal));
|
|
155
|
+
}
|
|
141
156
|
/** The fleet-wide preamble, run BEFORE the tick's row-list guard: an auth
|
|
142
157
|
* reload does not depend on a successful canvas scan, and the thresholds it
|
|
143
158
|
* returns are read once for the whole tick. */
|
|
@@ -168,8 +183,8 @@ export class BrokerSupervisionReconciler {
|
|
|
168
183
|
}
|
|
169
184
|
await operationIdContext.fresh(async () => {
|
|
170
185
|
try {
|
|
171
|
-
this.handleYieldStall(row, entry.
|
|
172
|
-
await this.handleWedgeDetection(
|
|
186
|
+
this.handleYieldStall(row, entry, ctx.fleet, now);
|
|
187
|
+
await this.handleWedgeDetection(row, entry, ctx.fleet, now);
|
|
173
188
|
this.handleFatalFault(id);
|
|
174
189
|
this.handleUnattendedParking(id, getNode(id), entry.pid, now, ctx, unattendedParkMs);
|
|
175
190
|
}
|
|
@@ -391,62 +406,74 @@ export class BrokerSupervisionReconciler {
|
|
|
391
406
|
emitEvent({ level: 'error', event: 'broker.capacity.tmux_list_clients_failed', error: err });
|
|
392
407
|
}
|
|
393
408
|
}
|
|
394
|
-
handleYieldStall(row,
|
|
409
|
+
handleYieldStall(row, entry, fleet, now) {
|
|
395
410
|
const id = row.node_id;
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
411
|
+
const incarnation = incarnationOf(entry.pid, row.pi_pid_identity);
|
|
412
|
+
if (row.intent !== 'refresh' || busySinceForIncarnation(row) !== null || incarnation === null) {
|
|
413
|
+
if (incarnation !== null)
|
|
414
|
+
this.yieldStalls.delete(incarnation);
|
|
399
415
|
return;
|
|
400
416
|
}
|
|
401
|
-
|
|
402
|
-
if (
|
|
403
|
-
|
|
417
|
+
let state = this.yieldStalls.get(incarnation);
|
|
418
|
+
if (state === undefined) {
|
|
419
|
+
state = { since: now };
|
|
420
|
+
this.yieldStalls.set(incarnation, state);
|
|
404
421
|
return;
|
|
405
422
|
}
|
|
406
|
-
if (yieldStallVerdict(true, row.intent, false, now - since) !== 'kill')
|
|
423
|
+
if (yieldStallVerdict(true, row.intent, false, now - state.since) !== 'kill')
|
|
407
424
|
return;
|
|
408
|
-
|
|
425
|
+
const current = getNode(id);
|
|
426
|
+
if (fleet.get(id) !== entry || current?.pi_pid !== entry.pid || current.pi_pid_identity !== row.pi_pid_identity || recordedPidLiveness(entry.pid, row.pi_pid_identity) !== 'alive')
|
|
409
427
|
return;
|
|
410
|
-
|
|
411
|
-
|
|
428
|
+
if (state.termedAt === undefined) {
|
|
429
|
+
if (!this.signalYieldStall(id, entry.pid, 'SIGTERM'))
|
|
430
|
+
return;
|
|
431
|
+
state.termedAt = now;
|
|
412
432
|
emitEvent({
|
|
413
433
|
level: 'warn',
|
|
414
434
|
event: 'broker.yield_stall.sigterm',
|
|
415
435
|
...(isSafeNodeId(id) ? { node_id: id } : {}),
|
|
416
436
|
fields: {
|
|
417
437
|
...(isSafeNodeId(id) ? {} : { affected_node_id: id }),
|
|
418
|
-
pid,
|
|
419
|
-
elapsed_ms: now - since,
|
|
438
|
+
pid: entry.pid,
|
|
439
|
+
elapsed_ms: now - state.since,
|
|
420
440
|
grace_ms: YIELD_STALL_GRACE_MS,
|
|
421
441
|
},
|
|
422
442
|
});
|
|
423
|
-
try {
|
|
424
|
-
process.kill(pid, 'SIGTERM');
|
|
425
|
-
}
|
|
426
|
-
catch {
|
|
427
|
-
/* already gone */
|
|
428
|
-
}
|
|
429
|
-
this.yieldTermAt.set(id, now);
|
|
430
443
|
}
|
|
431
|
-
else if (now -
|
|
444
|
+
else if (now - state.termedAt >= KILL_ESCALATE_MS && this.signalYieldStall(id, entry.pid, 'SIGKILL')) {
|
|
432
445
|
emitEvent({
|
|
433
446
|
level: 'warn',
|
|
434
447
|
event: 'broker.yield_stall.sigkill',
|
|
435
448
|
...(isSafeNodeId(id) ? { node_id: id } : {}),
|
|
436
449
|
fields: {
|
|
437
450
|
...(isSafeNodeId(id) ? {} : { affected_node_id: id }),
|
|
438
|
-
pid,
|
|
439
|
-
elapsed_ms: now -
|
|
451
|
+
pid: entry.pid,
|
|
452
|
+
elapsed_ms: now - state.termedAt,
|
|
440
453
|
grace_ms: KILL_ESCALATE_MS,
|
|
441
|
-
stall_elapsed_ms: now - since,
|
|
454
|
+
stall_elapsed_ms: now - state.since,
|
|
442
455
|
},
|
|
443
456
|
});
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
457
|
+
}
|
|
458
|
+
}
|
|
459
|
+
signalYieldStall(id, pid, signal) {
|
|
460
|
+
try {
|
|
461
|
+
this.signal(pid, signal);
|
|
462
|
+
return true;
|
|
463
|
+
}
|
|
464
|
+
catch (err) {
|
|
465
|
+
emitEvent({
|
|
466
|
+
level: 'error',
|
|
467
|
+
event: 'broker.yield_stall.signal_failed',
|
|
468
|
+
...(isSafeNodeId(id) ? { node_id: id } : {}),
|
|
469
|
+
error: err,
|
|
470
|
+
fields: {
|
|
471
|
+
...(isSafeNodeId(id) ? {} : { affected_node_id: id }),
|
|
472
|
+
pid,
|
|
473
|
+
signal,
|
|
474
|
+
},
|
|
475
|
+
});
|
|
476
|
+
return false;
|
|
450
477
|
}
|
|
451
478
|
}
|
|
452
479
|
async readTreeSample(pid) {
|
|
@@ -467,29 +494,36 @@ export class BrokerSupervisionReconciler {
|
|
|
467
494
|
return null;
|
|
468
495
|
}
|
|
469
496
|
}
|
|
470
|
-
async handleWedgeDetection(
|
|
471
|
-
|
|
472
|
-
|
|
497
|
+
async handleWedgeDetection(row, entry, fleet, now) {
|
|
498
|
+
const id = row.node_id;
|
|
499
|
+
const incarnation = incarnationOf(entry.pid, row.pi_pid_identity);
|
|
500
|
+
const since = busySinceForIncarnation(row);
|
|
501
|
+
if (since === null || incarnation === null) {
|
|
502
|
+
if (incarnation !== null)
|
|
503
|
+
this.wedgeNotifiedAt.delete(incarnation);
|
|
473
504
|
return;
|
|
474
505
|
}
|
|
475
|
-
const
|
|
476
|
-
const
|
|
477
|
-
const
|
|
506
|
+
const quietForMs = now - since;
|
|
507
|
+
const sample = quietForMs >= WEDGE_QUIET_MS ? await this.sampleTree(entry.pid) : null;
|
|
508
|
+
const current = getNode(id);
|
|
509
|
+
const currentSince = current === null ? null : busySinceForIncarnation(current);
|
|
510
|
+
if (fleet.get(id) !== entry || current === null || current.pi_pid !== entry.pid || current.pi_pid_identity !== row.pi_pid_identity || currentSince !== since)
|
|
511
|
+
return;
|
|
478
512
|
const cpuPercent = sample?.cpuPercent ?? null;
|
|
479
513
|
const sampleHasDescendants = sample?.hasDescendants ?? null;
|
|
480
514
|
if (wedgeVerdict({ busy: true, quietForMs, cpuPercent, hasDescendants: sampleHasDescendants }) !== 'wedged')
|
|
481
515
|
return;
|
|
482
|
-
if (
|
|
516
|
+
if (this.wedgeNotifiedAt.get(incarnation) === since)
|
|
483
517
|
return;
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
const meta = getNode(id);
|
|
487
|
-
const name = meta !== null ? fullName(meta) : id;
|
|
518
|
+
this.wedgeNotifiedAt.set(incarnation, since);
|
|
519
|
+
const name = fullName(current);
|
|
488
520
|
const minutes = Math.round((quietForMs ?? WEDGE_QUIET_MS) / 60_000);
|
|
489
521
|
const hasDescendants = sampleHasDescendants ?? true;
|
|
490
|
-
const kickSuppressionReason = !hasDescendants &&
|
|
522
|
+
const kickSuppressionReason = !hasDescendants && recordedPidLiveness(entry.pid, row.pi_pid_identity) !== 'alive'
|
|
491
523
|
? 'pid_identity_unproven'
|
|
492
524
|
: null;
|
|
525
|
+
let kickAttempted = false;
|
|
526
|
+
let kicked = false;
|
|
493
527
|
let label;
|
|
494
528
|
if (hasDescendants) {
|
|
495
529
|
label =
|
|
@@ -506,16 +540,32 @@ export class BrokerSupervisionReconciler {
|
|
|
506
540
|
`confirmed alive.`;
|
|
507
541
|
}
|
|
508
542
|
else {
|
|
509
|
-
|
|
510
|
-
`Child wedged — ${name} (${id}) is alive and mid-turn but has shown NO engine progress for ` +
|
|
511
|
-
`${minutes}+ minutes and it has no subprocess to kill (the engine itself stalled). Kicking it ` +
|
|
512
|
-
`now: SIGTERM to the broker — the daemon will resume a cleanly ` +
|
|
513
|
-
`aborted session with a continuation, or start a recovery cycle if the abort cannot settle.`;
|
|
543
|
+
kickAttempted = true;
|
|
514
544
|
try {
|
|
515
|
-
|
|
545
|
+
this.signal(entry.pid, 'SIGTERM');
|
|
546
|
+
kicked = true;
|
|
547
|
+
label =
|
|
548
|
+
`Child wedged — ${name} (${id}) is alive and mid-turn but has shown NO engine progress for ` +
|
|
549
|
+
`${minutes}+ minutes and it has no subprocess to kill (the engine itself stalled). The daemon sent ` +
|
|
550
|
+
`SIGTERM to the broker and will resume a cleanly aborted session with a continuation, or start a ` +
|
|
551
|
+
`recovery cycle if the abort cannot settle.`;
|
|
516
552
|
}
|
|
517
|
-
catch {
|
|
518
|
-
|
|
553
|
+
catch (err) {
|
|
554
|
+
emitEvent({
|
|
555
|
+
level: 'error',
|
|
556
|
+
event: 'broker.wedge.signal_failed',
|
|
557
|
+
...(isSafeNodeId(id) ? { node_id: id } : {}),
|
|
558
|
+
error: err,
|
|
559
|
+
fields: {
|
|
560
|
+
...(isSafeNodeId(id) ? {} : { affected_node_id: id }),
|
|
561
|
+
pid: entry.pid,
|
|
562
|
+
signal: 'SIGTERM',
|
|
563
|
+
},
|
|
564
|
+
});
|
|
565
|
+
label =
|
|
566
|
+
`Child wedged — ${name} (${id}) is alive and mid-turn but has shown NO engine progress for ` +
|
|
567
|
+
`${minutes}+ minutes and it has no subprocess to kill (the engine itself stalled). The daemon ` +
|
|
568
|
+
`attempted SIGTERM to the broker, but the signal call failed.`;
|
|
519
569
|
}
|
|
520
570
|
}
|
|
521
571
|
emitEvent({
|
|
@@ -524,11 +574,12 @@ export class BrokerSupervisionReconciler {
|
|
|
524
574
|
...(isSafeNodeId(id) ? { node_id: id } : {}),
|
|
525
575
|
fields: {
|
|
526
576
|
...(isSafeNodeId(id) ? {} : { affected_node_id: id }),
|
|
527
|
-
pid,
|
|
577
|
+
pid: entry.pid,
|
|
528
578
|
quiet_ms: quietForMs,
|
|
529
579
|
cpu_percent: cpuPercent,
|
|
530
580
|
has_descendants: hasDescendants,
|
|
531
|
-
|
|
581
|
+
kick_attempted: kickAttempted,
|
|
582
|
+
kicked,
|
|
532
583
|
...(kickSuppressionReason === null ? {} : { kick_suppression_reason: kickSuppressionReason }),
|
|
533
584
|
},
|
|
534
585
|
});
|
|
@@ -554,7 +605,8 @@ export class BrokerSupervisionReconciler {
|
|
|
554
605
|
child: id,
|
|
555
606
|
quiet_ms: quietForMs,
|
|
556
607
|
cpu_percent: cpuPercent,
|
|
557
|
-
|
|
608
|
+
kick_attempted: kickAttempted,
|
|
609
|
+
kicked,
|
|
558
610
|
...(kickSuppressionReason === null ? {} : { kick_suppression_reason: kickSuppressionReason }),
|
|
559
611
|
});
|
|
560
612
|
}
|
|
@@ -13,6 +13,7 @@ export interface StorageMaintenanceContext {
|
|
|
13
13
|
export declare class StorageMaintenanceReconciler {
|
|
14
14
|
private lastSpareSweepAt;
|
|
15
15
|
private lastGhostSweepAt;
|
|
16
|
+
private focusGcInFlight;
|
|
16
17
|
private readonly worktreeSweep;
|
|
17
18
|
run(now: number, ctx: StorageMaintenanceContext): void;
|
|
18
19
|
/** Drop rows whose on-disk node dir is gone. Such a row can never be revived,
|
|
@@ -25,6 +26,10 @@ export declare class StorageMaintenanceReconciler {
|
|
|
25
26
|
/** Reap only physical placement residue of terminal nodes. The durable row,
|
|
26
27
|
* graph, and on-disk state remain available for a later manual revive. */
|
|
27
28
|
private reapDeadResidue;
|
|
29
|
+
/** Register one detached focus-GC pass, never overlapping another. The pane
|
|
30
|
+
* probe is a subprocess, and running it on the tick blocks the API socket for
|
|
31
|
+
* its full duration on every beat. */
|
|
32
|
+
private scheduleFocusGc;
|
|
28
33
|
/** Delete focus rows whose recorded tmux pane no longer exists. A failed tmux
|
|
29
34
|
* probe is "can't tell" and never authorizes deleting every viewport. */
|
|
30
35
|
private gcStaleFocuses;
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import { statSync } from 'node:fs';
|
|
2
2
|
import { join } from 'node:path';
|
|
3
|
-
import { closeFocusRow, listFocuses, listNodes, reapGhostRows, } from '../../core/canvas/index.js';
|
|
3
|
+
import { closeFocusRow, getFocusById, listFocuses, listNodes, reapGhostRows, } from '../../core/canvas/index.js';
|
|
4
4
|
import { isSafeNodeId, jobDir, nodeDir } from '../../core/canvas/paths.js';
|
|
5
5
|
import { emitEvent } from '../../core/events/emit.js';
|
|
6
6
|
import { operationIdContext } from '../../core/events/operation-id.js';
|
|
7
|
-
import {
|
|
7
|
+
import { listLivePanesAsync, tearDownNode } from '../../core/runtime/placement.js';
|
|
8
8
|
import { reapStaleSpares } from '../../core/runtime/warm-pool.js';
|
|
9
9
|
import { ManagedWorktreeSweepReconciler } from './managed-worktree-sweep.js';
|
|
10
10
|
// How long a dead node's on-disk record must be quiet before its leftover
|
|
@@ -18,13 +18,16 @@ const GHOST_SWEEP_INTERVAL_MS = 60 * 1000;
|
|
|
18
18
|
export class StorageMaintenanceReconciler {
|
|
19
19
|
lastSpareSweepAt = Number.NEGATIVE_INFINITY;
|
|
20
20
|
lastGhostSweepAt = Number.NEGATIVE_INFINITY;
|
|
21
|
+
focusGcInFlight = false;
|
|
21
22
|
worktreeSweep = new ManagedWorktreeSweepReconciler();
|
|
22
23
|
run(now, ctx) {
|
|
23
24
|
// Placement reap MUST precede focus GC, so GC catches a focus row the reap
|
|
24
25
|
// strands. Both run after the lifecycle tick, so a same-tick revive has
|
|
25
|
-
// already had the chance to re-anchor its focus.
|
|
26
|
+
// already had the chance to re-anchor its focus. The GC is detached, but it
|
|
27
|
+
// reads the focus table synchronously before its first await, so it still
|
|
28
|
+
// observes the post-reap state this ordering exists to give it.
|
|
26
29
|
this.reapDeadResidue(now);
|
|
27
|
-
this.
|
|
30
|
+
this.scheduleFocusGc(ctx);
|
|
28
31
|
this.reapGhostRows(now);
|
|
29
32
|
this.gcWarmPool(now);
|
|
30
33
|
// Registers detached async Git work and returns immediately: this lane is
|
|
@@ -140,9 +143,23 @@ export class StorageMaintenanceReconciler {
|
|
|
140
143
|
});
|
|
141
144
|
}
|
|
142
145
|
}
|
|
146
|
+
/** Register one detached focus-GC pass, never overlapping another. The pane
|
|
147
|
+
* probe is a subprocess, and running it on the tick blocks the API socket for
|
|
148
|
+
* its full duration on every beat. */
|
|
149
|
+
scheduleFocusGc(ctx) {
|
|
150
|
+
if (this.focusGcInFlight)
|
|
151
|
+
return;
|
|
152
|
+
if (!ctx.lifecycle.acceptsDetachedWork())
|
|
153
|
+
return;
|
|
154
|
+
this.focusGcInFlight = true;
|
|
155
|
+
const pass = this.gcStaleFocuses().finally(() => {
|
|
156
|
+
this.focusGcInFlight = false;
|
|
157
|
+
});
|
|
158
|
+
ctx.lifecycle.registerDetached(pass);
|
|
159
|
+
}
|
|
143
160
|
/** Delete focus rows whose recorded tmux pane no longer exists. A failed tmux
|
|
144
161
|
* probe is "can't tell" and never authorizes deleting every viewport. */
|
|
145
|
-
gcStaleFocuses() {
|
|
162
|
+
async gcStaleFocuses() {
|
|
146
163
|
let rows;
|
|
147
164
|
try {
|
|
148
165
|
rows = listFocuses();
|
|
@@ -157,7 +174,7 @@ export class StorageMaintenanceReconciler {
|
|
|
157
174
|
return;
|
|
158
175
|
let live;
|
|
159
176
|
try {
|
|
160
|
-
live =
|
|
177
|
+
live = await listLivePanesAsync();
|
|
161
178
|
}
|
|
162
179
|
catch (err) {
|
|
163
180
|
operationIdContext.fresh(() => {
|
|
@@ -170,8 +187,14 @@ export class StorageMaintenanceReconciler {
|
|
|
170
187
|
for (const focus of rows) {
|
|
171
188
|
if (focus.pane === null || live.has(focus.pane))
|
|
172
189
|
continue;
|
|
173
|
-
const
|
|
190
|
+
const proceed = operationIdContext.fresh(() => {
|
|
174
191
|
try {
|
|
192
|
+
// The probe awaited above let other event-loop work run, so a viewer
|
|
193
|
+
// may have re-pointed this row at a pane the probe never saw. Re-read
|
|
194
|
+
// and match the pane before deleting; the read and the delete are both
|
|
195
|
+
// synchronous, so nothing can interleave between them.
|
|
196
|
+
if (getFocusById(focus.focus_id)?.pane !== focus.pane)
|
|
197
|
+
return true;
|
|
175
198
|
emitEvent({
|
|
176
199
|
level: 'info',
|
|
177
200
|
event: 'focus.gc.removed',
|
|
@@ -190,7 +213,7 @@ export class StorageMaintenanceReconciler {
|
|
|
190
213
|
return false;
|
|
191
214
|
}
|
|
192
215
|
});
|
|
193
|
-
if (!
|
|
216
|
+
if (!proceed)
|
|
194
217
|
return;
|
|
195
218
|
}
|
|
196
219
|
}
|
|
@@ -12,9 +12,9 @@
|
|
|
12
12
|
// executes them concurrently, so without a per-turn flag the second sibling
|
|
13
13
|
// would find the doc already exposed and slip past the hold its sibling just
|
|
14
14
|
// raised.
|
|
15
|
-
// • A
|
|
16
|
-
//
|
|
17
|
-
//
|
|
15
|
+
// • A command that matches no `pre-command` doc must never pay a daemon
|
|
16
|
+
// round-trip, even when another document gates a different command. Proven
|
|
17
|
+
// against an unavailable test API socket: a lookup fails loudly.
|
|
18
18
|
//
|
|
19
19
|
// The gate runs against the real canvas api server, so the subject the matcher
|
|
20
20
|
// gates on is the one crtrd actually serves.
|
|
@@ -104,6 +104,7 @@ function registerSubstrate() {
|
|
|
104
104
|
bash: (command) => fire('tool_call', { toolName: 'bash', input: { command } }),
|
|
105
105
|
endTurn: async () => void (await fire('turn_end', {})),
|
|
106
106
|
compact: async () => void (await fire('session_compact', {})),
|
|
107
|
+
toolResult: (toolName, input) => fire('tool_result', { toolName, input, content: [] }),
|
|
107
108
|
};
|
|
108
109
|
}
|
|
109
110
|
function assertHeld(result, what) {
|
|
@@ -207,14 +208,13 @@ test('compaction re-arms the hold and leaves an exposure ledger that reloads', a
|
|
|
207
208
|
const reheld = assertHeld(await pi.bash('deploy prod'), 'the same command after compaction');
|
|
208
209
|
assert.ok(reheld.reason.includes(DOCTRINE_BODY), 'guidance the transcript no longer carries is delivered again');
|
|
209
210
|
});
|
|
210
|
-
test('a
|
|
211
|
-
writeDoctrineDoc('command');
|
|
211
|
+
test('a command that matches no gate never reaches the daemon', async () => {
|
|
212
|
+
writeDoctrineDoc('pre-command');
|
|
212
213
|
seedNode('gate-quiet');
|
|
213
214
|
const pi = registerSubstrate();
|
|
214
|
-
//
|
|
215
|
-
// does not fails loud with `daemon_unavailable
|
|
215
|
+
// The test API socket is unavailable. Both hooks must return before their
|
|
216
|
+
// subject lookup; one that does not fails loud with `daemon_unavailable`.
|
|
216
217
|
await closeServer();
|
|
217
|
-
assert.equal(await pi.
|
|
218
|
-
assert.equal(await pi.
|
|
219
|
-
assert.equal(await pi.bash('deploy prod'), undefined, 'a post-execution `command` doc never holds anything');
|
|
218
|
+
assert.equal(await pi.bash('git status'), undefined, 'the pre-call hook ignores an unrelated pre-command document');
|
|
219
|
+
assert.equal(await pi.toolResult('bash', { command: 'git status' }), undefined, 'the post-result hook ignores an unrelated pre-command document');
|
|
220
220
|
});
|