@yemi33/minions 0.1.2423 → 0.1.2425
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/install-internal-minions.js +1209 -0
- package/bin/minions.js +75 -9
- package/dashboard/docs/typography.md +27 -12
- package/dashboard/js/command-center.js +13 -11
- package/dashboard/js/live-stream.js +1 -1
- package/dashboard/js/memory-search.js +388 -43
- package/dashboard/js/modal-qa.js +5 -5
- package/dashboard/js/qa.js +39 -26
- package/dashboard/js/refresh.js +37 -18
- package/dashboard/js/render-agents.js +14 -18
- package/dashboard/js/render-dispatch.js +14 -74
- package/dashboard/js/render-inbox.js +3 -3
- package/dashboard/js/render-meetings.js +8 -8
- package/dashboard/js/render-other.js +45 -20
- package/dashboard/js/render-pipelines.js +6 -6
- package/dashboard/js/render-plans.js +14 -14
- package/dashboard/js/render-prd.js +46 -46
- package/dashboard/js/render-prs.js +9 -65
- package/dashboard/js/render-schedules.js +5 -5
- package/dashboard/js/render-skills.js +6 -6
- package/dashboard/js/render-utils.js +4 -4
- package/dashboard/js/render-watches.js +4 -16
- package/dashboard/js/render-work-items.js +95 -107
- package/dashboard/js/settings.js +175 -74
- package/dashboard/layout.html +3 -2
- package/dashboard/pages/inbox.html +1 -1
- package/dashboard/pages/work.html +1 -1
- package/dashboard/shared/model-display.js +9 -0
- package/dashboard/shared/watches-source.js +43 -0
- package/dashboard/shared/welcome-popup.js +259 -0
- package/dashboard/slim/body.html +6 -6
- package/dashboard/slim/js/chat.js +4 -2
- package/dashboard/slim/js/history.js +10 -3
- package/dashboard/slim/js/modals-tiles.js +85 -16
- package/dashboard/slim/js/status.js +12 -16
- package/dashboard/slim/styles.css +24 -4
- package/dashboard/styles.css +62 -27
- package/dashboard-build.js +8 -2
- package/dashboard.js +3854 -2514
- package/docs/README.md +6 -4
- package/docs/api-errors.md +144 -0
- package/docs/auto-discovery.md +83 -61
- package/docs/capture-demos.js +173 -30
- package/docs/command-center.md +5 -1
- package/docs/completion-reports.md +127 -11
- package/docs/constants.md +15 -3
- package/docs/constellation-style-telemetry.md +2 -2
- package/docs/copilot-cli-schema.md +32 -22
- package/docs/cross-repo-plans.md +30 -30
- package/docs/demo/memory-system.html +1 -1
- package/docs/deprecated.json +132 -18
- package/docs/diagnostics-memory.md +8 -4
- package/docs/human-vs-automated.md +2 -2
- package/docs/internal-install.md +190 -0
- package/docs/kb-pr3223-cascade-archiving.md +16 -0
- package/docs/kb-pr696-merge-conflict-docs.md +23 -0
- package/docs/kb-sweep.md +2 -2
- package/docs/keep-processes.md +8 -1
- package/docs/live-checkout-mode.md +12 -7
- package/docs/managed-spawn.md +21 -3
- package/docs/named-agents.md +3 -2
- package/docs/plan-lifecycle.md +32 -32
- package/docs/pr-auto-fix-dispatch.md +3 -3
- package/docs/pr-review-fix-loop.md +6 -6
- package/docs/pr-screenshots/pr-899/worker-pool-worktrees-AFTER.png +0 -0
- package/docs/pr-screenshots/pr-899/worker-pool-worktrees-BEFORE.png +0 -0
- package/docs/pr-screenshots/pr-979/auto-fix-pane-AFTER.png +0 -0
- package/docs/pr-screenshots/pr-979/auto-fix-pane-BEFORE.png +0 -0
- package/docs/pr-screenshots/pr-985/pr-column-em-dash-AFTER.png +0 -0
- package/docs/pr-screenshots/pr-985/pr-column-em-dash-BEFORE.png +0 -0
- package/docs/qa-runbook-lifecycle.md +67 -51
- package/docs/qa-runbooks.md +2 -2
- package/docs/runtime-adapters.md +101 -50
- package/docs/security.md +28 -0
- package/docs/self-improvement.md +47 -13
- package/docs/skills.md +29 -0
- package/docs/slim-ux/concepts.md +2 -2
- package/docs/specs/agent-configurability.md +43 -41
- package/docs/specs/agent-rename.md +13 -13
- package/docs/team-memory.md +283 -14
- package/docs/tutorials/03-plan-driven-feature.md +3 -3
- package/docs/tutorials/04-runtimes-and-harness.md +18 -21
- package/docs/tutorials/08-operations-and-recovery.md +3 -2
- package/docs/tutorials/README.md +4 -3
- package/docs/visual-evidence-ci.md +103 -0
- package/docs/watches.md +35 -30
- package/docs/worktree-lifecycle.md +151 -13
- package/engine/acp-transport.js +1 -0
- package/engine/ado.js +80 -33
- package/engine/agent-api-validation.js +571 -0
- package/engine/agent-worker-pool.js +7 -6
- package/engine/api-contracts/agent-content.js +761 -0
- package/engine/api-contracts/cc-ops.js +219 -0
- package/engine/api-contracts/config-runtime.js +419 -0
- package/engine/api-contracts/core.js +138 -0
- package/engine/api-contracts/index.js +506 -0
- package/engine/api-contracts/orchestration.js +602 -0
- package/engine/api-contracts/pull-requests.js +323 -0
- package/engine/api-contracts/qa-process.js +764 -0
- package/engine/api-contracts/work-plan-prd.js +494 -0
- package/engine/api-validation.js +543 -0
- package/engine/cc-api-validation.js +586 -0
- package/engine/cc-worker-pool.js +9 -1
- package/engine/cleanup.js +393 -271
- package/engine/cli.js +65 -7
- package/engine/comment-classifier.js +70 -5
- package/engine/consolidation.js +123 -5
- package/engine/content-api-validation.js +606 -0
- package/engine/create-pr-worktree.js +22 -1
- package/engine/db/migrations/027-review-learning-lifecycle.js +162 -0
- package/engine/dispatch.js +33 -0
- package/engine/distribution.js +189 -0
- package/engine/execution-model.js +59 -0
- package/engine/features.js +12 -0
- package/engine/github.js +67 -25
- package/engine/harness-context.js +313 -0
- package/engine/inbox-store.js +1 -1
- package/engine/kb-sweep.js +161 -38
- package/engine/keep-process-sweep.js +43 -3
- package/engine/lifecycle.js +1066 -165
- package/engine/llm.js +4 -0
- package/engine/managed-spawn.js +463 -48
- package/engine/meeting.js +197 -34
- package/engine/memory-retrieval.js +280 -0
- package/engine/memory-store.js +821 -38
- package/engine/model-discovery.js +17 -3
- package/engine/pipeline.js +802 -14
- package/engine/plan-prd-validation.js +745 -0
- package/engine/playbook.js +206 -17
- package/engine/pooled-agent-process.js +6 -1
- package/engine/pr-action.js +1 -1
- package/engine/pr-issue-validation.js +653 -0
- package/engine/pr-resolve.js +75 -13
- package/engine/prd-store.js +39 -6
- package/engine/preflight.js +7 -2
- package/engine/process-utils.js +79 -31
- package/engine/projects.js +198 -12
- package/engine/promotion.js +371 -0
- package/engine/qa-from-prd.js +3 -0
- package/engine/qa-process-validation.js +572 -0
- package/engine/qa-runbooks.js +41 -37
- package/engine/qa-runners/maestro.js +11 -7
- package/engine/qa-runners/playwright.js +37 -14
- package/engine/qa-runners.js +9 -10
- package/engine/qa-runs.js +124 -11
- package/engine/qa-sessions.js +139 -40
- package/engine/quarantine-refs.js +81 -11
- package/engine/queries.js +163 -9
- package/engine/review-learning-backfill.js +482 -0
- package/engine/review-learning.js +1236 -0
- package/engine/runtimes/claude.js +60 -2
- package/engine/runtimes/codex.js +57 -16
- package/engine/runtimes/contract.js +21 -0
- package/engine/runtimes/copilot.js +51 -3
- package/engine/runtimes/index.js +1 -0
- package/engine/scheduler.js +201 -32
- package/engine/settings-validation.js +909 -0
- package/engine/shared.js +705 -208
- package/engine/spawn-agent.js +48 -21
- package/engine/steering-constraints.js +31 -0
- package/engine/steering-store.js +10 -2
- package/engine/steering.js +14 -2
- package/engine/timeout.js +404 -111
- package/engine/untrusted-fence.js +24 -10
- package/engine/watch-actions.js +7 -2
- package/engine/watches.js +353 -51
- package/engine/work-item-validation.js +561 -0
- package/engine/work-items-store.js +61 -1
- package/engine/worktree-gc.js +656 -37
- package/engine/worktree-preflight.js +154 -0
- package/engine.js +2068 -342
- package/package.json +1 -1
- package/playbooks/_pr-description-audit.md +88 -41
- package/playbooks/build-fix-complex.md +3 -3
- package/playbooks/docs.md +2 -2
- package/playbooks/fix.md +18 -18
- package/playbooks/implement.md +18 -18
- package/playbooks/qa-session-draft.md +10 -12
- package/playbooks/qa-session-execute.md +16 -12
- package/playbooks/qa-validate.md +8 -7
- package/playbooks/shared-rules.md +45 -0
- package/playbooks/test.md +4 -0
- package/prompts/cc-system.md +1 -1
- package/skills/check-self-authored-review-comment/SKILL.md +2 -2
package/engine/timeout.js
CHANGED
|
@@ -23,7 +23,6 @@ const MINIONS_DIR = shared.MINIONS_DIR;
|
|
|
23
23
|
// Only needed for engine().activeProcesses and engine().engineRestartGraceUntil — log/ts come from shared.js
|
|
24
24
|
let _engine = null;
|
|
25
25
|
function engine() { if (!_engine) _engine = require('../engine'); return _engine; }
|
|
26
|
-
const _pendingBranchPreservations = new Set();
|
|
27
26
|
|
|
28
27
|
function resolveDispatchBranchPath(item, config) {
|
|
29
28
|
if (item?.worktreePath) return item.worktreePath;
|
|
@@ -51,11 +50,52 @@ function preserveDispatchBranch(item, config) {
|
|
|
51
50
|
});
|
|
52
51
|
}
|
|
53
52
|
|
|
53
|
+
function resolveDispatchProject(item, config) {
|
|
54
|
+
const persistedProject = item?.meta?.project || {};
|
|
55
|
+
return getProjects(config).find(project => project.name === persistedProject.name) || persistedProject;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
function getWorktreeProcessAnchorReason(item, managedSpawnSpawnedCount = 0) {
|
|
59
|
+
if (!item?.worktreePath) return null;
|
|
60
|
+
const decision = require('./worktree-gc').shouldGcDispatchWorktree({
|
|
61
|
+
worktreePath: item.worktreePath,
|
|
62
|
+
agentId: item.agent,
|
|
63
|
+
managedSpawnSpawnedCount,
|
|
64
|
+
isPoolMember: () => false,
|
|
65
|
+
});
|
|
66
|
+
return decision.reason === 'keep-processes-anchored' || decision.reason === 'managed-spawn-anchored'
|
|
67
|
+
? decision.reason
|
|
68
|
+
: null;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
function gcRecoveredDispatchWorktree(item, project, config) {
|
|
72
|
+
if (!item?.worktreePath || !project?.localPath) return;
|
|
73
|
+
const worktreeRoot = path.resolve(
|
|
74
|
+
project.localPath,
|
|
75
|
+
config.engine?.worktreeRoot || ENGINE_DEFAULTS.worktreeRoot,
|
|
76
|
+
);
|
|
77
|
+
require('./worktree-gc').gcDispatchWorktreeIfOrphan({
|
|
78
|
+
worktreePath: item.worktreePath,
|
|
79
|
+
gitRoot: project.localPath,
|
|
80
|
+
worktreeRoot,
|
|
81
|
+
agentId: item.agent,
|
|
82
|
+
excludeDispatchId: item.id,
|
|
83
|
+
log,
|
|
84
|
+
});
|
|
85
|
+
}
|
|
86
|
+
|
|
54
87
|
async function _finalizeDeadDispatch(item, reason, failureClass, config, deps = {}) {
|
|
55
88
|
const preserveBranch = deps.preserveBranch || preserveDispatchBranch;
|
|
56
89
|
const restoreLiveCheckout = deps.restoreLiveCheckout ||
|
|
57
90
|
require('./live-checkout').maybeRestoreLiveCheckoutFromRecord;
|
|
58
91
|
const complete = deps.complete || dispatch().completeDispatch;
|
|
92
|
+
const deferredTerminal = item?._worktreeProtectionDeferred?.terminal || {};
|
|
93
|
+
const terminalReason = deferredTerminal.reason || reason;
|
|
94
|
+
const terminalResultSummary = deferredTerminal.resultSummary || '';
|
|
95
|
+
const terminalCompleteOpts = deferredTerminal.completeOpts
|
|
96
|
+
&& typeof deferredTerminal.completeOpts === 'object'
|
|
97
|
+
? deferredTerminal.completeOpts
|
|
98
|
+
: (failureClass ? { failureClass } : {});
|
|
59
99
|
|
|
60
100
|
if (resolveDispatchBranchPath(item, config) && item.meta?.branch) {
|
|
61
101
|
try {
|
|
@@ -64,6 +104,43 @@ async function _finalizeDeadDispatch(item, reason, failureClass, config, deps =
|
|
|
64
104
|
log('warn', `Orphan branch preservation failed for ${item.id}: ${err.message}`);
|
|
65
105
|
}
|
|
66
106
|
}
|
|
107
|
+
const project = resolveDispatchProject(item, config);
|
|
108
|
+
const anchorReason = getWorktreeProcessAnchorReason(item);
|
|
109
|
+
let protection = {
|
|
110
|
+
allowRemoval: !anchorReason,
|
|
111
|
+
preserved: false,
|
|
112
|
+
reason: anchorReason || 'not-checked',
|
|
113
|
+
};
|
|
114
|
+
if (item.worktreePath && item.meta?.branch && project.localPath) {
|
|
115
|
+
protection = await engine()._preserveFailedDispatchWorktree({
|
|
116
|
+
effectiveResult: DISPATCH_RESULT.ERROR,
|
|
117
|
+
rootDir: project.localPath,
|
|
118
|
+
worktreePath: item.worktreePath,
|
|
119
|
+
branchName: item.meta.branch,
|
|
120
|
+
dispatchId: item.id,
|
|
121
|
+
mainBranch: shared.resolveMainBranch(project.localPath, project.mainBranch),
|
|
122
|
+
project,
|
|
123
|
+
gitOpts: { env: shared.gitEnv() },
|
|
124
|
+
failureReason: terminalReason,
|
|
125
|
+
retainWorktree: !!anchorReason,
|
|
126
|
+
});
|
|
127
|
+
}
|
|
128
|
+
if (protection.terminalizationSafe === false) {
|
|
129
|
+
try {
|
|
130
|
+
dispatch().deferDispatchForWorktreeProtection(item.id, protection, {
|
|
131
|
+
reason: terminalReason,
|
|
132
|
+
resultSummary: terminalResultSummary,
|
|
133
|
+
completeOpts: terminalCompleteOpts,
|
|
134
|
+
});
|
|
135
|
+
} catch (deferErr) {
|
|
136
|
+
log('error', `Orphan cleanup could not persist deferred worktree protection for ${item.id}: ${deferErr.message}`);
|
|
137
|
+
}
|
|
138
|
+
log('error', `Orphan cleanup deferred terminalization for ${item.id}: ${protection.reason}`);
|
|
139
|
+
return { deferred: true, protection };
|
|
140
|
+
}
|
|
141
|
+
if (protection.allowRemoval !== false) {
|
|
142
|
+
gcRecoveredDispatchWorktree(item, project, config);
|
|
143
|
+
}
|
|
67
144
|
if (shared.isLiveCheckoutDispatchRecord(item, config)) {
|
|
68
145
|
try {
|
|
69
146
|
await restoreLiveCheckout({
|
|
@@ -78,7 +155,8 @@ async function _finalizeDeadDispatch(item, reason, failureClass, config, deps =
|
|
|
78
155
|
log('warn', 'live-checkout restore (orphan): ' + err.message);
|
|
79
156
|
}
|
|
80
157
|
}
|
|
81
|
-
complete(item.id, DISPATCH_RESULT.ERROR,
|
|
158
|
+
complete(item.id, DISPATCH_RESULT.ERROR, terminalReason, terminalResultSummary, terminalCompleteOpts);
|
|
159
|
+
return { deferred: false, protection };
|
|
82
160
|
}
|
|
83
161
|
|
|
84
162
|
// Lazy require for dispatch module (also circular via engine)
|
|
@@ -636,6 +714,35 @@ function readFileTail(filePath, maxBytes) {
|
|
|
636
714
|
}
|
|
637
715
|
}
|
|
638
716
|
|
|
717
|
+
// W-ms0vk9nv000z985b — read the live-output tail for a dispatch without
|
|
718
|
+
// throwing when the log is missing/unreadable. Mirrors the 64KB window the
|
|
719
|
+
// process-exit sentinel scan uses.
|
|
720
|
+
function liveLogTailForItem(liveLogPath) {
|
|
721
|
+
try { return readFileTail(liveLogPath, 65536); } catch { return ''; }
|
|
722
|
+
}
|
|
723
|
+
|
|
724
|
+
// W-ms0vk9nv000z985b — is there durable, trusted evidence that this dispatch
|
|
725
|
+
// already produced a completion report?
|
|
726
|
+
//
|
|
727
|
+
// Returns the parsed report only when it passes the SAME trust boundary the
|
|
728
|
+
// normal close path applies: valid shape + recognized status, and a nonce that
|
|
729
|
+
// matches the value minted for this spawn (carried across steering resumes).
|
|
730
|
+
// A report with a missing/mismatched nonce is untrusted and returns null, so a
|
|
731
|
+
// forged sidecar can never suppress a genuine timeout.
|
|
732
|
+
function readAcceptedCompletionReport(item, procInfo) {
|
|
733
|
+
const expectedNonce = procInfo?._completionNonce || null;
|
|
734
|
+
if (!expectedNonce) return null;
|
|
735
|
+
try {
|
|
736
|
+
const report = require('./lifecycle').parseCompletionReportFile(item);
|
|
737
|
+
if (!report) return null;
|
|
738
|
+
if (typeof report.nonce !== 'string' || report.nonce !== expectedNonce) return null;
|
|
739
|
+
return report;
|
|
740
|
+
} catch (e) {
|
|
741
|
+
log('warn', `resume-stall completion-report probe failed for ${item?.id}: ${e.message}`);
|
|
742
|
+
return null;
|
|
743
|
+
}
|
|
744
|
+
}
|
|
745
|
+
|
|
639
746
|
function parseProcessExitCode(logText) {
|
|
640
747
|
if (!logText) return null;
|
|
641
748
|
const exitPattern = /(?:^|\n)\[process-exit\]\s+(?:code=)?(-?\d+|spawn-failed)(?=\s|$)/g;
|
|
@@ -683,15 +790,7 @@ function checkTimeouts(config) {
|
|
|
683
790
|
const engineRestartGraceUntil = engine().engineRestartGraceUntil;
|
|
684
791
|
const engineRestartGraceExempt = engine().engineRestartGraceExempt;
|
|
685
792
|
const { completeDispatch } = dispatch();
|
|
686
|
-
|
|
687
|
-
// deliberately dropped from this destructure. The recovery path uses the
|
|
688
|
-
// sidecar (parseCompletionReportFile) as the sole structured-completion
|
|
689
|
-
// source. The stdout parser added no new signal on the timeout/exit-recovery
|
|
690
|
-
// path — only a disagreement risk between the sidecar and the stdout block.
|
|
691
|
-
// Removing this caller unblocks the completion-fallback-parsers deprecation
|
|
692
|
-
// sweep window in docs/deprecated.json (its `counter==0 ⇒ parser unused`
|
|
693
|
-
// claim is now logically sound).
|
|
694
|
-
const { runPostCompletionHooks, parseAgentOutput, parseCompletionReportFile, detectNonTerminalResultSummary } = require('./lifecycle');
|
|
793
|
+
const { runPostCompletionHooks } = require('./lifecycle');
|
|
695
794
|
|
|
696
795
|
const timeout = config.engine?.agentTimeout || ENGINE_DEFAULTS.agentTimeout;
|
|
697
796
|
const defaultStaleOrphanTimeout = config.engine?.heartbeatTimeout || ENGINE_DEFAULTS.heartbeatTimeout;
|
|
@@ -720,99 +819,249 @@ function checkTimeouts(config) {
|
|
|
720
819
|
const dispatchData = getDispatch();
|
|
721
820
|
const deadItems = [];
|
|
722
821
|
const legacyAnnotationClears = new Set();
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
return;
|
|
822
|
+
const finalizationPromises = [];
|
|
823
|
+
|
|
824
|
+
function completeFromOutput(item, liveLogPath, processExitCode, detectedLogText, hasProcess, expectedProcInfo = activeProcesses.get(item.id)) {
|
|
825
|
+
if (hasProcess && (
|
|
826
|
+
expectedProcInfo?._steeringAt
|
|
827
|
+
|| expectedProcInfo?._steeringMessage
|
|
828
|
+
|| expectedProcInfo?._steeringNoSession
|
|
829
|
+
)) {
|
|
830
|
+
return null;
|
|
733
831
|
}
|
|
734
|
-
|
|
735
|
-
if (
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
832
|
+
const finalizationOwner = 'timeout-output';
|
|
833
|
+
if (!shared.claimDispatchFinalization(item.id, finalizationOwner)) return;
|
|
834
|
+
|
|
835
|
+
const finalizationPromise = Promise.resolve().then(async () => {
|
|
836
|
+
if (resolveDispatchBranchPath(item, config) && item.meta?.branch) {
|
|
837
|
+
try {
|
|
838
|
+
await preserveDispatchBranch(item, config);
|
|
839
|
+
} catch (err) {
|
|
840
|
+
log('warn', `Output-completion branch preservation failed for ${item.id}: ${err.message}`);
|
|
841
|
+
}
|
|
743
842
|
}
|
|
744
|
-
}
|
|
745
|
-
const isSuccess = processExitCode === 0;
|
|
746
|
-
log('info', `Agent ${item.agent} (${item.id}) completed via output detection (exit code ${processExitCode}, ${isSuccess ? 'success' : 'error'})`);
|
|
747
843
|
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
const
|
|
761
|
-
|
|
762
|
-
const gateSummary = outputResultSummary || (!fullLogForHooks.includes('"type":') ? fullLogForHooks : '');
|
|
763
|
-
// P-a7b2c1d9: pass `null` for structuredCompletion. By the time
|
|
764
|
-
// checkTimeouts runs, the sidecar `completion.json` (parseCompletionReportFile)
|
|
765
|
-
// is the documented source of truth; the fenced ```completion stdout
|
|
766
|
-
// parser is intentionally bypassed here.
|
|
767
|
-
completionDetection = isSuccess
|
|
768
|
-
? detectNonTerminalResultSummary(gateSummary, null, parseCompletionReportFile(item), { detectPhantom: true })
|
|
769
|
-
: null;
|
|
770
|
-
} catch (e) { log('warn', 'completion summary gate: ' + e.message); }
|
|
771
|
-
|
|
772
|
-
const _completedAsError = completionDetection || !isSuccess;
|
|
773
|
-
completeDispatch(item.id, completionDetection ? DISPATCH_RESULT.ERROR : (isSuccess ? DISPATCH_RESULT.SUCCESS : DISPATCH_RESULT.ERROR),
|
|
774
|
-
completionDetection ? completionDetection.reason : (isSuccess ? 'Completed (detected from output)' : `Exited with code ${processExitCode} (detected from output)`),
|
|
775
|
-
outputResultSummary,
|
|
776
|
-
completionDetection ? { processWorkItemFailure: false } : {});
|
|
777
|
-
|
|
778
|
-
// PL-live-checkout-reliability-hardening — restore the operator's checkout
|
|
779
|
-
// for a live-mode dispatch that finished AFTER an engine restart. This path
|
|
780
|
-
// (output-detected completion of a re-attached process) has no in-memory
|
|
781
|
-
// onAgentClose closure, so without this the tree is stranded on the agent
|
|
782
|
-
// branch. Explicit dispatch provenance plus current config are the live-mode
|
|
783
|
-
// signal; worktree-mode records fail closed.
|
|
784
|
-
if (shared.isLiveCheckoutDispatchRecord(item, config)) {
|
|
844
|
+
if (!(getDispatch().active || []).some(active => active.id === item.id)) return;
|
|
845
|
+
if (hasProcess) {
|
|
846
|
+
const currentProcInfo = activeProcesses.get(item.id);
|
|
847
|
+
if (currentProcInfo !== expectedProcInfo ||
|
|
848
|
+
currentProcInfo?._steeringAt ||
|
|
849
|
+
currentProcInfo?._steeringMessage ||
|
|
850
|
+
currentProcInfo?._steeringNoSession) {
|
|
851
|
+
log('info', `${item.id}: output-detected completion yielded to the close/steering owner after branch preservation`);
|
|
852
|
+
return;
|
|
853
|
+
}
|
|
854
|
+
}
|
|
855
|
+
|
|
856
|
+
const fullLogForHooks = safeRead(liveLogPath) || detectedLogText;
|
|
857
|
+
const outputLogPath = path.join(AGENTS_DIR, item.agent, 'output.log');
|
|
785
858
|
try {
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
859
|
+
const { text } = shared.parseStreamJsonOutput(fullLogForHooks);
|
|
860
|
+
safeWrite(outputLogPath, `# Output for dispatch ${item.id}\n# Exit code: ${processExitCode}\n# Completed: ${ts()}\n# Detected via output scan\n\n## Result\n${text || '(no text)'}\n`);
|
|
861
|
+
} catch (e) { log('warn', 'parse output result: ' + e.message); }
|
|
862
|
+
|
|
863
|
+
const expectedNonce = expectedProcInfo?._completionNonce || null;
|
|
864
|
+
const completionNonceRequired = config?.engine?.completionNonceRequired ?? ENGINE_DEFAULTS.completionNonceRequired;
|
|
865
|
+
const type = item.type || item.workType || item.meta?.item?.type;
|
|
866
|
+
const wiMeta = item.meta?.item?.meta || {};
|
|
867
|
+
const keepProcessesRequested = !!item.meta?.keep_processes || !!wiMeta.keep_processes;
|
|
868
|
+
const managedSpawnRequested = !!item.meta?.managed_spawn || !!wiMeta.managed_spawn;
|
|
869
|
+
const preFinalizationFailure = keepProcessesRequested || managedSpawnRequested
|
|
870
|
+
? {
|
|
871
|
+
reason: 'Output recovery cannot safely evaluate keep_processes/managed_spawn acceptance; retrying through the normal close path on the next attempt',
|
|
872
|
+
}
|
|
873
|
+
: null;
|
|
874
|
+
if (managedSpawnRequested) {
|
|
875
|
+
const staleManagedSidecar = path.join(AGENTS_DIR, item.agent, 'managed-spawn.json');
|
|
876
|
+
try { fs.unlinkSync(staleManagedSidecar); } catch (e) {
|
|
877
|
+
if (e.code !== 'ENOENT') log('warn', `Output recovery could not discard stale managed-spawn sidecar for ${item.id}: ${e.message}`);
|
|
878
|
+
}
|
|
879
|
+
}
|
|
880
|
+
const hookResult = await runPostCompletionHooks(
|
|
881
|
+
item,
|
|
882
|
+
item.agent,
|
|
883
|
+
processExitCode,
|
|
884
|
+
fullLogForHooks,
|
|
885
|
+
config,
|
|
886
|
+
{
|
|
887
|
+
detectPhantom: true,
|
|
888
|
+
expectedNonce,
|
|
889
|
+
completionNonceRequired,
|
|
890
|
+
completionReportRequired: !shared.READ_ONLY_ROOT_TASK_TYPES.has(type),
|
|
891
|
+
deferWorktreeRemoval: true,
|
|
892
|
+
preFinalizationFailure,
|
|
893
|
+
},
|
|
894
|
+
) || {};
|
|
895
|
+
|
|
896
|
+
if (!(getDispatch().active || []).some(active => active.id === item.id)) return;
|
|
897
|
+
if (hasProcess && activeProcesses.get(item.id) !== expectedProcInfo) {
|
|
898
|
+
log('info', `${item.id}: output-detected completion yielded to the close owner after lifecycle validation`);
|
|
899
|
+
return;
|
|
900
|
+
}
|
|
796
901
|
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
902
|
+
const {
|
|
903
|
+
resultSummary = '',
|
|
904
|
+
autoRecovered = false,
|
|
905
|
+
trustedCompletionSuccess = false,
|
|
906
|
+
completionContractFailure = null,
|
|
907
|
+
completionReportFailure = null,
|
|
908
|
+
preFinalizationFailure: lifecyclePreFinalizationFailure = null,
|
|
909
|
+
structuredCompletion = null,
|
|
910
|
+
agentReportedFailure = false,
|
|
911
|
+
agentRetryable,
|
|
912
|
+
nonceMismatch = null,
|
|
913
|
+
benignNoop = false,
|
|
914
|
+
} = hookResult;
|
|
915
|
+
const hardContractFail = completionContractFailure?.severity === 'hard'
|
|
916
|
+
|| completionContractFailure?.nonTerminal === true;
|
|
917
|
+
const nonceFail = nonceMismatch?.severity === 'hard';
|
|
918
|
+
const completionReportFail = !!completionReportFailure;
|
|
919
|
+
const preFinalizationFail = !!lifecyclePreFinalizationFailure;
|
|
920
|
+
const effectiveResult = shared.resolveTerminalDispatchResult({
|
|
921
|
+
code: processExitCode,
|
|
922
|
+
agentReportedFailure,
|
|
923
|
+
autoRecovered,
|
|
924
|
+
benignNoop,
|
|
925
|
+
trustedCompletionSuccess,
|
|
926
|
+
hardContractFail,
|
|
927
|
+
nonceFail,
|
|
928
|
+
completionReportFail,
|
|
929
|
+
managedSpawnAcceptanceFail: preFinalizationFail,
|
|
930
|
+
});
|
|
931
|
+
|
|
932
|
+
let reason = '';
|
|
933
|
+
if (nonceFail) {
|
|
934
|
+
reason = nonceMismatch.reason || 'completion nonce mismatch';
|
|
935
|
+
} else if (hardContractFail) {
|
|
936
|
+
reason = completionContractFailure.reason || 'completion contract failed';
|
|
937
|
+
} else if (completionReportFail) {
|
|
938
|
+
reason = completionReportFailure.reason;
|
|
939
|
+
} else if (preFinalizationFail) {
|
|
940
|
+
reason = lifecyclePreFinalizationFailure.reason;
|
|
941
|
+
} else if (agentReportedFailure) {
|
|
942
|
+
reason = String(
|
|
943
|
+
structuredCompletion?.summary
|
|
944
|
+
|| structuredCompletion?.failure_class
|
|
945
|
+
|| 'Agent reported failure',
|
|
946
|
+
).slice(0, 300);
|
|
947
|
+
} else if (effectiveResult === DISPATCH_RESULT.ERROR) {
|
|
948
|
+
reason = `Exited with code ${processExitCode} (detected from output)`;
|
|
949
|
+
} else {
|
|
950
|
+
reason = 'Completed (detected from output)';
|
|
951
|
+
}
|
|
952
|
+
|
|
953
|
+
const completionOpts = {
|
|
954
|
+
...(structuredCompletion?._path ? { completionReportPath: structuredCompletion._path } : {}),
|
|
955
|
+
...(structuredCompletion ? { structuredCompletion } : {}),
|
|
956
|
+
};
|
|
957
|
+
let completeOpts = completionOpts;
|
|
958
|
+
if (nonceFail) {
|
|
959
|
+
completeOpts = { ...completionOpts, failureClass: nonceMismatch.failureClass, agentRetryable: false };
|
|
960
|
+
} else if (completionReportFail) {
|
|
961
|
+
completeOpts = {
|
|
962
|
+
...completionOpts,
|
|
963
|
+
failureClass: completionReportFailure.failureClass,
|
|
964
|
+
agentRetryable: true,
|
|
965
|
+
...(hardContractFail ? { processWorkItemFailure: false } : {}),
|
|
966
|
+
};
|
|
967
|
+
} else if (preFinalizationFail) {
|
|
968
|
+
completeOpts = {
|
|
969
|
+
...completionOpts,
|
|
970
|
+
agentRetryable: true,
|
|
971
|
+
...(hardContractFail ? { processWorkItemFailure: false } : {}),
|
|
972
|
+
};
|
|
973
|
+
} else if (hardContractFail) {
|
|
974
|
+
completeOpts = { ...completionOpts, processWorkItemFailure: false };
|
|
975
|
+
} else if (effectiveResult === DISPATCH_RESULT.ERROR) {
|
|
976
|
+
completeOpts = {
|
|
977
|
+
...completionOpts,
|
|
978
|
+
...(typeof agentRetryable === 'boolean' ? { agentRetryable } : {}),
|
|
979
|
+
...(structuredCompletion?.failure_class ? { failureClass: structuredCompletion.failure_class } : {}),
|
|
980
|
+
};
|
|
981
|
+
}
|
|
982
|
+
|
|
983
|
+
const project = resolveDispatchProject(item, config);
|
|
984
|
+
const persistedAnchorReason = getWorktreeProcessAnchorReason(item);
|
|
985
|
+
const worktreeAnchored = !!persistedAnchorReason;
|
|
986
|
+
let failedWorktreeProtection = {
|
|
987
|
+
allowRemoval: !worktreeAnchored,
|
|
988
|
+
preserved: false,
|
|
989
|
+
reason: persistedAnchorReason || 'not-checked',
|
|
990
|
+
};
|
|
991
|
+
|
|
992
|
+
if (
|
|
993
|
+
effectiveResult === DISPATCH_RESULT.ERROR
|
|
994
|
+
&& item.worktreePath
|
|
995
|
+
&& item.meta?.branch
|
|
996
|
+
&& project.localPath
|
|
997
|
+
) {
|
|
998
|
+
failedWorktreeProtection = await engine()._preserveFailedDispatchWorktree({
|
|
999
|
+
effectiveResult,
|
|
1000
|
+
rootDir: project.localPath,
|
|
1001
|
+
worktreePath: item.worktreePath,
|
|
1002
|
+
branchName: item.meta.branch,
|
|
1003
|
+
dispatchId: item.id,
|
|
1004
|
+
mainBranch: shared.resolveMainBranch(project.localPath, project.mainBranch),
|
|
1005
|
+
project,
|
|
1006
|
+
gitOpts: { env: shared.gitEnv() },
|
|
1007
|
+
failureReason: reason,
|
|
1008
|
+
retainWorktree: worktreeAnchored,
|
|
1009
|
+
});
|
|
1010
|
+
}
|
|
1011
|
+
|
|
1012
|
+
if (failedWorktreeProtection.terminalizationSafe === false) {
|
|
1013
|
+
try {
|
|
1014
|
+
dispatch().deferDispatchForWorktreeProtection(item.id, failedWorktreeProtection, {
|
|
1015
|
+
reason,
|
|
1016
|
+
resultSummary,
|
|
1017
|
+
completeOpts,
|
|
1018
|
+
});
|
|
1019
|
+
} catch (deferErr) {
|
|
1020
|
+
log('error', `Output-completion could not persist deferred worktree protection for ${item.id}: ${deferErr.message}`);
|
|
1021
|
+
}
|
|
1022
|
+
dispatch().updateAgentStatus(item.id, AGENT_STATUS.FAILED,
|
|
1023
|
+
'Completion deferred until failed worktree has durable recovery protection');
|
|
1024
|
+
log('error', `Output-completion deferred terminalization for ${item.id}: ${failedWorktreeProtection.reason}`);
|
|
1025
|
+
return;
|
|
1026
|
+
}
|
|
1027
|
+
|
|
1028
|
+
if (failedWorktreeProtection.allowRemoval !== false) {
|
|
1029
|
+
gcRecoveredDispatchWorktree(item, project, config);
|
|
1030
|
+
}
|
|
1031
|
+
|
|
1032
|
+
const workerState = shared.resolveTerminalWorkerState({
|
|
1033
|
+
code: processExitCode,
|
|
1034
|
+
effectiveResult,
|
|
1035
|
+
errorReason: reason,
|
|
1036
|
+
});
|
|
1037
|
+
dispatch().updateAgentStatus(item.id, workerState.status, workerState.detail);
|
|
1038
|
+
log('info', `Agent ${item.agent} (${item.id}) completed via output detection (exit code ${processExitCode}, ${effectiveResult})`);
|
|
1039
|
+
completeDispatch(item.id, effectiveResult, reason, resultSummary, completeOpts);
|
|
1040
|
+
|
|
1041
|
+
if (shared.isLiveCheckoutDispatchRecord(item, config)) {
|
|
1042
|
+
try {
|
|
1043
|
+
await require('./live-checkout').maybeRestoreLiveCheckoutFromRecord({
|
|
1044
|
+
item,
|
|
1045
|
+
config,
|
|
1046
|
+
isTerminalFailure: effectiveResult === DISPATCH_RESULT.ERROR,
|
|
1047
|
+
resultLabel: effectiveResult,
|
|
1048
|
+
log: (lvl, msg) => log(lvl, msg),
|
|
1049
|
+
writeInboxAlert: dispatch().writeInboxAlert,
|
|
1050
|
+
});
|
|
1051
|
+
} catch (e) { log('warn', 'live-checkout restore (output completion): ' + e.message); }
|
|
1052
|
+
}
|
|
1053
|
+
|
|
1054
|
+
if (hasProcess) {
|
|
1055
|
+
killTrackedProcess(activeProcesses.get(item.id));
|
|
1056
|
+
activeProcesses.delete(item.id);
|
|
1057
|
+
}
|
|
1058
|
+
}).catch(err => {
|
|
1059
|
+
log('warn', `Output-completion finalization failed for ${item.id}: ${err.message}`);
|
|
1060
|
+
}).finally(() => {
|
|
1061
|
+
shared.releaseDispatchFinalization(item.id, finalizationOwner);
|
|
1062
|
+
});
|
|
1063
|
+
finalizationPromises.push(finalizationPromise);
|
|
1064
|
+
return finalizationPromise;
|
|
816
1065
|
}
|
|
817
1066
|
|
|
818
1067
|
for (const item of (dispatchData.active || [])) {
|
|
@@ -839,6 +1088,22 @@ function checkTimeouts(config) {
|
|
|
839
1088
|
const silentMs = Date.now() - lastActivity;
|
|
840
1089
|
const silentSec = Math.round(silentMs / 1000);
|
|
841
1090
|
|
|
1091
|
+
const deferredProtection = item._worktreeProtectionDeferred;
|
|
1092
|
+
if (deferredProtection && !processAlive) {
|
|
1093
|
+
const anchorReason = getWorktreeProcessAnchorReason(item);
|
|
1094
|
+
const retryAt = Date.parse(deferredProtection.retryAfter || '');
|
|
1095
|
+
if (anchorReason || (Number.isFinite(retryAt) && Date.now() < retryAt)) continue;
|
|
1096
|
+
const terminal = deferredProtection.terminal || {};
|
|
1097
|
+
log('warn', `Retrying deferred failed-worktree preservation for ${item.id} after process anchor release`);
|
|
1098
|
+
deadItems.push({
|
|
1099
|
+
item,
|
|
1100
|
+
reason: terminal.reason || 'Failed worktree lacks durable recovery protection',
|
|
1101
|
+
failureClass: terminal.completeOpts?.failureClass,
|
|
1102
|
+
});
|
|
1103
|
+
activeProcesses.delete(item.id);
|
|
1104
|
+
continue;
|
|
1105
|
+
}
|
|
1106
|
+
|
|
842
1107
|
// Check if the agent actually completed by looking for the [process-exit] sentinel.
|
|
843
1108
|
//
|
|
844
1109
|
// The sentinel is written synchronously by spawn-agent.js's proc.on('close') handler
|
|
@@ -918,6 +1183,38 @@ function checkTimeouts(config) {
|
|
|
918
1183
|
const resumeSilentMs = Date.now() - resumeHeartbeatAt;
|
|
919
1184
|
if (resumeSilentMs > runtimeResumeHeartbeatTimeout) {
|
|
920
1185
|
const resumeSilentSec = Math.round(resumeSilentMs / 1000);
|
|
1186
|
+
// W-ms0vk9nv000z985b — durable evidence beats a stale heartbeat.
|
|
1187
|
+
//
|
|
1188
|
+
// The resume watchdog exists for one hypothesis: the resumed runtime
|
|
1189
|
+
// is wedged and never came back to life. A nonce-valid completion
|
|
1190
|
+
// report on disk falsifies that hypothesis outright — the agent ran,
|
|
1191
|
+
// finished its work (pushed commits, opened/updated the PR) and
|
|
1192
|
+
// recorded the outcome. Marking that dispatch TIMEOUT is a FALSE
|
|
1193
|
+
// failure: it discards a real success, re-queues finished work and
|
|
1194
|
+
// (for fix dispatches) re-runs against an already-resolved PR.
|
|
1195
|
+
//
|
|
1196
|
+
// The nonce is minted per spawn and carried across steering resumes,
|
|
1197
|
+
// so a matching report provably came from THIS dispatch. Terminalize
|
|
1198
|
+
// from the report through the same completeFromOutput path the
|
|
1199
|
+
// process-exit sentinel uses — full lifecycle validation, PR
|
|
1200
|
+
// attachment, and worktree protection all still apply, so a report
|
|
1201
|
+
// claiming failure still fails. Nothing here weakens the report
|
|
1202
|
+
// contract; it only stops a watchdog from overwriting it.
|
|
1203
|
+
const acceptedReport = readAcceptedCompletionReport(item, procInfo);
|
|
1204
|
+
if (acceptedReport) {
|
|
1205
|
+
log('warn', `Runtime resume produced no output heartbeat for ${resumeSilentSec}s for ${item.agent} (${item.id}), but an accepted completion report is present (status=${acceptedReport.status}) — terminalizing from the report instead of timing out`);
|
|
1206
|
+
try { fs.appendFileSync(liveLogPath, `\n[runtime-resume-completion] Resume heartbeat stalled after ${resumeSilentSec}s, but a valid completion report was found. Completing from the report instead of failing as a timeout.\n`); } catch { /* optional */ }
|
|
1207
|
+
procInfo._runtimeResumeAwaitingFirstOutput = false;
|
|
1208
|
+
// hasProcess=false: activeProcesses is torn down immediately below,
|
|
1209
|
+
// so the owner-yield re-checks inside completeFromOutput must not
|
|
1210
|
+
// race against our own cleanup. procInfo is still passed as the
|
|
1211
|
+
// expected proc info so the per-spawn nonce is available to the
|
|
1212
|
+
// lifecycle hooks.
|
|
1213
|
+
completeFromOutput(item, liveLogPath, 0, liveLogTailForItem(liveLogPath), false, procInfo);
|
|
1214
|
+
activeProcesses.delete(item.id);
|
|
1215
|
+
killTrackedProcess(procInfo, { graceMs: 5000 });
|
|
1216
|
+
continue;
|
|
1217
|
+
}
|
|
921
1218
|
const reason = `Runtime resume stalled — no output heartbeat for ${resumeSilentSec}s`;
|
|
922
1219
|
log('warn', `Runtime resume stalled: ${item.agent} (${item.id}) — no output heartbeat for ${resumeSilentSec}s; killing and retrying fresh`);
|
|
923
1220
|
dispatch().updateAgentStatus(item.id, AGENT_STATUS.TIMED_OUT, reason);
|
|
@@ -1034,18 +1331,12 @@ function checkTimeouts(config) {
|
|
|
1034
1331
|
|
|
1035
1332
|
// Clean up dead items
|
|
1036
1333
|
for (const { item, reason, failureClass } of deadItems) {
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
_finalizeDeadDispatch(item, reason, failureClass, config, { complete: completeDispatch })
|
|
1044
|
-
.catch(err => log('warn', `Orphan cleanup failed for ${item.id}: ${err.message}`))
|
|
1045
|
-
.finally(() => _pendingBranchPreservations.delete(item.id));
|
|
1046
|
-
continue;
|
|
1047
|
-
}
|
|
1048
|
-
completeDispatch(item.id, DISPATCH_RESULT.ERROR, reason, '', failureClass ? { failureClass } : {});
|
|
1334
|
+
const finalizationOwner = 'timeout-orphan';
|
|
1335
|
+
if (!shared.claimDispatchFinalization(item.id, finalizationOwner)) continue;
|
|
1336
|
+
const finalizationPromise = _finalizeDeadDispatch(item, reason, failureClass, config, { complete: completeDispatch })
|
|
1337
|
+
.catch(err => log('warn', `Orphan cleanup failed for ${item.id}: ${err.message}`))
|
|
1338
|
+
.finally(() => shared.releaseDispatchFinalization(item.id, finalizationOwner));
|
|
1339
|
+
finalizationPromises.push(finalizationPromise);
|
|
1049
1340
|
}
|
|
1050
1341
|
|
|
1051
1342
|
// Clear legacy blocking-tool annotations; process liveness no longer depends on tool parsing.
|
|
@@ -1111,6 +1402,7 @@ function checkTimeouts(config) {
|
|
|
1111
1402
|
return items;
|
|
1112
1403
|
}, { defaultValue: [], skipWriteIfUnchanged: true });
|
|
1113
1404
|
}
|
|
1405
|
+
return Promise.allSettled(finalizationPromises);
|
|
1114
1406
|
}
|
|
1115
1407
|
|
|
1116
1408
|
// ─── Exports ─────────────────────────────────────────────────────────────────
|
|
@@ -1123,6 +1415,7 @@ module.exports = {
|
|
|
1123
1415
|
isOsPidAliveForDispatch,
|
|
1124
1416
|
parseProcessExitCode, terminalResultIndicatesError, parseTerminalResultFallbackExitCode, // exported for testing
|
|
1125
1417
|
readFileTail, runtimeSupportsMidRunSessionId, // exported for testing
|
|
1418
|
+
readAcceptedCompletionReport, liveLogTailForItem, // exported for testing (W-ms0vk9nv000z985b)
|
|
1126
1419
|
// exported for testing
|
|
1127
1420
|
rememberDeferredSteering, checkDeferredStranded, _runSteeringKillLadder, _collectDescendantPids,
|
|
1128
1421
|
_resetSteeringStoreCacheForTest, _setSteeringStoreForTest,
|