@mcrescenzo/opencode-workflows 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +14 -0
- package/CODE_OF_CONDUCT.md +134 -0
- package/CONTRIBUTING.md +95 -0
- package/LICENSE +21 -0
- package/README.md +746 -0
- package/SECURITY.md +38 -0
- package/commands/repo-bughunt.md +94 -0
- package/commands/repo-review.md +148 -0
- package/commands/workflow-live-gates-release-check.md +143 -0
- package/docs/workflow-plugin.md +400 -0
- package/opencode-workflows.js +5 -0
- package/package.json +86 -0
- package/skills/opencode-workflow-authoring/SKILL.md +180 -0
- package/skills/repo-review-command-protocol/SKILL.md +56 -0
- package/skills/workflow-model-tiering/SKILL.md +57 -0
- package/skills/workflow-plan-review/SKILL.md +91 -0
- package/workflow-kernel/approval-hashing.js +39 -0
- package/workflow-kernel/async-util.js +33 -0
- package/workflow-kernel/audited-shell-policy.js +200 -0
- package/workflow-kernel/authority-policy.js +670 -0
- package/workflow-kernel/budget-accounting.js +142 -0
- package/workflow-kernel/capability-adapter.js +753 -0
- package/workflow-kernel/child-agent-runner.js +1264 -0
- package/workflow-kernel/constants.js +117 -0
- package/workflow-kernel/diagnostics.js +152 -0
- package/workflow-kernel/drain-runtime.js +421 -0
- package/workflow-kernel/errors.js +181 -0
- package/workflow-kernel/event-journal.js +487 -0
- package/workflow-kernel/extension-registry.js +144 -0
- package/workflow-kernel/free-text-redactor.js +91 -0
- package/workflow-kernel/gate-shapes.js +82 -0
- package/workflow-kernel/git-util.js +45 -0
- package/workflow-kernel/index.js +72 -0
- package/workflow-kernel/integration-mode.js +155 -0
- package/workflow-kernel/lane-effort-policy.js +134 -0
- package/workflow-kernel/lifecycle-control.js +608 -0
- package/workflow-kernel/live-gate-probes.js +916 -0
- package/workflow-kernel/notification-toast-cards.js +393 -0
- package/workflow-kernel/notification-toast-policy.js +179 -0
- package/workflow-kernel/notification-toast-scope.js +100 -0
- package/workflow-kernel/notification-toast.js +287 -0
- package/workflow-kernel/path-policy.js +219 -0
- package/workflow-kernel/result-readback.js +106 -0
- package/workflow-kernel/role-template-loading.js +606 -0
- package/workflow-kernel/run-context.js +139 -0
- package/workflow-kernel/run-observability.js +43 -0
- package/workflow-kernel/run-store-fs.js +231 -0
- package/workflow-kernel/run-store-locks.js +180 -0
- package/workflow-kernel/run-store-projections.js +421 -0
- package/workflow-kernel/run-store-rehydrate.js +68 -0
- package/workflow-kernel/run-store-state.js +147 -0
- package/workflow-kernel/run-store-status-format.js +1154 -0
- package/workflow-kernel/run-store-status.js +107 -0
- package/workflow-kernel/sandbox-executor.js +1131 -0
- package/workflow-kernel/session-access.js +56 -0
- package/workflow-kernel/structured-output.js +143 -0
- package/workflow-kernel/test-fix-drain-adapter.js +119 -0
- package/workflow-kernel/text-json.js +136 -0
- package/workflow-kernel/workflow-plugin.js +3017 -0
- package/workflow-kernel/workflow-source.js +444 -0
- package/workflow-kernel/worktree-adapter.js +256 -0
- package/workflow-kernel/worktree-git.js +147 -0
- package/workflows/repo-bughunt.js +362 -0
- package/workflows/repo-cleanup.js +383 -0
- package/workflows/repo-complexity.js +404 -0
- package/workflows/repo-deps.js +457 -0
- package/workflows/repo-modernize.js +395 -0
- package/workflows/repo-perf.js +394 -0
- package/workflows/repo-review.js +831 -0
- package/workflows/repo-security-audit.js +466 -0
- package/workflows/repo-test-gaps.js +377 -0
|
@@ -0,0 +1,1154 @@
|
|
|
1
|
+
// Concern (5): status formatting for external consumption. Reads a run's durable state.json
|
|
2
|
+
// (reconciling stale-active/interrupted status on the way), then projects it into compact /
|
|
3
|
+
// full / result status shapes, summarizes run lists, and serves the workflow_status /
|
|
4
|
+
// workflow_reconcile / workflow_cleanup tool surfaces. Extracted from run-store-status.js
|
|
5
|
+
// (opencode-workflows-nbp).
|
|
6
|
+
//
|
|
7
|
+
// This concern is read-mostly: it reads the persisted state snapshot plus lock/lifecycle/
|
|
8
|
+
// salvage projections and the in-memory `runs` registry, and (in the reconcile/cleanup paths)
|
|
9
|
+
// rewrites state.json/closeout.json and deletes eligible run dirs. See
|
|
10
|
+
// {@link import("./run-context.js").RunContext}.
|
|
11
|
+
|
|
12
|
+
import fs from "node:fs/promises";
|
|
13
|
+
import path from "node:path";
|
|
14
|
+
import {
|
|
15
|
+
ACTIVE_STATUSES,
|
|
16
|
+
AMBIGUOUS_EDIT_STATUSES,
|
|
17
|
+
DEFAULT_CHILD_PROMPT_TIMEOUT_MS,
|
|
18
|
+
DEFAULT_KEEP_RUNS,
|
|
19
|
+
DEFAULT_WORKFLOW_EVENTS_LIMIT,
|
|
20
|
+
DURABLE_STATE_VERSION,
|
|
21
|
+
LANE_OUTCOMES,
|
|
22
|
+
MAX_WORKFLOW_EVENTS_LIMIT,
|
|
23
|
+
MAX_RESULT_READ_FILE_BYTES,
|
|
24
|
+
MAX_STATUS_STRING_CHARS,
|
|
25
|
+
} from "./constants.js";
|
|
26
|
+
import { extractTextFromError, redactValue, truncateText } from "./text-json.js";
|
|
27
|
+
import { redactFreeTextSecrets } from "./free-text-redactor.js";
|
|
28
|
+
import { resultReadbackProjection } from "./result-readback.js";
|
|
29
|
+
import { WorkflowAuthorityError } from "./errors.js";
|
|
30
|
+
import { addTokens, zeroTokens } from "./budget-accounting.js";
|
|
31
|
+
import { AD_HOC_AUTHORITY_PROFILE, assertWriteWorkflowAllowed } from "./authority-policy.js";
|
|
32
|
+
import {
|
|
33
|
+
assertContainedRealPath,
|
|
34
|
+
assertContainedRunDir,
|
|
35
|
+
assertSafeRunId,
|
|
36
|
+
pathExists,
|
|
37
|
+
processAppearsAlive,
|
|
38
|
+
readJsonFile,
|
|
39
|
+
runDirForRoot,
|
|
40
|
+
runRoots,
|
|
41
|
+
runs,
|
|
42
|
+
writeJsonAtomic,
|
|
43
|
+
} from "./run-store-fs.js";
|
|
44
|
+
import {
|
|
45
|
+
acquireWorkflowLock,
|
|
46
|
+
cleanupLockPath,
|
|
47
|
+
clearStaleRunLocks,
|
|
48
|
+
readLifecycleRequests,
|
|
49
|
+
runLocksForEntry,
|
|
50
|
+
} from "./run-store-locks.js";
|
|
51
|
+
import { attachSalvageCandidates, recoverySummary } from "./run-store-projections.js";
|
|
52
|
+
import { computeLastProgressAt } from "./run-store-state.js";
|
|
53
|
+
import { createWorktreeAdapter } from "./worktree-adapter.js";
|
|
54
|
+
|
|
55
|
+
async function readRunEntry(root, id, options = {}) {
|
|
56
|
+
const dir = runDirForRoot(root, id);
|
|
57
|
+
const statePath = path.join(dir, "state.json");
|
|
58
|
+
const base = { id, root, dir };
|
|
59
|
+
try {
|
|
60
|
+
await assertContainedRunDir(root, dir);
|
|
61
|
+
const raw = await fs.readFile(statePath, "utf8");
|
|
62
|
+
const state = JSON.parse(raw);
|
|
63
|
+
const runId = typeof state.id === "string" ? state.id : id;
|
|
64
|
+
const entryBase = { ...base, id: runId, status: state.status ?? "partial", kind: "valid", state };
|
|
65
|
+
const staleLocksCleared = options.clearStaleLocks === true ? await clearStaleRunLocks(entryBase) : [];
|
|
66
|
+
const locks = await runLocksForEntry(entryBase);
|
|
67
|
+
const lifecycleRequests = await readLifecycleRequests(dir);
|
|
68
|
+
if (Object.keys(locks).length > 0) state.locks = locks;
|
|
69
|
+
if (Object.keys(lifecycleRequests).length > 0) state.lifecycleRequests = lifecycleRequests;
|
|
70
|
+
if (staleLocksCleared.length > 0) state.staleLocksCleared = staleLocksCleared;
|
|
71
|
+
if (ACTIVE_STATUSES.has(state.status) && !runs.has(runId)) {
|
|
72
|
+
if (await processAppearsAlive(state.process)) {
|
|
73
|
+
state.status = "active-unknown";
|
|
74
|
+
state.error = state.error ?? "Workflow appears active in another OpenCode process";
|
|
75
|
+
return { ...base, id: runId, status: state.status, kind: "valid", state };
|
|
76
|
+
}
|
|
77
|
+
if (options.reconcile !== true) {
|
|
78
|
+
state.status = "stale-active";
|
|
79
|
+
state.error = state.error ?? "Workflow appears stale; run workflow_reconcile to persist interrupted state";
|
|
80
|
+
return await attachSalvageCandidates({ ...base, id: runId, status: state.status, kind: "valid", state });
|
|
81
|
+
}
|
|
82
|
+
state.status = "interrupted";
|
|
83
|
+
state.finishedAt = state.finishedAt ?? new Date().toISOString();
|
|
84
|
+
state.error = state.error ?? "Workflow was active when no owning OpenCode process was found";
|
|
85
|
+
state.recovery = await recoverySummary(dir, state);
|
|
86
|
+
state.durability = {
|
|
87
|
+
...(state.durability ?? {}),
|
|
88
|
+
stateVersion: state.stateVersion ?? DURABLE_STATE_VERSION,
|
|
89
|
+
ledgers: state.recovery.ledgers,
|
|
90
|
+
recovery: state.recovery,
|
|
91
|
+
};
|
|
92
|
+
try {
|
|
93
|
+
await writeJsonAtomic(statePath, state);
|
|
94
|
+
await writeJsonAtomic(path.join(dir, "closeout.json"), {
|
|
95
|
+
runId,
|
|
96
|
+
status: state.status,
|
|
97
|
+
finishedAt: state.finishedAt,
|
|
98
|
+
errorSummary: state.error,
|
|
99
|
+
durability: state.durability,
|
|
100
|
+
});
|
|
101
|
+
} catch {
|
|
102
|
+
// Status should still report the reconciliation even if persisting it fails.
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
return await attachSalvageCandidates({ ...base, id: runId, status: state.status ?? "partial", kind: "valid", state });
|
|
106
|
+
} catch (error) {
|
|
107
|
+
if (error.code === "ENOENT") return { ...base, status: "partial", kind: "partial", error: "Missing state.json" };
|
|
108
|
+
return { ...base, status: "corrupt", kind: "corrupt", error: extractTextFromError(error) };
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
// Bounded operator next-step hints (opencode-workflows-ux.4). Derived purely from the run's
|
|
113
|
+
// status, a few safe booleans (presence of a result file / salvage candidates / apply hashes),
|
|
114
|
+
// bounded lane failure classes, and the already-validated run id. These strings intentionally
|
|
115
|
+
// NEVER interpolate prompts, tool output, raw lane results, meta, error text, or filesystem paths,
|
|
116
|
+
// so they cannot leak run evidence or secrets: they are static recommended-command templates.
|
|
117
|
+
// The redaction/bounds coverage (opencode-workflows-ux.6) governs the evidence fields;
|
|
118
|
+
// nextActions adds none.
|
|
119
|
+
const MAX_NEXT_ACTIONS = 5;
|
|
120
|
+
const MAX_DIAGNOSTIC_ITEMS = 50;
|
|
121
|
+
|
|
122
|
+
function nonEmptyString(value) {
|
|
123
|
+
return typeof value === "string" && value.trim().length > 0;
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
function hasApplyableDiffPlan(state) {
|
|
127
|
+
const plan = state?.editPlan;
|
|
128
|
+
if (!plan || typeof plan !== "object") return false;
|
|
129
|
+
if (!nonEmptyString(state.sourceHash)) return false;
|
|
130
|
+
if (!nonEmptyString(plan.baseCommit)) return false;
|
|
131
|
+
if (!nonEmptyString(plan.diffPlanHash)) return false;
|
|
132
|
+
if (!nonEmptyString(plan.domainMutationHash)) return false;
|
|
133
|
+
const patchCount = Number.isFinite(plan.patchCount)
|
|
134
|
+
? plan.patchCount
|
|
135
|
+
: Array.isArray(plan.patches)
|
|
136
|
+
? plan.patches.length
|
|
137
|
+
: null;
|
|
138
|
+
return patchCount === null || patchCount > 0;
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
const STRUCTURED_LANE_FAILURE_PATTERN = /\b(structured|schema|json(?:\s+parse)?|OutputFormatJsonSchema)\b/i;
|
|
142
|
+
|
|
143
|
+
function failedRunRecoveryForState(state) {
|
|
144
|
+
const failures = laneFailureSummaries(state);
|
|
145
|
+
if (failures.length === 0) return { kind: "unknown", retryable: true };
|
|
146
|
+
const retryableFailures = failures.filter((failure) => failure.retryable === true || failure.failureClass === "transient" || failure.failureClass === "transient_exhausted");
|
|
147
|
+
const terminalFailures = failures.filter((failure) => failure.retryable === false || failure.failureClass === "terminal");
|
|
148
|
+
const terminalStructured = terminalFailures.some((failure) => STRUCTURED_LANE_FAILURE_PATTERN.test(failure.errorSummary ?? ""));
|
|
149
|
+
if (terminalStructured) return { kind: "terminal-structured-output", retryable: false };
|
|
150
|
+
if (terminalFailures.length > 0 && retryableFailures.length === 0) return { kind: "terminal-lane", retryable: false };
|
|
151
|
+
if (terminalFailures.length > 0 && retryableFailures.length > 0) return { kind: "mixed-lane", retryable: false };
|
|
152
|
+
if (retryableFailures.length > 0) return { kind: "retryable-lane", retryable: true };
|
|
153
|
+
return { kind: "unknown", retryable: true };
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
function boundedString(value) {
|
|
157
|
+
if (value === undefined || value === null) return undefined;
|
|
158
|
+
return truncateText(String(value), MAX_STATUS_STRING_CHARS);
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
function boundedPathRecords(paths = []) {
|
|
162
|
+
return (Array.isArray(paths) ? paths : []).slice(0, MAX_DIAGNOSTIC_ITEMS).map((change) => ({
|
|
163
|
+
path: boundedString(typeof change === "string" ? change : change?.path),
|
|
164
|
+
status: boundedString(typeof change === "string" ? "M" : change?.status),
|
|
165
|
+
supported: typeof change === "object" ? change.supported !== false : true,
|
|
166
|
+
}));
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
function integrationDiagnosticsForState(state) {
|
|
170
|
+
const result = state.integrationPlan?.integrationResult;
|
|
171
|
+
if (!result || typeof result !== "object") return undefined;
|
|
172
|
+
const lanes = Array.isArray(result.lanes) ? result.lanes : [];
|
|
173
|
+
const conflicts = Array.isArray(result.conflicts) ? result.conflicts : [];
|
|
174
|
+
const diagnostic = {
|
|
175
|
+
status: boundedString(result.status),
|
|
176
|
+
reason: boundedString(result.reason),
|
|
177
|
+
culpritLane: boundedString(result.culpritLane),
|
|
178
|
+
errorSummary: boundedString(result.error ?? result.validation?.error ?? result.validation?.reason),
|
|
179
|
+
conflictCount: conflicts.length,
|
|
180
|
+
conflicts: conflicts.slice(0, MAX_DIAGNOSTIC_ITEMS).map((conflict) => ({
|
|
181
|
+
path: boundedString(conflict.path),
|
|
182
|
+
lanes: (Array.isArray(conflict.lanes) ? conflict.lanes : []).slice(0, MAX_DIAGNOSTIC_ITEMS).map(boundedString),
|
|
183
|
+
})),
|
|
184
|
+
mergedLanes: (Array.isArray(result.mergedLanes) ? result.mergedLanes : []).slice(0, MAX_DIAGNOSTIC_ITEMS).map(boundedString),
|
|
185
|
+
affectedLanes: lanes.slice(0, MAX_DIAGNOSTIC_ITEMS).map((lane) => ({
|
|
186
|
+
callId: boundedString(lane.callId),
|
|
187
|
+
branch: boundedString(lane.branch),
|
|
188
|
+
paths: boundedPathRecords(lane.paths),
|
|
189
|
+
})),
|
|
190
|
+
};
|
|
191
|
+
if (conflicts.length > MAX_DIAGNOSTIC_ITEMS) diagnostic.conflictsTruncated = conflicts.length - MAX_DIAGNOSTIC_ITEMS;
|
|
192
|
+
if (lanes.length > MAX_DIAGNOSTIC_ITEMS) diagnostic.affectedLanesTruncated = lanes.length - MAX_DIAGNOSTIC_ITEMS;
|
|
193
|
+
return diagnostic;
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
function redactedIntegrationPlan(state) {
|
|
197
|
+
const plan = state.integrationPlan;
|
|
198
|
+
if (!plan) return undefined;
|
|
199
|
+
const result = plan.integrationResult && typeof plan.integrationResult === "object"
|
|
200
|
+
? { ...plan.integrationResult, patches: undefined }
|
|
201
|
+
: plan.integrationResult;
|
|
202
|
+
return redactValue({ ...plan, patches: undefined, integrationResult: result }, { maxString: MAX_STATUS_STRING_CHARS });
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
function nextActionsForEntry(entry) {
|
|
206
|
+
if (entry.kind !== "valid") {
|
|
207
|
+
const actions = entry.kind === "partial"
|
|
208
|
+
? [
|
|
209
|
+
"Run is incomplete: state.json is missing or not yet written; inspect the run directory.",
|
|
210
|
+
"workflow_cleanup dryRun=true — review whether this incomplete run is safe to prune.",
|
|
211
|
+
]
|
|
212
|
+
: ["workflow_cleanup dryRun=true — review pruning this unreadable run; its state.json is corrupt."];
|
|
213
|
+
return actions.slice(0, MAX_NEXT_ACTIONS);
|
|
214
|
+
}
|
|
215
|
+
const state = entry.state;
|
|
216
|
+
const id = state.id;
|
|
217
|
+
const status = state.status;
|
|
218
|
+
const hasResult = Boolean(state.resultPath);
|
|
219
|
+
const hasSalvage = Array.isArray(entry.salvageCandidates) && entry.salvageCandidates.length > 0;
|
|
220
|
+
const actions = [];
|
|
221
|
+
switch (status) {
|
|
222
|
+
case "completed":
|
|
223
|
+
case "applied":
|
|
224
|
+
case "committed":
|
|
225
|
+
case "integrated": {
|
|
226
|
+
if (hasResult) actions.push(`workflow_status runId=${id} detail=result — read the redacted run result.`);
|
|
227
|
+
actions.push(`workflow_status runId=${id} detail=full — review lane outcomes, usage, and diagnostics.`);
|
|
228
|
+
break;
|
|
229
|
+
}
|
|
230
|
+
case "awaiting-diff-approval":
|
|
231
|
+
case "failed-with-diff-plan": {
|
|
232
|
+
actions.push(`workflow_status runId=${id} detail=full — review the proposed diff plan and its approval hashes.`);
|
|
233
|
+
if (hasApplyableDiffPlan(state)) {
|
|
234
|
+
actions.push(`workflow_apply runId=${id} approvalIntent=apply <approvedSourceHash/baseCommit/diffPlanHash/domainMutationHash from detail=full> — apply after review.`);
|
|
235
|
+
} else {
|
|
236
|
+
actions.push("No applyable diff plan hashes are present; inspect detail=full before choosing a recovery path.");
|
|
237
|
+
}
|
|
238
|
+
break;
|
|
239
|
+
}
|
|
240
|
+
case "review-required": {
|
|
241
|
+
actions.push(`workflow_status runId=${id} detail=full — inspect review-required diagnostics before choosing a recovery path.`);
|
|
242
|
+
actions.push("Review-required runs are not directly applyable; resolve the conflict or rerun with corrected workflow input.");
|
|
243
|
+
break;
|
|
244
|
+
}
|
|
245
|
+
case "apply-failed": {
|
|
246
|
+
actions.push(`workflow_status runId=${id} detail=full — inspect the apply failure and the staged diff plan.`);
|
|
247
|
+
if (hasApplyableDiffPlan(state)) {
|
|
248
|
+
actions.push(`workflow_apply runId=${id} approvalIntent=apply <hashes from detail=full> — retry apply after resolving the cause.`);
|
|
249
|
+
} else {
|
|
250
|
+
actions.push("No applyable diff plan hashes are present; inspect detail=full before retrying apply.");
|
|
251
|
+
}
|
|
252
|
+
break;
|
|
253
|
+
}
|
|
254
|
+
case "failed": {
|
|
255
|
+
const recovery = failedRunRecoveryForState(state);
|
|
256
|
+
actions.push(`workflow_status runId=${id} detail=full — inspect failure context (laneOutcomes, errorSummary, diagnostics).`);
|
|
257
|
+
if (recovery.kind === "terminal-structured-output") {
|
|
258
|
+
actions.push("Fix or inspect terminal structured-output/schema lane failures before rerunning; resume is not recommended.");
|
|
259
|
+
} else if (recovery.kind === "terminal-lane") {
|
|
260
|
+
actions.push("Fix terminal lane configuration or workflow source before rerunning; resume is not recommended.");
|
|
261
|
+
} else if (recovery.kind === "mixed-lane") {
|
|
262
|
+
actions.push("Fix terminal lane failures first; retryable lanes can resume only after the terminal cause is addressed.");
|
|
263
|
+
} else {
|
|
264
|
+
actions.push(`workflow_run resumeRunId=${id} — resume; completed lanes replay from cache at no new spend.`);
|
|
265
|
+
}
|
|
266
|
+
break;
|
|
267
|
+
}
|
|
268
|
+
case "timed-out": {
|
|
269
|
+
const timeoutRecovery = timeoutResumeEligibilityForState(state);
|
|
270
|
+
actions.push(`workflow_status runId=${id} detail=full — inspect timeout context (laneOutcomes, errorSummary, diagnostics).`);
|
|
271
|
+
if (timeoutRecovery.eligible) {
|
|
272
|
+
const nextDeadline = Number.isInteger(timeoutRecovery.requiredResumeArgs.maxRuntimeMsGreaterThan)
|
|
273
|
+
? `maxRuntimeMs>${timeoutRecovery.requiredResumeArgs.maxRuntimeMsGreaterThan}`
|
|
274
|
+
: "maxRuntimeMs=<raised deadline>";
|
|
275
|
+
actions.push(`workflow_run resumeRunId=${id} resumePolicy=extend-deadline ${nextDeadline} — resume eligible read-only timeout; completed lanes replay from cache.`);
|
|
276
|
+
} else {
|
|
277
|
+
actions.push(`Timed-out run is blocked from resume: ${timeoutRecovery.blockedReasons.slice(0, 3).join("; ") || "unknown reason"}.`);
|
|
278
|
+
}
|
|
279
|
+
break;
|
|
280
|
+
}
|
|
281
|
+
case "cancelled": {
|
|
282
|
+
actions.push(`workflow_status runId=${id} detail=full — review what completed before cancellation.`);
|
|
283
|
+
actions.push("Cancelled runs are non-resumable by design; start a fresh workflow_run if more work is needed.");
|
|
284
|
+
break;
|
|
285
|
+
}
|
|
286
|
+
case "stale-active": {
|
|
287
|
+
actions.push(`workflow_reconcile runId=${id} — persist interrupted recovery state and clear stale locks.`);
|
|
288
|
+
break;
|
|
289
|
+
}
|
|
290
|
+
case "active-unknown": {
|
|
291
|
+
actions.push("This run appears active in another OpenCode process; check that process before acting.");
|
|
292
|
+
actions.push(`workflow_status runId=${id} detail=full — inspect last persisted progress.`);
|
|
293
|
+
break;
|
|
294
|
+
}
|
|
295
|
+
case "interrupted":
|
|
296
|
+
case "paused": {
|
|
297
|
+
actions.push(`workflow_run resumeRunId=${id} — resume; completed lanes replay from cache at no new spend.`);
|
|
298
|
+
break;
|
|
299
|
+
}
|
|
300
|
+
case "running":
|
|
301
|
+
case "apply-running": {
|
|
302
|
+
actions.push(`workflow_status runId=${id} detail=full — watch progress and lane activity.`);
|
|
303
|
+
actions.push(`workflow_pause runId=${id} / workflow_cancel runId=${id} — pause (resumable) or cancel (abandon) the run.`);
|
|
304
|
+
actions.push(`workflow_kill runId=${id} — force-terminate if a lane is wedged and cancel/pause do not return.`);
|
|
305
|
+
break;
|
|
306
|
+
}
|
|
307
|
+
case "cancelling":
|
|
308
|
+
case "pausing": {
|
|
309
|
+
actions.push(`workflow_status runId=${id} detail=full — watch the run settle.`);
|
|
310
|
+
actions.push(`workflow_kill runId=${id} — force-terminate if it does not settle because a lane is wedged.`);
|
|
311
|
+
break;
|
|
312
|
+
}
|
|
313
|
+
case "pending-approval": {
|
|
314
|
+
actions.push("Re-run workflow_run with approve=true and the matching approvalHash to launch this envelope.");
|
|
315
|
+
break;
|
|
316
|
+
}
|
|
317
|
+
default: {
|
|
318
|
+
actions.push(`workflow_status runId=${id} detail=full — inspect full run state.`);
|
|
319
|
+
break;
|
|
320
|
+
}
|
|
321
|
+
}
|
|
322
|
+
if (hasSalvage) {
|
|
323
|
+
actions.push(`workflow_salvage runId=${id} — recover orphaned read-only lane results from child transcripts (preview first).`);
|
|
324
|
+
}
|
|
325
|
+
return actions.slice(0, MAX_NEXT_ACTIONS);
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
// No-progress staleness threshold (opencode-workflows-jbs3.8). An ACTIVE run whose lastProgressAt
|
|
329
|
+
// is older than this is reported `staleness.stale: true` in the COMPACT status so an agent can
|
|
330
|
+
// detect a wedged / no-heartbeat run from the default (compact) view alone — without pulling
|
|
331
|
+
// detail=full. Terminal / non-active runs are never "stale" (no further progress is expected), so
|
|
332
|
+
// `stale` is false for them regardless of age. The threshold is exported and echoed in the payload
|
|
333
|
+
// (staleness.thresholdMs) so the contract is self-documenting for the agent reading it.
|
|
334
|
+
const STALE_PROGRESS_THRESHOLD_MS = 10 * 60 * 1000;
|
|
335
|
+
// How long a permanently-unresumable interrupted run keeps its cleanup protection before
|
|
336
|
+
// it becomes eligible for reaping. Without this escape valve an interrupted run pins its
|
|
337
|
+
// run dir (and, via reconcile, its stranded worktree/branch) forever. Conservative default:
|
|
338
|
+
// a run idle this long is almost certainly never going to be salvaged. Overridable per call.
|
|
339
|
+
const INTERRUPTED_RUN_TTL_MS = 7 * 24 * 60 * 60 * 1000;
|
|
340
|
+
|
|
341
|
+
function lastProgressAtForState(state) {
|
|
342
|
+
if (typeof state.lastProgressAt === "string") return state.lastProgressAt;
|
|
343
|
+
// Snapshots written before jbs3.8 (or by tests) lack the persisted field; derive it the same
|
|
344
|
+
// way writeState does so the staleness signal still works on older / synthetic state.json.
|
|
345
|
+
return computeLastProgressAt(state);
|
|
346
|
+
}
|
|
347
|
+
|
|
348
|
+
function stalenessSignal(state, lastProgressAt, now = Date.now()) {
|
|
349
|
+
const parsed = lastProgressAt ? Date.parse(lastProgressAt) : Number.NaN;
|
|
350
|
+
const ageMs = Number.isFinite(parsed) ? Math.max(0, now - parsed) : null;
|
|
351
|
+
const active = ACTIVE_STATUSES.has(state.status);
|
|
352
|
+
return {
|
|
353
|
+
lastProgressAt: lastProgressAt ?? null,
|
|
354
|
+
ageMs,
|
|
355
|
+
thresholdMs: STALE_PROGRESS_THRESHOLD_MS,
|
|
356
|
+
stale: active && ageMs !== null && ageMs > STALE_PROGRESS_THRESHOLD_MS,
|
|
357
|
+
};
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
const LANE_FAILURE_OUTCOMES = new Set(["failure", "cancelled", "timeout", "budget_stopped"]);
|
|
361
|
+
const MAX_COMPACT_LANE_FAILURES = 25;
|
|
362
|
+
const ACTIVE_LANE_STATUSES = new Set(["worktree-created", "running", "committed"]);
|
|
363
|
+
const MAX_ACTIVE_LANES = 25;
|
|
364
|
+
const TIMEOUT_RETRYABLE_OUTCOMES = new Set(["timeout", "cancelled", "budget_stopped"]);
|
|
365
|
+
const WRITE_AUTHORITY_FLAGS = ["shell", "network", "mcp", "edit", "worktreeEdit", "integration"];
|
|
366
|
+
|
|
367
|
+
function activeLaneSummaries(state, now = Date.now()) {
|
|
368
|
+
const records = Array.isArray(state.laneRecords) ? state.laneRecords : [];
|
|
369
|
+
const active = [];
|
|
370
|
+
for (const record of records) {
|
|
371
|
+
if (!record || record.outcome || !ACTIVE_LANE_STATUSES.has(record.status)) continue;
|
|
372
|
+
const startedAt = typeof record.startedAt === "string" ? record.startedAt : null;
|
|
373
|
+
const lastActivityAt = record.lastActivityAt ?? record.updatedAt ?? record.completedAt ?? startedAt;
|
|
374
|
+
const startedMs = startedAt ? Date.parse(startedAt) : Number.NaN;
|
|
375
|
+
const activityMs = lastActivityAt ? Date.parse(lastActivityAt) : Number.NaN;
|
|
376
|
+
active.push({
|
|
377
|
+
callId: record.callId,
|
|
378
|
+
status: record.status,
|
|
379
|
+
title: record.title ? truncateText(redactFreeTextSecrets(record.title), 160) : null,
|
|
380
|
+
taskSummary: record.taskSummary ? truncateText(redactFreeTextSecrets(record.taskSummary), 160) : null,
|
|
381
|
+
childID: record.childID ?? null,
|
|
382
|
+
role: record.role ?? null,
|
|
383
|
+
model: record.model ?? null,
|
|
384
|
+
startedAt,
|
|
385
|
+
lastActivityAt: lastActivityAt ?? null,
|
|
386
|
+
ageMs: Number.isFinite(startedMs) ? Math.max(0, now - startedMs) : null,
|
|
387
|
+
idleMs: Number.isFinite(activityMs) ? Math.max(0, now - activityMs) : null,
|
|
388
|
+
progressEmitted: Boolean(record.childID || (lastActivityAt && lastActivityAt !== startedAt)),
|
|
389
|
+
...(record.tokens ? { tokens: record.tokens } : {}),
|
|
390
|
+
});
|
|
391
|
+
}
|
|
392
|
+
return active;
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
function ledgerSummaryRecordCount(state, key) {
|
|
396
|
+
const summary = state?.durability?.ledgers?.[key];
|
|
397
|
+
return Number.isFinite(summary?.records) ? summary.records : null;
|
|
398
|
+
}
|
|
399
|
+
|
|
400
|
+
function ledgerSummaryPhases(state, key) {
|
|
401
|
+
const phases = state?.durability?.ledgers?.[key]?.phases;
|
|
402
|
+
return phases && typeof phases === "object" && !Array.isArray(phases) ? phases : {};
|
|
403
|
+
}
|
|
404
|
+
|
|
405
|
+
function hasAnyPhase(phases, names) {
|
|
406
|
+
return names.some((name) => Number.isFinite(phases[name]) && phases[name] > 0);
|
|
407
|
+
}
|
|
408
|
+
|
|
409
|
+
function preservedWorktreeReasons(state = {}) {
|
|
410
|
+
const cleanup = state.worktreeCleanup;
|
|
411
|
+
if (!cleanup || typeof cleanup !== "object" || Array.isArray(cleanup)) return [];
|
|
412
|
+
const reasons = [];
|
|
413
|
+
for (const [group, records] of Object.entries(cleanup)) {
|
|
414
|
+
if (!Array.isArray(records)) continue;
|
|
415
|
+
for (const record of records) {
|
|
416
|
+
if (!record?.preserved) continue;
|
|
417
|
+
const dirty = record.dirty === true || record.reason === "dirty";
|
|
418
|
+
reasons.push(`preserved ${dirty ? "dirty " : ""}${group} worktree is present`);
|
|
419
|
+
}
|
|
420
|
+
}
|
|
421
|
+
return reasons;
|
|
422
|
+
}
|
|
423
|
+
|
|
424
|
+
function timeoutResumeEligibilityForState(state = {}) {
|
|
425
|
+
const blockedReasons = [];
|
|
426
|
+
const records = Array.isArray(state.laneRecords) ? state.laneRecords : [];
|
|
427
|
+
const completedLaneCount = records.filter((record) => record?.outcome === "success" || record?.status === "completed").length;
|
|
428
|
+
const activeOrTimedOutLaneCount = records.filter((record) => {
|
|
429
|
+
if (!record) return false;
|
|
430
|
+
if (!record.outcome && ACTIVE_LANE_STATUSES.has(record.status)) return true;
|
|
431
|
+
return record.outcome === "timeout" || record.outcome === "cancelled";
|
|
432
|
+
}).length;
|
|
433
|
+
const requiredResumeArgs = {
|
|
434
|
+
resumePolicy: "extend-deadline",
|
|
435
|
+
maxRuntimeMsGreaterThan: Number.isInteger(state.maxRuntimeMs) ? state.maxRuntimeMs : null,
|
|
436
|
+
};
|
|
437
|
+
|
|
438
|
+
if (state.status !== "timed-out") blockedReasons.push(`status is ${state.status ?? "unknown"}, not timed-out`);
|
|
439
|
+
if (!Number.isInteger(state.maxRuntimeMs) || state.maxRuntimeMs <= 0) {
|
|
440
|
+
blockedReasons.push("persisted maxRuntimeMs is missing, so a raised deadline cannot be proven");
|
|
441
|
+
}
|
|
442
|
+
|
|
443
|
+
const authority = state.authority && typeof state.authority === "object" ? state.authority : {};
|
|
444
|
+
const hasWriteAuthority = WRITE_AUTHORITY_FLAGS.some((flag) => authority[flag] === true);
|
|
445
|
+
if (authority.readOnly !== true || authority.mode !== "readOnly" || hasWriteAuthority) {
|
|
446
|
+
blockedReasons.push("authority is not strictly read-only");
|
|
447
|
+
}
|
|
448
|
+
|
|
449
|
+
if (state.editPlan) {
|
|
450
|
+
blockedReasons.push(state.editPlan.diffPlanHash || Number.isFinite(state.editPlan.patchCount) ? "diff plan is present" : "editPlan is present");
|
|
451
|
+
}
|
|
452
|
+
if (state.integrationPlan) {
|
|
453
|
+
const partialIntegration = Array.isArray(state.integrationPlan.lanes) && state.integrationPlan.lanes.length > 0 && !state.integrationPlan.integrationResult;
|
|
454
|
+
blockedReasons.push(partialIntegration ? "partial integration plan is present" : "integrationPlan is present");
|
|
455
|
+
}
|
|
456
|
+
if (Array.isArray(state.editWorktrees) && state.editWorktrees.length > 0) blockedReasons.push("edit worktrees are present");
|
|
457
|
+
if (Array.isArray(state.integrationWorktrees) && state.integrationWorktrees.length > 0) blockedReasons.push("integration worktrees are present");
|
|
458
|
+
blockedReasons.push(...preservedWorktreeReasons(state));
|
|
459
|
+
|
|
460
|
+
const domainRecords = ledgerSummaryRecordCount(state, "domain-ledger");
|
|
461
|
+
const applyRecords = ledgerSummaryRecordCount(state, "apply-ledger");
|
|
462
|
+
if (domainRecords === null || applyRecords === null) {
|
|
463
|
+
blockedReasons.push("durable ledger summary is missing");
|
|
464
|
+
} else {
|
|
465
|
+
if (domainRecords > 0) {
|
|
466
|
+
const domainPhases = ledgerSummaryPhases(state, "domain-ledger");
|
|
467
|
+
blockedReasons.push(hasAnyPhase(domainPhases, ["staged", "started", "executed"]) && !hasAnyPhase(domainPhases, ["completed", "failed"])
|
|
468
|
+
? "staged domain mutation ledger is present"
|
|
469
|
+
: "domain mutation ledger is present");
|
|
470
|
+
}
|
|
471
|
+
if (applyRecords > 0) {
|
|
472
|
+
const applyPhases = ledgerSummaryPhases(state, "apply-ledger");
|
|
473
|
+
blockedReasons.push(hasAnyPhase(applyPhases, ["started", "before-write", "after-write"]) && !hasAnyPhase(applyPhases, ["completed", "failed"])
|
|
474
|
+
? "apply ledger is incomplete"
|
|
475
|
+
: "apply ledger is present");
|
|
476
|
+
}
|
|
477
|
+
}
|
|
478
|
+
|
|
479
|
+
const runLock = state.locks?.run;
|
|
480
|
+
if (runLock) {
|
|
481
|
+
const lockState = runLock.active ? "active" : runLock.stale ? "stale" : runLock.corrupt ? "corrupt" : runLock.unreadable ? "unreadable" : "present";
|
|
482
|
+
blockedReasons.push(`run.lock is ${lockState}`);
|
|
483
|
+
}
|
|
484
|
+
|
|
485
|
+
for (const record of records) {
|
|
486
|
+
if (!record) continue;
|
|
487
|
+
if (record.outcome === "success") continue;
|
|
488
|
+
if (!record.outcome && ACTIVE_LANE_STATUSES.has(record.status)) continue;
|
|
489
|
+
if (TIMEOUT_RETRYABLE_OUTCOMES.has(record.outcome)) continue;
|
|
490
|
+
if (record.outcome === "failure" && record.retryable !== false && record.failureClass !== "terminal") continue;
|
|
491
|
+
blockedReasons.push(`lane ${record.callId ?? "unknown"} outcome is not safely retryable`);
|
|
492
|
+
}
|
|
493
|
+
|
|
494
|
+
return {
|
|
495
|
+
eligible: blockedReasons.length === 0,
|
|
496
|
+
blockedReasons,
|
|
497
|
+
completedLaneCount,
|
|
498
|
+
activeOrTimedOutLaneCount,
|
|
499
|
+
requiredResumeArgs,
|
|
500
|
+
};
|
|
501
|
+
}
|
|
502
|
+
|
|
503
|
+
// Trimmed per-lane failure summary promoted into the COMPACT status (opencode-workflows-jbs3.8) so
|
|
504
|
+
// an agent sees WHICH lane(s) failed and WHY at default detail — not only via detail=full's whole
|
|
505
|
+
// laneRecords array. Strictly allowlisted: callId/role/model identify the lane, outcome is the
|
|
506
|
+
// coarse LANE_OUTCOMES bucket, failureClass is the transient/transient_exhausted/terminal taxonomy
|
|
507
|
+
// (jbs3.2), and errorSummary was already redacted+bounded at projection-write time (re-bounded here
|
|
508
|
+
// defensively). It NEVER includes title/taskSummary/raw result, so it cannot leak prompts or lane
|
|
509
|
+
// output, and it is capped so a fan-out with hundreds of failed lanes cannot bloat the compact view.
|
|
510
|
+
function laneFailureSummaries(state) {
|
|
511
|
+
const records = Array.isArray(state.laneRecords) ? state.laneRecords : [];
|
|
512
|
+
const failures = [];
|
|
513
|
+
for (const record of records) {
|
|
514
|
+
if (!record || !LANE_FAILURE_OUTCOMES.has(record.outcome)) continue;
|
|
515
|
+
failures.push({
|
|
516
|
+
callId: record.callId,
|
|
517
|
+
role: record.role ?? null,
|
|
518
|
+
model: record.model ?? null,
|
|
519
|
+
outcome: record.outcome,
|
|
520
|
+
failureClass: record.failureClass ?? null,
|
|
521
|
+
retryable: record.retryable ?? null,
|
|
522
|
+
errorSummary: record.errorSummary ? truncateText(redactFreeTextSecrets(record.errorSummary), MAX_STATUS_STRING_CHARS) : null,
|
|
523
|
+
});
|
|
524
|
+
}
|
|
525
|
+
return failures;
|
|
526
|
+
}
|
|
527
|
+
|
|
528
|
+
function durationMs(start, end) {
|
|
529
|
+
const startMs = typeof start === "string" ? Date.parse(start) : Number.NaN;
|
|
530
|
+
const endMs = typeof end === "string" ? Date.parse(end) : Number.NaN;
|
|
531
|
+
if (!Number.isFinite(startMs) || !Number.isFinite(endMs)) return null;
|
|
532
|
+
return Math.max(0, endMs - startMs);
|
|
533
|
+
}
|
|
534
|
+
|
|
535
|
+
function operatorMetricsForState(state = {}) {
|
|
536
|
+
const approvalWait = state.approvalWait && typeof state.approvalWait === "object" ? state.approvalWait : {};
|
|
537
|
+
return {
|
|
538
|
+
firstResultAt: state.firstResultAt ?? state.operatorMetrics?.firstResultAt ?? null,
|
|
539
|
+
timeToFirstResultMs: Number.isFinite(state.timeToFirstResultMs)
|
|
540
|
+
? state.timeToFirstResultMs
|
|
541
|
+
: (Number.isFinite(state.operatorMetrics?.timeToFirstResultMs) ? state.operatorMetrics.timeToFirstResultMs : null),
|
|
542
|
+
awaitingDiffApprovalAt: approvalWait.startedAt ?? state.operatorMetrics?.awaitingDiffApprovalAt ?? null,
|
|
543
|
+
appliedAt: approvalWait.completedAt ?? state.operatorMetrics?.appliedAt ?? null,
|
|
544
|
+
approvalWaitMs: Number.isFinite(approvalWait.durationMs)
|
|
545
|
+
? approvalWait.durationMs
|
|
546
|
+
: (Number.isFinite(state.operatorMetrics?.approvalWaitMs) ? state.operatorMetrics.approvalWaitMs : null),
|
|
547
|
+
};
|
|
548
|
+
}
|
|
549
|
+
|
|
550
|
+
function compactStatusForEntry(entry) {
|
|
551
|
+
if (entry.kind !== "valid") {
|
|
552
|
+
return {
|
|
553
|
+
id: entry.id,
|
|
554
|
+
status: entry.status,
|
|
555
|
+
errorSummary: truncateText(redactFreeTextSecrets(entry.error ?? ""), MAX_STATUS_STRING_CHARS),
|
|
556
|
+
nextActions: nextActionsForEntry(entry),
|
|
557
|
+
};
|
|
558
|
+
}
|
|
559
|
+
const state = entry.state;
|
|
560
|
+
const effectiveAuthorityProfile = state.authority?.profile ?? AD_HOC_AUTHORITY_PROFILE;
|
|
561
|
+
const compact = {
|
|
562
|
+
id: state.id,
|
|
563
|
+
status: state.status,
|
|
564
|
+
meta: redactValue(state.meta ?? {}),
|
|
565
|
+
declaredProfile: declaredProfileForState(state),
|
|
566
|
+
effectiveAuthorityProfile,
|
|
567
|
+
startedAt: state.startedAt ?? null,
|
|
568
|
+
finishedAt: state.finishedAt ?? null,
|
|
569
|
+
currentPhase: state.currentPhase ?? null,
|
|
570
|
+
agentsStarted: state.agentsStarted ?? 0,
|
|
571
|
+
maxAgents: state.maxAgents ?? 0,
|
|
572
|
+
concurrency: state.concurrency ?? 0,
|
|
573
|
+
laneTimeoutMs: state.laneTimeoutMs ?? DEFAULT_CHILD_PROMPT_TIMEOUT_MS,
|
|
574
|
+
maxRuntimeMs: state.maxRuntimeMs ?? null,
|
|
575
|
+
activeAgents: state.activeAgents ?? 0,
|
|
576
|
+
queuedAgents: state.queuedAgents ?? 0,
|
|
577
|
+
tokens: state.tokens ?? zeroTokens(),
|
|
578
|
+
replayedTokens: state.replayedTokens ?? zeroTokens(),
|
|
579
|
+
cost: state.cost ?? 0,
|
|
580
|
+
replayedCost: state.replayedCost ?? 0,
|
|
581
|
+
...(state.autoApproved ? { autoApproved: state.autoApproved } : {}),
|
|
582
|
+
laneOutcomes: state.laneOutcomes ?? Object.fromEntries(LANE_OUTCOMES.map((outcome) => [outcome, 0])),
|
|
583
|
+
droppedLaneCount: state.droppedLaneCount ?? 0,
|
|
584
|
+
background: state.background === true,
|
|
585
|
+
operatorMetrics: operatorMetricsForState(state),
|
|
586
|
+
locks: state.locks,
|
|
587
|
+
lifecycleRequests: state.lifecycleRequests,
|
|
588
|
+
...(state.resultPath ? { resultPath: state.resultPath } : {}),
|
|
589
|
+
errorSummary: state.error ? truncateText(redactFreeTextSecrets(state.error), MAX_STATUS_STRING_CHARS) : null,
|
|
590
|
+
nextActions: nextActionsForEntry(entry),
|
|
591
|
+
};
|
|
592
|
+
const lastProgressAt = lastProgressAtForState(state);
|
|
593
|
+
compact.lastProgressAt = lastProgressAt;
|
|
594
|
+
compact.staleness = stalenessSignal(state, lastProgressAt);
|
|
595
|
+
const laneFailures = laneFailureSummaries(state);
|
|
596
|
+
const activeLanes = activeLaneSummaries(state);
|
|
597
|
+
if (activeLanes.length > 0) {
|
|
598
|
+
compact.activeLanes = activeLanes.slice(0, MAX_ACTIVE_LANES);
|
|
599
|
+
compact.activeLaneCount = activeLanes.length;
|
|
600
|
+
if (activeLanes.length > MAX_ACTIVE_LANES) {
|
|
601
|
+
compact.activeLanesTruncated = activeLanes.length - MAX_ACTIVE_LANES;
|
|
602
|
+
}
|
|
603
|
+
}
|
|
604
|
+
if (laneFailures.length > 0) {
|
|
605
|
+
compact.laneFailures = laneFailures.slice(0, MAX_COMPACT_LANE_FAILURES);
|
|
606
|
+
if (laneFailures.length > MAX_COMPACT_LANE_FAILURES) {
|
|
607
|
+
compact.laneFailuresTruncated = laneFailures.length - MAX_COMPACT_LANE_FAILURES;
|
|
608
|
+
}
|
|
609
|
+
}
|
|
610
|
+
if (Array.isArray(entry.salvageCandidates) && entry.salvageCandidates.length > 0) {
|
|
611
|
+
compact.salvageCandidates = entry.salvageCandidates;
|
|
612
|
+
}
|
|
613
|
+
if (state.status === "timed-out") compact.timeoutRecovery = timeoutResumeEligibilityForState(state);
|
|
614
|
+
return compact;
|
|
615
|
+
}
|
|
616
|
+
|
|
617
|
+
function declaredProfileForState(state) { return state.meta?.profile ?? state.meta?.authorityProfile ?? state.meta?.authority?.profile ?? null; }
|
|
618
|
+
|
|
619
|
+
async function resultStatusForEntry(entry) {
|
|
620
|
+
const resultStatus = compactStatusForEntry(entry);
|
|
621
|
+
if (entry.kind !== "valid") return resultStatus;
|
|
622
|
+
const resultPath = entry.state?.resultPath;
|
|
623
|
+
resultStatus.resultPath = resultPath ?? null;
|
|
624
|
+
if (!resultPath) return resultStatus;
|
|
625
|
+
try {
|
|
626
|
+
await assertContainedRealPath(entry.dir, resultPath, "Workflow result path");
|
|
627
|
+
const stat = await fs.stat(resultPath);
|
|
628
|
+
resultStatus.resultFileBytes = stat.size;
|
|
629
|
+
if (stat.size > MAX_RESULT_READ_FILE_BYTES) {
|
|
630
|
+
resultStatus.resultError = `Result file exceeds ${MAX_RESULT_READ_FILE_BYTES} byte readback ceiling`;
|
|
631
|
+
return resultStatus;
|
|
632
|
+
}
|
|
633
|
+
Object.assign(resultStatus, resultReadbackProjection(JSON.parse(await fs.readFile(resultPath, "utf8"))));
|
|
634
|
+
} catch (error) {
|
|
635
|
+
resultStatus.resultError = truncateText(`Unable to read result file: ${extractTextFromError(error)}`, MAX_STATUS_STRING_CHARS);
|
|
636
|
+
}
|
|
637
|
+
return resultStatus;
|
|
638
|
+
}
|
|
639
|
+
|
|
640
|
+
function normalizeEventLimit(value) {
|
|
641
|
+
if (Number.isInteger(value) && value > 0) return Math.min(value, MAX_WORKFLOW_EVENTS_LIMIT);
|
|
642
|
+
return DEFAULT_WORKFLOW_EVENTS_LIMIT;
|
|
643
|
+
}
|
|
644
|
+
|
|
645
|
+
function normalizeEventOffset(value) {
|
|
646
|
+
return Number.isInteger(value) && value >= 0 ? value : 0;
|
|
647
|
+
}
|
|
648
|
+
|
|
649
|
+
function eventTimestampInRange(event, { sinceTimestamp, beforeTimestamp } = {}) {
|
|
650
|
+
const ts = typeof event?.ts === "string" ? Date.parse(event.ts) : Number.NaN;
|
|
651
|
+
if (sinceTimestamp) {
|
|
652
|
+
const since = Date.parse(sinceTimestamp);
|
|
653
|
+
if (Number.isFinite(since) && Number.isFinite(ts) && ts < since) return false;
|
|
654
|
+
}
|
|
655
|
+
if (beforeTimestamp) {
|
|
656
|
+
const before = Date.parse(beforeTimestamp);
|
|
657
|
+
if (Number.isFinite(before) && Number.isFinite(ts) && ts >= before) return false;
|
|
658
|
+
}
|
|
659
|
+
return true;
|
|
660
|
+
}
|
|
661
|
+
|
|
662
|
+
function eventTypeMatches(event, typePrefix) {
|
|
663
|
+
if (!typePrefix) return true;
|
|
664
|
+
return typeof event?.type === "string" && event.type.startsWith(typePrefix);
|
|
665
|
+
}
|
|
666
|
+
|
|
667
|
+
async function readEventsFile(filePath) {
|
|
668
|
+
const events = [];
|
|
669
|
+
let invalidLineCount = 0;
|
|
670
|
+
try {
|
|
671
|
+
const content = await fs.readFile(filePath, "utf8");
|
|
672
|
+
for (const line of content.split(/\r?\n/)) {
|
|
673
|
+
if (!line.trim()) continue;
|
|
674
|
+
try {
|
|
675
|
+
events.push(JSON.parse(line));
|
|
676
|
+
} catch {
|
|
677
|
+
invalidLineCount += 1;
|
|
678
|
+
}
|
|
679
|
+
}
|
|
680
|
+
} catch (error) {
|
|
681
|
+
if (error.code !== "ENOENT") throw error;
|
|
682
|
+
}
|
|
683
|
+
return { events, invalidLineCount };
|
|
684
|
+
}
|
|
685
|
+
|
|
686
|
+
async function eventsForEntry(entry, args = {}) {
|
|
687
|
+
if (entry.kind !== "valid") {
|
|
688
|
+
return {
|
|
689
|
+
runId: entry.id,
|
|
690
|
+
status: entry.status,
|
|
691
|
+
events: [],
|
|
692
|
+
errorSummary: truncateText(redactFreeTextSecrets(entry.error ?? ""), MAX_STATUS_STRING_CHARS),
|
|
693
|
+
totalEvents: 0,
|
|
694
|
+
totalMatching: 0,
|
|
695
|
+
returned: 0,
|
|
696
|
+
invalidLineCount: 0,
|
|
697
|
+
nextOffset: null,
|
|
698
|
+
};
|
|
699
|
+
}
|
|
700
|
+
const limit = normalizeEventLimit(args.limit);
|
|
701
|
+
const offset = normalizeEventOffset(args.offset);
|
|
702
|
+
const order = args.order === "oldest" ? "oldest" : "newest";
|
|
703
|
+
const { events, invalidLineCount } = await readEventsFile(path.join(entry.dir, "events.jsonl"));
|
|
704
|
+
const matching = events
|
|
705
|
+
.filter((event) => eventTypeMatches(event, args.typePrefix))
|
|
706
|
+
.filter((event) => eventTimestampInRange(event, args));
|
|
707
|
+
const ordered = order === "newest" ? [...matching].reverse() : matching;
|
|
708
|
+
const page = ordered.slice(offset, offset + limit);
|
|
709
|
+
return {
|
|
710
|
+
runId: entry.id,
|
|
711
|
+
status: entry.status,
|
|
712
|
+
order,
|
|
713
|
+
typePrefix: args.typePrefix ?? null,
|
|
714
|
+
sinceTimestamp: args.sinceTimestamp ?? null,
|
|
715
|
+
beforeTimestamp: args.beforeTimestamp ?? null,
|
|
716
|
+
offset,
|
|
717
|
+
limit,
|
|
718
|
+
maxLimit: MAX_WORKFLOW_EVENTS_LIMIT,
|
|
719
|
+
totalEvents: events.length,
|
|
720
|
+
totalMatching: matching.length,
|
|
721
|
+
returned: page.length,
|
|
722
|
+
nextOffset: offset + page.length < ordered.length ? offset + page.length : null,
|
|
723
|
+
invalidLineCount,
|
|
724
|
+
events: redactValue(page, { maxString: MAX_STATUS_STRING_CHARS, maxArray: MAX_WORKFLOW_EVENTS_LIMIT }),
|
|
725
|
+
};
|
|
726
|
+
}
|
|
727
|
+
|
|
728
|
+
async function eventsText(context, args = {}) {
|
|
729
|
+
const runId = assertSafeRunId(String(args.runId ?? ""), "runId");
|
|
730
|
+
const entries = [];
|
|
731
|
+
for (const root of runRoots(context)) {
|
|
732
|
+
const dir = runDirForRoot(root, runId);
|
|
733
|
+
if (await pathExists(dir)) entries.push(await readRunEntry(root, runId));
|
|
734
|
+
}
|
|
735
|
+
if (entries.length === 0) throw new Error(`Workflow run not found: ${runId}`);
|
|
736
|
+
const result = await eventsForEntry(entries[0], args);
|
|
737
|
+
if (args.format === "summary") {
|
|
738
|
+
const lines = [
|
|
739
|
+
`Workflow ${result.runId} events (${result.returned}/${result.totalMatching}, order=${result.order}, offset=${result.offset}, limit=${result.limit})`,
|
|
740
|
+
result.typePrefix ? `Filter: type starts with ${result.typePrefix}` : undefined,
|
|
741
|
+
result.invalidLineCount > 0 ? `Skipped invalid/truncated lines: ${result.invalidLineCount}` : undefined,
|
|
742
|
+
...result.events.map((event) => `${event.ts ?? "-"} ${event.type ?? "event"}${event.callId ? ` callId=${event.callId}` : ""}`),
|
|
743
|
+
result.nextOffset !== null ? `Next page: workflow_events({ runId: "${result.runId}", offset: ${result.nextOffset}, limit: ${result.limit} })` : undefined,
|
|
744
|
+
].filter(Boolean);
|
|
745
|
+
return lines.join("\n");
|
|
746
|
+
}
|
|
747
|
+
return JSON.stringify(result, null, 2);
|
|
748
|
+
}
|
|
749
|
+
|
|
750
|
+
async function notificationStatusForEntry(entry, state) {
|
|
751
|
+
const notificationPath = state.notification?.notificationPath ?? path.join(entry.dir, "notification.json");
|
|
752
|
+
let notification;
|
|
753
|
+
try {
|
|
754
|
+
await assertContainedRealPath(entry.dir, notificationPath, "Workflow notification path");
|
|
755
|
+
notification = await readJsonFile(notificationPath, undefined);
|
|
756
|
+
} catch {
|
|
757
|
+
// Missing or out-of-run notification file (e.g. tampered state.json redirecting the
|
|
758
|
+
// read elsewhere): fall back to the persisted record rather than returning arbitrary contents.
|
|
759
|
+
return state.notification;
|
|
760
|
+
}
|
|
761
|
+
const projected = notification ? redactValue(notification, { maxString: MAX_STATUS_STRING_CHARS }) : state.notification;
|
|
762
|
+
if (projected && typeof projected === "object") {
|
|
763
|
+
const lastAttemptAt = projected.delivery?.lastAttemptAt;
|
|
764
|
+
const sentAt = projected.sentAt;
|
|
765
|
+
projected.delivery = {
|
|
766
|
+
...(projected.delivery ?? {}),
|
|
767
|
+
deliveryLatencyMs: durationMs(lastAttemptAt, sentAt),
|
|
768
|
+
totalLatencyMs: durationMs(projected.createdAt, sentAt),
|
|
769
|
+
};
|
|
770
|
+
}
|
|
771
|
+
return projected;
|
|
772
|
+
}
|
|
773
|
+
|
|
774
|
+
async function fullStatusForEntry(entry) {
|
|
775
|
+
if (entry.kind !== "valid") return { id: entry.id, status: entry.status, root: entry.root, dir: entry.dir, error: truncateText(redactFreeTextSecrets(entry.error ?? ""), MAX_STATUS_STRING_CHARS), nextActions: nextActionsForEntry(entry) };
|
|
776
|
+
const state = entry.state;
|
|
777
|
+
const activeLanes = activeLaneSummaries(state);
|
|
778
|
+
const effectiveAuthorityProfile = state.authority?.profile ?? AD_HOC_AUTHORITY_PROFILE;
|
|
779
|
+
const tokens = state.tokens ?? zeroTokens();
|
|
780
|
+
const replayedTokens = state.replayedTokens ?? zeroTokens();
|
|
781
|
+
const cost = state.cost ?? 0;
|
|
782
|
+
const replayedCost = state.replayedCost ?? 0;
|
|
783
|
+
const integrationDiagnostics = integrationDiagnosticsForState(state);
|
|
784
|
+
const redacted = {
|
|
785
|
+
id: state.id,
|
|
786
|
+
status: state.status,
|
|
787
|
+
sourcePath: state.sourcePath,
|
|
788
|
+
sourceHash: state.sourceHash,
|
|
789
|
+
meta: redactValue(state.meta),
|
|
790
|
+
declaredProfile: declaredProfileForState(state),
|
|
791
|
+
effectiveAuthorityProfile,
|
|
792
|
+
authority: state.authority,
|
|
793
|
+
argsPreview: state.argsPreview,
|
|
794
|
+
startedAt: state.startedAt,
|
|
795
|
+
resumedAt: state.resumedAt,
|
|
796
|
+
finishedAt: state.finishedAt,
|
|
797
|
+
lastProgressAt: lastProgressAtForState(state),
|
|
798
|
+
staleness: stalenessSignal(state, lastProgressAtForState(state)),
|
|
799
|
+
lastEventAt: state.lastEventAt,
|
|
800
|
+
lastEventType: state.lastEventType,
|
|
801
|
+
currentPhase: state.currentPhase,
|
|
802
|
+
agentsStarted: state.agentsStarted,
|
|
803
|
+
maxAgents: state.maxAgents,
|
|
804
|
+
concurrency: state.concurrency,
|
|
805
|
+
laneTimeoutMs: state.laneTimeoutMs ?? DEFAULT_CHILD_PROMPT_TIMEOUT_MS,
|
|
806
|
+
maxRuntimeMs: state.maxRuntimeMs ?? null,
|
|
807
|
+
defaultChildModel: state.defaultChildModel,
|
|
808
|
+
activeAgents: state.activeAgents,
|
|
809
|
+
queuedAgents: state.queuedAgents,
|
|
810
|
+
tokens,
|
|
811
|
+
replayedTokens,
|
|
812
|
+
cost,
|
|
813
|
+
replayedCost,
|
|
814
|
+
usage: {
|
|
815
|
+
liveTokens: tokens,
|
|
816
|
+
replayedTokens,
|
|
817
|
+
totalTokens: addTokens(tokens, replayedTokens),
|
|
818
|
+
liveCost: cost,
|
|
819
|
+
replayedCost,
|
|
820
|
+
totalCost: cost + replayedCost,
|
|
821
|
+
},
|
|
822
|
+
cacheStats: state.cacheStats,
|
|
823
|
+
budgetCeilings: state.budgetCeilings,
|
|
824
|
+
autoApproved: state.autoApproved,
|
|
825
|
+
debugCapture: state.debugCapture,
|
|
826
|
+
operatorMetrics: operatorMetricsForState(state),
|
|
827
|
+
laneOutcomes: state.laneOutcomes,
|
|
828
|
+
droppedLaneCount: state.droppedLaneCount,
|
|
829
|
+
capabilities: state.capabilities,
|
|
830
|
+
diagnostics: integrationDiagnostics ? { ...(state.diagnostics ?? {}), integration: integrationDiagnostics } : state.diagnostics,
|
|
831
|
+
durability: state.durability,
|
|
832
|
+
recovery: state.recovery,
|
|
833
|
+
salvageCandidates: entry.salvageCandidates,
|
|
834
|
+
activeLanes: activeLanes.slice(0, MAX_ACTIVE_LANES),
|
|
835
|
+
activeLaneCount: activeLanes.length,
|
|
836
|
+
...(activeLanes.length > MAX_ACTIVE_LANES ? { activeLanesTruncated: activeLanes.length - MAX_ACTIVE_LANES } : {}),
|
|
837
|
+
laneRecords: state.laneRecords,
|
|
838
|
+
closeout: state.closeout,
|
|
839
|
+
domainFinalization: state.domainFinalization,
|
|
840
|
+
worktreeCleanup: state.worktreeCleanup,
|
|
841
|
+
locks: state.locks,
|
|
842
|
+
staleLocksCleared: state.staleLocksCleared,
|
|
843
|
+
lifecycleRequests: state.lifecycleRequests,
|
|
844
|
+
notification: await notificationStatusForEntry(entry, state),
|
|
845
|
+
nestedSnapshots: state.nestedSnapshots,
|
|
846
|
+
background: state.background,
|
|
847
|
+
resultPath: state.resultPath,
|
|
848
|
+
errorSummary: state.error ? truncateText(redactFreeTextSecrets(state.error), MAX_STATUS_STRING_CHARS) : undefined,
|
|
849
|
+
timeoutRecovery: state.status === "timed-out" ? timeoutResumeEligibilityForState(state) : undefined,
|
|
850
|
+
nextActions: nextActionsForEntry(entry),
|
|
851
|
+
root: entry.root,
|
|
852
|
+
dir: entry.dir,
|
|
853
|
+
};
|
|
854
|
+
if (state.editPlan) {
|
|
855
|
+
redacted.editWorktrees = state.editWorktrees;
|
|
856
|
+
redacted.editPlan = { ...state.editPlan, patches: undefined };
|
|
857
|
+
}
|
|
858
|
+
if (state.integrationPlan) {
|
|
859
|
+
redacted.integrationWorktrees = state.integrationWorktrees;
|
|
860
|
+
redacted.integrationPlan = redactedIntegrationPlan(state);
|
|
861
|
+
}
|
|
862
|
+
return redacted;
|
|
863
|
+
}
|
|
864
|
+
|
|
865
|
+
async function statusForEntry(entry, detail = "compact") {
|
|
866
|
+
if (detail === "full") return await fullStatusForEntry(entry);
|
|
867
|
+
if (detail === "result") return await resultStatusForEntry(entry);
|
|
868
|
+
return compactStatusForEntry(entry);
|
|
869
|
+
}
|
|
870
|
+
|
|
871
|
+
async function listRunEntries(context, options = {}) {
|
|
872
|
+
const entries = [];
|
|
873
|
+
for (const root of runRoots(context)) {
|
|
874
|
+
if (!(await pathExists(root))) continue;
|
|
875
|
+
const dirents = await fs.readdir(root, { withFileTypes: true });
|
|
876
|
+
for (const dirent of dirents) {
|
|
877
|
+
if (!dirent.isDirectory() && !dirent.isSymbolicLink()) continue;
|
|
878
|
+
try {
|
|
879
|
+
entries.push(await readRunEntry(root, dirent.name, options));
|
|
880
|
+
} catch (error) {
|
|
881
|
+
entries.push({ id: dirent.name, root, dir: path.resolve(root, dirent.name), status: "malformed", kind: "corrupt", error: extractTextFromError(error) });
|
|
882
|
+
}
|
|
883
|
+
}
|
|
884
|
+
}
|
|
885
|
+
entries.sort((a, b) => {
|
|
886
|
+
const aStarted = a.state?.startedAt ?? "";
|
|
887
|
+
const bStarted = b.state?.startedAt ?? "";
|
|
888
|
+
return String(bStarted).localeCompare(String(aStarted));
|
|
889
|
+
});
|
|
890
|
+
return entries;
|
|
891
|
+
}
|
|
892
|
+
|
|
893
|
+
function cleanupProtectionReason(entry, { now = Date.now(), interruptedTtlMs = INTERRUPTED_RUN_TTL_MS } = {}) {
|
|
894
|
+
if (entry.kind !== "valid") return "corrupt-or-partial";
|
|
895
|
+
const status = entry.state.status;
|
|
896
|
+
if (Object.keys(entry.state.locks ?? {}).length > 0) return "locked";
|
|
897
|
+
if (runs.has(entry.state.id)) return "active-in-process";
|
|
898
|
+
if (entry.state.pinned === true) return "pinned";
|
|
899
|
+
if (ACTIVE_STATUSES.has(status)) return "active-status";
|
|
900
|
+
if (AMBIGUOUS_EDIT_STATUSES.has(status)) return "ambiguous-edit-status";
|
|
901
|
+
if (status === "interrupted") {
|
|
902
|
+
// Normally protected so a future process can salvage its completed-lane work. But a
|
|
903
|
+
// permanently-unresumable interrupted run would otherwise pin its run dir forever, so
|
|
904
|
+
// once it has been idle longer than the TTL we drop protection and let cleanup reap it.
|
|
905
|
+
// Stay conservative: only release when we can PROVE it is old (a parseable, finite age
|
|
906
|
+
// past the TTL); an unknown/unparseable last-progress timestamp keeps the protection.
|
|
907
|
+
const lastProgressAt = lastProgressAtForState(entry.state);
|
|
908
|
+
const parsed = lastProgressAt ? Date.parse(lastProgressAt) : Number.NaN;
|
|
909
|
+
const ageMs = Number.isFinite(parsed) ? now - parsed : null;
|
|
910
|
+
if (ageMs !== null && ageMs > interruptedTtlMs) return undefined;
|
|
911
|
+
return "interrupted-recovery";
|
|
912
|
+
}
|
|
913
|
+
// A paused run is resumable: a concurrent OpenCode process can re-acquire its run dir
|
|
914
|
+
// and run.lock at any time. It releases its run.lock and leaves the in-memory map while
|
|
915
|
+
// paused, so neither the "locked" nor the "active-in-process" guard above catches it.
|
|
916
|
+
// Deleting it mid-resume would destroy a live run, so protect it explicitly.
|
|
917
|
+
if (status === "paused") return "paused-resumable";
|
|
918
|
+
if (status === "failed" || status === "budget_stopped") return "resumable-run";
|
|
919
|
+
if (status === "apply-failed") return "retryable-apply-failed";
|
|
920
|
+
return undefined;
|
|
921
|
+
}
|
|
922
|
+
|
|
923
|
+
function summarizeEntries(entries) {
|
|
924
|
+
if (entries.length === 0) return "No workflow runs found.";
|
|
925
|
+
return entries.flatMap((entry) => {
|
|
926
|
+
const nextLines = nextActionsForEntry(entry).map((action) => ` next: ${action}`);
|
|
927
|
+
if (entry.kind !== "valid") return [`${entry.id} ${entry.status} ${entry.dir}: ${truncateText(redactFreeTextSecrets(entry.error ?? ""), 160)}`, ...nextLines];
|
|
928
|
+
const state = entry.state;
|
|
929
|
+
const outcomes = state.laneOutcomes ? Object.entries(state.laneOutcomes).map(([key, value]) => `${key}:${value}`).join(" ") : "outcomes:unknown";
|
|
930
|
+
const cache = state.cacheStats ? `cache hit:${state.cacheStats.hits} miss:${state.cacheStats.misses} invalidated:${state.cacheStats.invalidated}` : "cache:unknown";
|
|
931
|
+
const staleness = stalenessSignal(state, lastProgressAtForState(state));
|
|
932
|
+
const line = [
|
|
933
|
+
`${state.id} ${state.status}`,
|
|
934
|
+
`effectiveProfile=${state.authority?.profile || AD_HOC_AUTHORITY_PROFILE}`,
|
|
935
|
+
`phase=${state.currentPhase || "-"}`,
|
|
936
|
+
`agents=${state.agentsStarted || 0}/${state.maxAgents || 0}`,
|
|
937
|
+
`active=${state.activeAgents || 0}`,
|
|
938
|
+
`dropped=${state.droppedLaneCount || 0}`,
|
|
939
|
+
outcomes,
|
|
940
|
+
cache,
|
|
941
|
+
staleness.stale ? `STALE no-progress>${Math.round(staleness.ageMs / 1000)}s` : "",
|
|
942
|
+
state.error ? `error=${truncateText(redactFreeTextSecrets(state.error), 160)}` : "",
|
|
943
|
+
].filter(Boolean).join(" ");
|
|
944
|
+
// Name each failed lane + its error class in the human view so an operator sees WHICH lane
|
|
945
|
+
// failed and WHY without expanding to detail=full — the text mirror of compact.laneFailures.
|
|
946
|
+
const failureLines = laneFailureSummaries(state)
|
|
947
|
+
.slice(0, MAX_COMPACT_LANE_FAILURES)
|
|
948
|
+
.map((lane) => ` lane-failed ${lane.callId} [${lane.outcome}${lane.failureClass ? `/${lane.failureClass}` : ""}]${lane.role ? ` role=${lane.role}` : ""}${lane.errorSummary ? `: ${truncateText(redactFreeTextSecrets(lane.errorSummary), 160)}` : ""}`);
|
|
949
|
+
const salvageLines = (entry.salvageCandidates ?? []).map((candidate) => ` salvage ${candidate.callId}: ${candidate.hint}`);
|
|
950
|
+
return [line, ...failureLines, ...salvageLines, ...nextLines];
|
|
951
|
+
}).join("\n");
|
|
952
|
+
}
|
|
953
|
+
|
|
954
|
+
async function statusText(context, args, optionsOverride = {}) {
|
|
955
|
+
if (typeof args.runId === "string" && args.runId.trim() === "" && args.detail === "result") {
|
|
956
|
+
throw new Error("workflow_status detail=result requires a concrete runId");
|
|
957
|
+
}
|
|
958
|
+
const runId = (typeof args.runId === "string" && args.runId.trim()) ? assertSafeRunId(args.runId.trim()) : undefined;
|
|
959
|
+
const format = args.format || "summary";
|
|
960
|
+
const detail = args.detail || "compact";
|
|
961
|
+
if (!runId && detail === "result") throw new Error("workflow_status detail=result requires runId");
|
|
962
|
+
if (args.reconcile === true && optionsOverride.allowReconcile !== true) {
|
|
963
|
+
throw new WorkflowAuthorityError("workflow_status is read-only. For recovery, use workflow_reconcile.");
|
|
964
|
+
}
|
|
965
|
+
const reconcile = args.reconcile === true && optionsOverride.allowReconcile === true;
|
|
966
|
+
const options = { reconcile, clearStaleLocks: reconcile };
|
|
967
|
+
let entries;
|
|
968
|
+
if (runId) {
|
|
969
|
+
entries = [];
|
|
970
|
+
for (const root of runRoots(context)) {
|
|
971
|
+
const dir = runDirForRoot(root, runId);
|
|
972
|
+
if (await pathExists(dir)) entries.push(await readRunEntry(root, runId, options));
|
|
973
|
+
}
|
|
974
|
+
if (entries.length === 0) throw new Error(`Workflow run not found: ${runId}`);
|
|
975
|
+
} else {
|
|
976
|
+
entries = await listRunEntries(context, options);
|
|
977
|
+
if (args.includePendingApproval !== true) entries = entries.filter((entry) => entry.state?.status !== "pending-approval");
|
|
978
|
+
entries = entries.slice(0, args.limit || 20);
|
|
979
|
+
}
|
|
980
|
+
// Crash-resource reclamation (opencode-workflows-jbs3.9): reconcile is the recovery pass that
|
|
981
|
+
// confirms a dead run interrupted via PID-liveness. Once a run is confirmed interrupted, prune
|
|
982
|
+
// the stranded git worktrees/branches it left behind so they do not accumulate unbounded across
|
|
983
|
+
// autonomous-drain crashes. Read-only workflow_status never reaches this (reconcile is false).
|
|
984
|
+
if (reconcile) await reclaimStrandedWorktrees(context, entries);
|
|
985
|
+
const redacted = await Promise.all(entries.map((entry) => statusForEntry(entry, detail)));
|
|
986
|
+
if (format === "json") return JSON.stringify(runId ? redacted[0] : redacted, null, 2);
|
|
987
|
+
return summarizeEntries(entries);
|
|
988
|
+
}
|
|
989
|
+
|
|
990
|
+
// Best-effort reclamation of git worktrees/branches stranded by crashed runs. Only acts on runs
|
|
991
|
+
// confirmed interrupted (dead via PID-liveness in readRunEntry) that recorded worktrees. Builds a
|
|
992
|
+
// worktree adapter lazily — runs without worktree records (and non-git working dirs) never require
|
|
993
|
+
// a git repo. Conservative by construction: adapter.remove() refuses dirty/locked/main/out-of-root
|
|
994
|
+
// worktrees, so in-flight lane work is preserved, not destroyed. Successfully removed records are
|
|
995
|
+
// dropped from the persisted run so a later reconcile does not re-attempt them; preserved records
|
|
996
|
+
// (dirty/locked) are kept so a future reconcile can retry once they become clean.
|
|
997
|
+
async function reclaimStrandedWorktrees(context, entries) {
|
|
998
|
+
const repoDir = context?.worktree || context?.directory;
|
|
999
|
+
if (!repoDir) return;
|
|
1000
|
+
let adapter;
|
|
1001
|
+
for (const entry of entries) {
|
|
1002
|
+
if (entry.kind !== "valid" || entry.state?.status !== "interrupted") continue;
|
|
1003
|
+
const edit = Array.isArray(entry.state.editWorktrees) ? entry.state.editWorktrees : [];
|
|
1004
|
+
const integration = Array.isArray(entry.state.integrationWorktrees) ? entry.state.integrationWorktrees : [];
|
|
1005
|
+
if (edit.length === 0 && integration.length === 0) continue;
|
|
1006
|
+
if (adapter === undefined) {
|
|
1007
|
+
try { adapter = await createWorktreeAdapter({ directory: repoDir }); }
|
|
1008
|
+
catch { adapter = null; }
|
|
1009
|
+
}
|
|
1010
|
+
if (!adapter?.remove) continue;
|
|
1011
|
+
|
|
1012
|
+
const reclaimed = [];
|
|
1013
|
+
const survivors = async (records) => {
|
|
1014
|
+
const kept = [];
|
|
1015
|
+
for (const record of records) {
|
|
1016
|
+
if (!record?.path) { kept.push(record); continue; }
|
|
1017
|
+
let summary;
|
|
1018
|
+
try {
|
|
1019
|
+
const result = await adapter.remove(record);
|
|
1020
|
+
summary = { role: record.role, callId: record.callId, laneId: record.laneId, path: result.path ?? record.path, branch: record.branch, removed: result.removed === true, preserved: result.preserved === true, reason: result.reason, branchDeleted: result.branchDeleted };
|
|
1021
|
+
} catch (error) {
|
|
1022
|
+
summary = { role: record.role, callId: record.callId, laneId: record.laneId, path: record.path, branch: record.branch, removed: false, preserved: true, reason: "reclaim-failed", error: truncateText(extractTextFromError(error), MAX_STATUS_STRING_CHARS) };
|
|
1023
|
+
}
|
|
1024
|
+
reclaimed.push(summary);
|
|
1025
|
+
if (!summary.removed) kept.push(record);
|
|
1026
|
+
}
|
|
1027
|
+
return kept;
|
|
1028
|
+
};
|
|
1029
|
+
|
|
1030
|
+
const keptEdit = await survivors(edit);
|
|
1031
|
+
const keptIntegration = await survivors(integration);
|
|
1032
|
+
if (reclaimed.length === 0) continue;
|
|
1033
|
+
entry.state.editWorktrees = keptEdit;
|
|
1034
|
+
entry.state.integrationWorktrees = keptIntegration;
|
|
1035
|
+
entry.state.worktreeCleanup = {
|
|
1036
|
+
...(entry.state.worktreeCleanup ?? {}),
|
|
1037
|
+
reclaimed: [...(entry.state.worktreeCleanup?.reclaimed ?? []), ...reclaimed],
|
|
1038
|
+
};
|
|
1039
|
+
try { await writeJsonAtomic(path.join(entry.dir, "state.json"), entry.state); } catch { /* best effort */ }
|
|
1040
|
+
}
|
|
1041
|
+
}
|
|
1042
|
+
|
|
1043
|
+
async function reconcileRuns(context, args = {}) {
|
|
1044
|
+
assertWriteWorkflowAllowed(context, "workflow_reconcile");
|
|
1045
|
+
return await statusText(context, { ...args, reconcile: true }, { allowReconcile: true });
|
|
1046
|
+
}
|
|
1047
|
+
|
|
1048
|
+
async function cleanupRuns(context, args = {}) {
|
|
1049
|
+
assertWriteWorkflowAllowed(context, "workflow_cleanup");
|
|
1050
|
+
const keep = Number.isInteger(args.keep) && args.keep >= 0 ? args.keep : DEFAULT_KEEP_RUNS;
|
|
1051
|
+
const dryRun = args.dryRun !== false;
|
|
1052
|
+
// Configurable interrupted-run TTL escape valve: any positive integer overrides the default.
|
|
1053
|
+
const interruptedTtlMs = Number.isInteger(args.interruptedTtlMs) && args.interruptedTtlMs > 0 ? args.interruptedTtlMs : INTERRUPTED_RUN_TTL_MS;
|
|
1054
|
+
// Snapshot "now" once so every protection decision in this pass uses a single clock.
|
|
1055
|
+
const now = Date.now();
|
|
1056
|
+
const protectionOpts = { now, interruptedTtlMs };
|
|
1057
|
+
const cleanupReleases = [];
|
|
1058
|
+
if (!dryRun) {
|
|
1059
|
+
for (const root of runRoots(context)) {
|
|
1060
|
+
if (await pathExists(root)) cleanupReleases.push(await acquireWorkflowLock(cleanupLockPath(root), { operation: "cleanup", root }));
|
|
1061
|
+
}
|
|
1062
|
+
}
|
|
1063
|
+
try {
|
|
1064
|
+
const entries = await listRunEntries(context, { reconcile: true });
|
|
1065
|
+
const deletable = [];
|
|
1066
|
+
let keptValid = 0;
|
|
1067
|
+
const protectedRuns = [];
|
|
1068
|
+
for (const entry of entries) {
|
|
1069
|
+
const reason = cleanupProtectionReason(entry, protectionOpts);
|
|
1070
|
+
if (reason) {
|
|
1071
|
+
protectedRuns.push({ id: entry.id, status: entry.status, dir: entry.dir, reason, ...(entry.state?.locks ? { locks: entry.state.locks } : {}), ...(entry.error ? { error: truncateText(redactFreeTextSecrets(entry.error), MAX_STATUS_STRING_CHARS) } : {}) });
|
|
1072
|
+
continue;
|
|
1073
|
+
}
|
|
1074
|
+
keptValid += 1;
|
|
1075
|
+
if (keptValid > keep) deletable.push(entry);
|
|
1076
|
+
}
|
|
1077
|
+
const protectedContainment = [];
|
|
1078
|
+
const protectedRevalidated = [];
|
|
1079
|
+
const deleted = [];
|
|
1080
|
+
if (!dryRun) {
|
|
1081
|
+
for (const entry of deletable) {
|
|
1082
|
+
try {
|
|
1083
|
+
await assertContainedRunDir(entry.root, entry.dir);
|
|
1084
|
+
// TOCTOU guard: cleanup.lock only serializes cleanup-vs-cleanup, not cleanup-vs-resume.
|
|
1085
|
+
// Between enumeration and this fs.rm a concurrent process can resume a paused/eligible
|
|
1086
|
+
// run — re-acquiring its run.lock and flipping status to running. Re-read the entry and
|
|
1087
|
+
// re-check protection immediately before delete; skip+report anything that became
|
|
1088
|
+
// protected (re-acquired a lock or changed status) so a live run is never deleted.
|
|
1089
|
+
const current = await readRunEntry(entry.root, entry.id, { reconcile: true });
|
|
1090
|
+
const reason = cleanupProtectionReason(current, protectionOpts);
|
|
1091
|
+
if (reason) {
|
|
1092
|
+
protectedRevalidated.push({ id: current.id, status: current.status, dir: current.dir, reason, ...(current.state?.locks ? { locks: current.state.locks } : {}) });
|
|
1093
|
+
continue;
|
|
1094
|
+
}
|
|
1095
|
+
await fs.rm(entry.dir, { recursive: true, force: true });
|
|
1096
|
+
deleted.push(entry);
|
|
1097
|
+
} catch (error) {
|
|
1098
|
+
protectedContainment.push({ id: entry.id, status: entry.status, dir: entry.dir, error: truncateText(extractTextFromError(error), MAX_STATUS_STRING_CHARS) });
|
|
1099
|
+
}
|
|
1100
|
+
}
|
|
1101
|
+
}
|
|
1102
|
+
const deletedDirs = dryRun ? deletable.map((entry) => entry.dir) : deleted.map((entry) => entry.dir);
|
|
1103
|
+
const payload = {
|
|
1104
|
+
dryRun,
|
|
1105
|
+
keep,
|
|
1106
|
+
deleteCount: deletedDirs.length,
|
|
1107
|
+
deleteDirs: deletedDirs,
|
|
1108
|
+
protectedRuns,
|
|
1109
|
+
protectedLocked: protectedRuns.filter((entry) => entry.reason === "locked"),
|
|
1110
|
+
protectedCorruptOrPartial: protectedRuns.filter((entry) => entry.reason === "corrupt-or-partial").map((entry) => ({ id: entry.id, status: entry.status, dir: entry.dir, error: entry.error })),
|
|
1111
|
+
protectedContainment,
|
|
1112
|
+
protectedRevalidated,
|
|
1113
|
+
};
|
|
1114
|
+
return JSON.stringify(payload, null, 2);
|
|
1115
|
+
} finally {
|
|
1116
|
+
for (const release of cleanupReleases.reverse()) await release();
|
|
1117
|
+
}
|
|
1118
|
+
}
|
|
1119
|
+
|
|
1120
|
+
async function readRunById(context, runId) {
|
|
1121
|
+
assertSafeRunId(runId);
|
|
1122
|
+
for (const root of runRoots(context)) {
|
|
1123
|
+
const dir = runDirForRoot(root, runId);
|
|
1124
|
+
if (await pathExists(dir)) return await readRunEntry(root, runId);
|
|
1125
|
+
}
|
|
1126
|
+
throw new Error(`Workflow run not found: ${runId}`);
|
|
1127
|
+
}
|
|
1128
|
+
|
|
1129
|
+
export {
|
|
1130
|
+
STALE_PROGRESS_THRESHOLD_MS,
|
|
1131
|
+
INTERRUPTED_RUN_TTL_MS,
|
|
1132
|
+
lastProgressAtForState,
|
|
1133
|
+
stalenessSignal,
|
|
1134
|
+
laneFailureSummaries,
|
|
1135
|
+
activeLaneSummaries,
|
|
1136
|
+
timeoutResumeEligibilityForState,
|
|
1137
|
+
readRunEntry,
|
|
1138
|
+
nextActionsForEntry,
|
|
1139
|
+
compactStatusForEntry,
|
|
1140
|
+
declaredProfileForState,
|
|
1141
|
+
resultStatusForEntry,
|
|
1142
|
+
eventsForEntry,
|
|
1143
|
+
eventsText,
|
|
1144
|
+
notificationStatusForEntry,
|
|
1145
|
+
fullStatusForEntry,
|
|
1146
|
+
statusForEntry,
|
|
1147
|
+
listRunEntries,
|
|
1148
|
+
cleanupProtectionReason,
|
|
1149
|
+
summarizeEntries,
|
|
1150
|
+
statusText,
|
|
1151
|
+
reconcileRuns,
|
|
1152
|
+
cleanupRuns,
|
|
1153
|
+
readRunById,
|
|
1154
|
+
};
|