@tea-agent/loop-agent 0.36.1-beta.0 → 0.36.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +11 -12
- package/dist/build-stamp.json +3 -3
- package/dist/cli/command-definitions.js +7 -0
- package/dist/cli/program.js +6 -1
- package/dist/commands/dag-request-interrupt.js +20 -0
- package/dist/executors/pi-sdk-executor.js +34 -0
- package/dist/shared/operator/capabilities.js +76 -3
- package/dist/task/source-prepare/semantic-intake.js +144 -23
- package/dist/worker/console/chat/semantic-activity.js +6 -0
- package/dist/worker/console/chat/workspace-landing.js +1 -0
- package/dist/worker/console/operation-runner.js +48 -0
- package/dist/worker/console/operation-wait.js +41 -0
- package/dist/worker/console/operator-actions.js +235 -26
- package/dist/worker/console/operator-user-error.js +4 -0
- package/dist/worker/console/recovery-cta.js +50 -3
- package/dist/worker/console/recovery-error-copy.js +198 -0
- package/dist/worker/console/static/assets/index-D83DYAFG.css +1 -0
- package/dist/worker/console/static/assets/index-IXm7oYjL.js +59 -0
- package/dist/worker/console/static/index.html +2 -2
- package/dist/worker/console/static-src/app/console-types.js +13 -9
- package/dist/worker/console/static-src/app/useOperatorActions.js +4 -2
- package/dist/worker/console/static-src/app/useRecoveryActions.js +65 -56
- package/dist/worker/console/static-src/app/useRecoveryConsole.js +68 -1
- package/dist/worker/observability/interrupt-eligibility.js +264 -0
- package/dist/worker/observe/static/operator-chrome.d.ts +1 -0
- package/dist/worker/observe/static/operator-chrome.js +5 -0
- package/dist/worker/observe/static/views/dag.js +12 -0
- package/dist/workflows/dag/failure-routing.js +4 -9
- package/dist/workflows/dag/init-hybrid.js +2 -4
- package/dist/workflows/dag/interrupt-request.js +559 -0
- package/dist/workflows/dag/lifecycle.js +0 -4
- package/dist/workflows/dag/node-execution.js +6 -16
- package/dist/workflows/dag/report.js +0 -6
- package/dist/workflows/dag/retry-policy.js +0 -11
- package/dist/workflows/dag/runner.js +72 -4
- package/docs/templates/frontend-task-constraints.md +7 -13
- package/package.json +1 -1
- package/skills/loop-agent/references/command-reference.md +1 -0
- package/dist/worker/console/static/assets/index-2OeZODxk.js +0 -57
- package/dist/worker/console/static/assets/index-DVJlUL8X.css +0 -1
|
@@ -355,6 +355,18 @@ export async function renderDagDetail(dagRunId, initial = true) {
|
|
|
355
355
|
primaryEntries.push(["状态一致性", "不一致,需要处理"]);
|
|
356
356
|
if (visibility.showResume) primaryEntries.push(["允许恢复", "是"]);
|
|
357
357
|
if (visibility.showReconcile) primaryEntries.push(["允许收口", "是"]);
|
|
358
|
+
const liveStatus = String(dag.effectiveStatus ?? dag.status ?? "").toLowerCase();
|
|
359
|
+
if (liveStatus === "running" || liveStatus === "active") {
|
|
360
|
+
const interruptLink = el(
|
|
361
|
+
"a",
|
|
362
|
+
"page-action-btn",
|
|
363
|
+
"在操作台中止此运行 →",
|
|
364
|
+
);
|
|
365
|
+
interruptLink.href = `/#/recovery?dagRunId=${encodeURIComponent(dag.dagRunId)}&intent=interrupt`;
|
|
366
|
+
interruptLink.setAttribute("data-testid", "inspect-dag-interrupt-link");
|
|
367
|
+
interruptLink.title = "只跳转到操作台 Recovery,不会在观测页执行中止";
|
|
368
|
+
primaryEntries.push(["协作中止", interruptLink]);
|
|
369
|
+
}
|
|
358
370
|
primaryEntries.push(
|
|
359
371
|
["开始时间", formatTs(dag.startedAt)],
|
|
360
372
|
["结束时间", formatTs(dag.finishedAt)],
|
|
@@ -44,9 +44,7 @@ function routeToProductLine(input) {
|
|
|
44
44
|
case "decision-envelope":
|
|
45
45
|
return "NeedsHuman";
|
|
46
46
|
case "skipped":
|
|
47
|
-
return
|
|
48
|
-
? "ContractMismatch"
|
|
49
|
-
: "DependencyFailure";
|
|
47
|
+
return "DependencyFailure";
|
|
50
48
|
case "shell-command":
|
|
51
49
|
if (/\b(ENOENT|PATH|command not found|No such file|not found in PATH|bash: .*: No such file)\b/i.test(raw)) {
|
|
52
50
|
return "EnvFailure";
|
|
@@ -92,12 +90,9 @@ export function routeDagFailure(input) {
|
|
|
92
90
|
if (!productLineFailureCategory)
|
|
93
91
|
return {};
|
|
94
92
|
const recommendedFollowUp = productLineFailureCategory === "ContractMismatch" &&
|
|
95
|
-
input.
|
|
96
|
-
? "
|
|
97
|
-
: productLineFailureCategory
|
|
98
|
-
isFrontendDesignGateNode(input.nodeId ?? "")
|
|
99
|
-
? "frontend-plan-revision-and-rerun"
|
|
100
|
-
: FOLLOW_UP_BY_PRODUCT_LINE[productLineFailureCategory];
|
|
93
|
+
isFrontendDesignGateNode(input.nodeId ?? "")
|
|
94
|
+
? "frontend-plan-revision-and-rerun"
|
|
95
|
+
: FOLLOW_UP_BY_PRODUCT_LINE[productLineFailureCategory];
|
|
101
96
|
return {
|
|
102
97
|
productLineFailureCategory,
|
|
103
98
|
recommendedFollowUp,
|
|
@@ -9,7 +9,7 @@ import { planMavenVerification, } from "../../verification/maven/index.js";
|
|
|
9
9
|
import { pathMatchesPattern } from "../../shared/git-progress.js";
|
|
10
10
|
import { BASELINE_FORBIDDEN_PATHS } from "./governance-constants.js";
|
|
11
11
|
import { buildDecisionEnvelopePromptContract } from "./decision-envelope.js";
|
|
12
|
-
import { DEFAULT_READ_ONLY_PI_RETRY_POLICY,
|
|
12
|
+
import { DEFAULT_READ_ONLY_PI_RETRY_POLICY, PROTOCOL_AWARE_PI_RETRY_POLICY, STRUCTURED_REQUIRED_PI_RETRY_POLICY, BACKEND_TEST_WRITER_COMPLETENESS_RETRY_POLICY, WRITER_TRANSPORT_RETRY_POLICY, isSafeReadOnlyPiRetryCandidate, isWriterTransportRetryCandidate, } from "./retry-policy.js";
|
|
13
13
|
import { REVIEW_JSON_VERDICT_OUTPUT_PROTOCOL, REVIEW_VERDICT_OUTPUT_PROTOCOL, } from "./output-protocol.js";
|
|
14
14
|
import { resolveAdapter } from "../../adapters/index.js";
|
|
15
15
|
import { loadHarnessManifest } from "../../governance/harness.js";
|
|
@@ -5783,9 +5783,7 @@ function applyDefaultReadOnlyRetryPolicy(spec) {
|
|
|
5783
5783
|
if (isSafeReadOnlyPiRetryCandidate(task)) {
|
|
5784
5784
|
task.retryPolicy = task.outputProtocol
|
|
5785
5785
|
? PROTOCOL_AWARE_PI_RETRY_POLICY
|
|
5786
|
-
:
|
|
5787
|
-
? PLANNER_OUTPUT_LIMIT_RETRY_POLICY
|
|
5788
|
-
: DEFAULT_READ_ONLY_PI_RETRY_POLICY;
|
|
5786
|
+
: DEFAULT_READ_ONLY_PI_RETRY_POLICY;
|
|
5789
5787
|
continue;
|
|
5790
5788
|
}
|
|
5791
5789
|
if (isWriterTransportRetryCandidate(task)) {
|
|
@@ -0,0 +1,559 @@
|
|
|
1
|
+
import { watch } from "node:fs";
|
|
2
|
+
import { mkdir, readFile } from "node:fs/promises";
|
|
3
|
+
import { hostname as localHostname } from "node:os";
|
|
4
|
+
import path from "node:path";
|
|
5
|
+
import { writeJsonAtomic } from "../../infrastructure/harness/atomic-write.js";
|
|
6
|
+
import { assessDagRunLiveness, getDagRunDir, locateDagRun, readDagRunState, } from "./lifecycle.js";
|
|
7
|
+
export const INTERRUPT_ARTIFACT_REL = path.posix.join(".runtime", "interrupt.json");
|
|
8
|
+
export const INTERRUPT_REASON_CODES = [
|
|
9
|
+
"requirements-changed",
|
|
10
|
+
"suspected-runaway",
|
|
11
|
+
"resource-protection",
|
|
12
|
+
"operator-request",
|
|
13
|
+
"other",
|
|
14
|
+
];
|
|
15
|
+
export const INTERRUPT_ERROR_CODES = [
|
|
16
|
+
"RUN_NOT_ACTIVE",
|
|
17
|
+
"RUN_NOT_CONSOLE_OWNED",
|
|
18
|
+
"RUNNER_IDENTITY_MISMATCH",
|
|
19
|
+
"INTERRUPT_ALREADY_PENDING",
|
|
20
|
+
"RUN_ALREADY_TERMINAL",
|
|
21
|
+
"INTERRUPT_NEEDS_RECONCILE",
|
|
22
|
+
"INVALID_INPUT",
|
|
23
|
+
];
|
|
24
|
+
export const INTERRUPT_ERROR_MESSAGES = {
|
|
25
|
+
RUN_NOT_ACTIVE: "这次运行不在本机活动目录,或已经不是 running,不能协作中止。",
|
|
26
|
+
RUN_NOT_CONSOLE_OWNED: "此运行不由当前 Console 持有,暂不支持协作中止。请用 Doctor / 报告查看。",
|
|
27
|
+
RUNNER_IDENTITY_MISMATCH: "运行身份已变化,不能中止这份过期请求。请刷新资格后再试。",
|
|
28
|
+
INTERRUPT_ALREADY_PENDING: "已有一条中止请求等待结算。请等待当前节点安全收敛,或用 Doctor 对账。",
|
|
29
|
+
RUN_ALREADY_TERMINAL: "运行已经结束,没有可中止的活 runner。",
|
|
30
|
+
INTERRUPT_NEEDS_RECONCILE: "中止请求已登记,但无法确认 runner 已处理。请用 Doctor 对账,不要当成已经中止。",
|
|
31
|
+
INVALID_INPUT: "中止请求参数不完整或格式不正确。",
|
|
32
|
+
};
|
|
33
|
+
export class DagInterruptError extends Error {
|
|
34
|
+
code;
|
|
35
|
+
constructor(code, message) {
|
|
36
|
+
super(message ?? INTERRUPT_ERROR_MESSAGES[code]);
|
|
37
|
+
this.name = "DagInterruptError";
|
|
38
|
+
this.code = code;
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
export function interruptArtifactPath(runDir) {
|
|
42
|
+
return path.join(runDir, ".runtime", "interrupt.json");
|
|
43
|
+
}
|
|
44
|
+
export function isInterruptReasonCode(value) {
|
|
45
|
+
return INTERRUPT_REASON_CODES.includes(value);
|
|
46
|
+
}
|
|
47
|
+
export function normalizeInterruptReasonDetail(raw) {
|
|
48
|
+
const detail = raw.trim();
|
|
49
|
+
if (detail.length < 10 || detail.length > 500) {
|
|
50
|
+
throw new DagInterruptError("INVALID_INPUT", "reasonDetail 去首尾空白后必须是 10–500 个字符,且不能是无意义占位。");
|
|
51
|
+
}
|
|
52
|
+
return detail;
|
|
53
|
+
}
|
|
54
|
+
export function runnerIdentityFromState(state) {
|
|
55
|
+
const runner = state.runner;
|
|
56
|
+
const fingerprint = state.controllerIdentityRef?.packageFingerprint?.trim();
|
|
57
|
+
if (!runner || !fingerprint)
|
|
58
|
+
return undefined;
|
|
59
|
+
if (!Number.isInteger(runner.pid) ||
|
|
60
|
+
runner.pid <= 0 ||
|
|
61
|
+
!runner.hostname?.trim() ||
|
|
62
|
+
!runner.startedAt?.trim()) {
|
|
63
|
+
return undefined;
|
|
64
|
+
}
|
|
65
|
+
return {
|
|
66
|
+
pid: runner.pid,
|
|
67
|
+
hostname: runner.hostname,
|
|
68
|
+
startedAt: runner.startedAt,
|
|
69
|
+
controllerFingerprint: fingerprint,
|
|
70
|
+
};
|
|
71
|
+
}
|
|
72
|
+
export function runnerIdentitiesMatch(left, right) {
|
|
73
|
+
return (left.pid === right.pid &&
|
|
74
|
+
left.hostname === right.hostname &&
|
|
75
|
+
left.startedAt === right.startedAt &&
|
|
76
|
+
left.controllerFingerprint === right.controllerFingerprint);
|
|
77
|
+
}
|
|
78
|
+
export function parseInterruptRequest(raw) {
|
|
79
|
+
if (!raw || typeof raw !== "object") {
|
|
80
|
+
throw new DagInterruptError("INVALID_INPUT", "interrupt.json 不是对象");
|
|
81
|
+
}
|
|
82
|
+
const rec = raw;
|
|
83
|
+
if (rec.schemaVersion !== 1) {
|
|
84
|
+
throw new DagInterruptError("INVALID_INPUT", "interrupt.json schemaVersion 必须为 1");
|
|
85
|
+
}
|
|
86
|
+
const identity = rec.expectedRunnerIdentity;
|
|
87
|
+
if (!identity || typeof identity !== "object") {
|
|
88
|
+
throw new DagInterruptError("INVALID_INPUT", "interrupt.json 缺少 expectedRunnerIdentity");
|
|
89
|
+
}
|
|
90
|
+
const ident = identity;
|
|
91
|
+
const reasonCode = String(rec.reasonCode ?? "");
|
|
92
|
+
if (!isInterruptReasonCode(reasonCode)) {
|
|
93
|
+
throw new DagInterruptError("INVALID_INPUT", "interrupt.json reasonCode 非法");
|
|
94
|
+
}
|
|
95
|
+
const status = String(rec.status ?? "");
|
|
96
|
+
if (status !== "requested" &&
|
|
97
|
+
status !== "acknowledged" &&
|
|
98
|
+
status !== "settled" &&
|
|
99
|
+
status !== "needs-reconcile") {
|
|
100
|
+
throw new DagInterruptError("INVALID_INPUT", "interrupt.json status 非法");
|
|
101
|
+
}
|
|
102
|
+
const requestedBy = rec.requestedBy;
|
|
103
|
+
if (!requestedBy || typeof requestedBy !== "object") {
|
|
104
|
+
throw new DagInterruptError("INVALID_INPUT", "interrupt.json 缺少 requestedBy");
|
|
105
|
+
}
|
|
106
|
+
const by = requestedBy;
|
|
107
|
+
const source = by.source === "cli" ? "cli" : "operator-console";
|
|
108
|
+
const pid = Number(ident.pid);
|
|
109
|
+
if (!Number.isInteger(pid) || pid <= 0) {
|
|
110
|
+
throw new DagInterruptError("INVALID_INPUT", "interrupt.json expectedRunnerIdentity.pid 非法");
|
|
111
|
+
}
|
|
112
|
+
return {
|
|
113
|
+
schemaVersion: 1,
|
|
114
|
+
revision: Number(rec.revision) || 1,
|
|
115
|
+
requestId: String(rec.requestId ?? ""),
|
|
116
|
+
runId: String(rec.runId ?? ""),
|
|
117
|
+
targetOperationId: String(rec.targetOperationId ?? ""),
|
|
118
|
+
reasonCode,
|
|
119
|
+
reasonDetail: String(rec.reasonDetail ?? ""),
|
|
120
|
+
requestedAt: String(rec.requestedAt ?? ""),
|
|
121
|
+
requestedBy: {
|
|
122
|
+
source,
|
|
123
|
+
...(typeof by.operatorSessionId === "string" && by.operatorSessionId.trim()
|
|
124
|
+
? { operatorSessionId: by.operatorSessionId.trim() }
|
|
125
|
+
: {}),
|
|
126
|
+
},
|
|
127
|
+
expectedRunnerIdentity: {
|
|
128
|
+
pid,
|
|
129
|
+
hostname: String(ident.hostname ?? ""),
|
|
130
|
+
startedAt: String(ident.startedAt ?? ""),
|
|
131
|
+
controllerFingerprint: String(ident.controllerFingerprint ?? ""),
|
|
132
|
+
},
|
|
133
|
+
status,
|
|
134
|
+
...(typeof rec.acknowledgedAt === "string"
|
|
135
|
+
? { acknowledgedAt: rec.acknowledgedAt }
|
|
136
|
+
: {}),
|
|
137
|
+
...(typeof rec.settledAt === "string" ? { settledAt: rec.settledAt } : {}),
|
|
138
|
+
...(typeof rec.outcome === "string" ? { outcome: rec.outcome } : {}),
|
|
139
|
+
};
|
|
140
|
+
}
|
|
141
|
+
export async function readInterruptRequest(runDir) {
|
|
142
|
+
try {
|
|
143
|
+
const raw = JSON.parse(await readFile(interruptArtifactPath(runDir), "utf-8"));
|
|
144
|
+
return parseInterruptRequest(raw);
|
|
145
|
+
}
|
|
146
|
+
catch (error) {
|
|
147
|
+
if (error &&
|
|
148
|
+
typeof error === "object" &&
|
|
149
|
+
"code" in error &&
|
|
150
|
+
error.code === "ENOENT") {
|
|
151
|
+
return undefined;
|
|
152
|
+
}
|
|
153
|
+
if (error instanceof DagInterruptError)
|
|
154
|
+
throw error;
|
|
155
|
+
throw new DagInterruptError("INVALID_INPUT", `无法读取 interrupt.json: ${error instanceof Error ? error.message : String(error)}`);
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
async function writeInterruptRequest(runDir, request) {
|
|
159
|
+
await mkdir(path.join(runDir, ".runtime"), { recursive: true });
|
|
160
|
+
await writeJsonAtomic(interruptArtifactPath(runDir), request);
|
|
161
|
+
}
|
|
162
|
+
function requestPayloadKey(request) {
|
|
163
|
+
const ident = request.expectedRunnerIdentity;
|
|
164
|
+
return [
|
|
165
|
+
request.runId,
|
|
166
|
+
request.targetOperationId,
|
|
167
|
+
request.reasonCode,
|
|
168
|
+
request.reasonDetail,
|
|
169
|
+
ident.pid,
|
|
170
|
+
ident.hostname,
|
|
171
|
+
ident.startedAt,
|
|
172
|
+
ident.controllerFingerprint,
|
|
173
|
+
].join("\0");
|
|
174
|
+
}
|
|
175
|
+
function assertKernelRunEligible(input) {
|
|
176
|
+
if (input.lifecycle !== "active") {
|
|
177
|
+
if (input.lifecycle === "completed" ||
|
|
178
|
+
input.lifecycle === "paused" ||
|
|
179
|
+
isTerminalRunStatus(input.state.status)) {
|
|
180
|
+
throw new DagInterruptError("RUN_ALREADY_TERMINAL");
|
|
181
|
+
}
|
|
182
|
+
throw new DagInterruptError("RUN_NOT_ACTIVE");
|
|
183
|
+
}
|
|
184
|
+
if (input.state.status !== "running") {
|
|
185
|
+
if (isTerminalRunStatus(input.state.status) || input.state.status === "paused") {
|
|
186
|
+
throw new DagInterruptError("RUN_ALREADY_TERMINAL");
|
|
187
|
+
}
|
|
188
|
+
throw new DagInterruptError("RUN_NOT_ACTIVE");
|
|
189
|
+
}
|
|
190
|
+
if (input.state.workerAssociation) {
|
|
191
|
+
throw new DagInterruptError("RUN_NOT_CONSOLE_OWNED");
|
|
192
|
+
}
|
|
193
|
+
const live = runnerIdentityFromState(input.state);
|
|
194
|
+
if (!live || !runnerIdentitiesMatch(live, input.expected)) {
|
|
195
|
+
throw new DagInterruptError("RUNNER_IDENTITY_MISMATCH");
|
|
196
|
+
}
|
|
197
|
+
const liveness = assessDagRunLiveness({ state: input.state });
|
|
198
|
+
if (liveness.status === "orphaned" || liveness.status === "unknown-host") {
|
|
199
|
+
throw new DagInterruptError("RUN_NOT_ACTIVE");
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
function isTerminalRunStatus(status) {
|
|
203
|
+
return (status === "finished" ||
|
|
204
|
+
status === "failed" ||
|
|
205
|
+
status === "partial_failed" ||
|
|
206
|
+
status === "superseded" ||
|
|
207
|
+
status === "abandoned");
|
|
208
|
+
}
|
|
209
|
+
export async function requestDagInterrupt(input) {
|
|
210
|
+
const requestId = input.requestId.trim();
|
|
211
|
+
const targetOperationId = input.targetOperationId.trim();
|
|
212
|
+
const runId = input.runId.trim();
|
|
213
|
+
if (!requestId || !targetOperationId || !runId) {
|
|
214
|
+
throw new DagInterruptError("INVALID_INPUT");
|
|
215
|
+
}
|
|
216
|
+
const reasonDetail = normalizeInterruptReasonDetail(input.reasonDetail);
|
|
217
|
+
const located = await locateDagRun(input.cwd, runId);
|
|
218
|
+
if (!located) {
|
|
219
|
+
throw new DagInterruptError("RUN_NOT_ACTIVE");
|
|
220
|
+
}
|
|
221
|
+
const state = await readDagRunState(located.runDir);
|
|
222
|
+
if (state.runId !== runId) {
|
|
223
|
+
throw new DagInterruptError("RUNNER_IDENTITY_MISMATCH");
|
|
224
|
+
}
|
|
225
|
+
assertKernelRunEligible({
|
|
226
|
+
lifecycle: located.lifecycle,
|
|
227
|
+
state,
|
|
228
|
+
expected: input.expectedRunnerIdentity,
|
|
229
|
+
});
|
|
230
|
+
const existing = await readInterruptRequest(located.runDir);
|
|
231
|
+
const next = {
|
|
232
|
+
schemaVersion: 1,
|
|
233
|
+
revision: (existing?.revision ?? 0) + 1,
|
|
234
|
+
requestId,
|
|
235
|
+
runId,
|
|
236
|
+
targetOperationId,
|
|
237
|
+
reasonCode: input.reasonCode,
|
|
238
|
+
reasonDetail,
|
|
239
|
+
requestedAt: (input.now ?? new Date()).toISOString(),
|
|
240
|
+
requestedBy: input.requestedBy ?? { source: "cli" },
|
|
241
|
+
expectedRunnerIdentity: input.expectedRunnerIdentity,
|
|
242
|
+
status: "requested",
|
|
243
|
+
};
|
|
244
|
+
if (existing) {
|
|
245
|
+
if (existing.status === "requested" ||
|
|
246
|
+
existing.status === "acknowledged") {
|
|
247
|
+
if (existing.requestId === requestId &&
|
|
248
|
+
requestPayloadKey(existing) === requestPayloadKey(next)) {
|
|
249
|
+
return {
|
|
250
|
+
ok: true,
|
|
251
|
+
interruptStatus: "requested",
|
|
252
|
+
request: existing,
|
|
253
|
+
runDir: located.runDir,
|
|
254
|
+
lifecycle: located.lifecycle,
|
|
255
|
+
};
|
|
256
|
+
}
|
|
257
|
+
if (existing.requestId === requestId) {
|
|
258
|
+
throw new DagInterruptError("INVALID_INPUT", "同一 requestId 不能绑定不同的中止内容。");
|
|
259
|
+
}
|
|
260
|
+
throw new DagInterruptError("INTERRUPT_ALREADY_PENDING");
|
|
261
|
+
}
|
|
262
|
+
if (existing.status === "needs-reconcile") {
|
|
263
|
+
throw new DagInterruptError("INTERRUPT_NEEDS_RECONCILE");
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
await writeInterruptRequest(located.runDir, next);
|
|
267
|
+
const confirmed = await readInterruptRequest(located.runDir);
|
|
268
|
+
if (!confirmed ||
|
|
269
|
+
confirmed.requestId !== requestId ||
|
|
270
|
+
requestPayloadKey(confirmed) !== requestPayloadKey(next)) {
|
|
271
|
+
throw new DagInterruptError("INTERRUPT_ALREADY_PENDING");
|
|
272
|
+
}
|
|
273
|
+
return {
|
|
274
|
+
ok: true,
|
|
275
|
+
interruptStatus: "requested",
|
|
276
|
+
request: confirmed,
|
|
277
|
+
runDir: located.runDir,
|
|
278
|
+
lifecycle: located.lifecycle,
|
|
279
|
+
};
|
|
280
|
+
}
|
|
281
|
+
export async function acknowledgeDagInterrupt(input) {
|
|
282
|
+
const existing = await readInterruptRequest(input.runDir);
|
|
283
|
+
if (!existing)
|
|
284
|
+
return undefined;
|
|
285
|
+
if (existing.runId !== input.runId) {
|
|
286
|
+
throw new DagInterruptError("RUNNER_IDENTITY_MISMATCH");
|
|
287
|
+
}
|
|
288
|
+
if (existing.status === "settled" || existing.status === "needs-reconcile") {
|
|
289
|
+
return existing;
|
|
290
|
+
}
|
|
291
|
+
if (!runnerIdentitiesMatch(existing.expectedRunnerIdentity, input.expectedRunnerIdentity)) {
|
|
292
|
+
throw new DagInterruptError("RUNNER_IDENTITY_MISMATCH");
|
|
293
|
+
}
|
|
294
|
+
if (existing.status === "acknowledged")
|
|
295
|
+
return existing;
|
|
296
|
+
const next = {
|
|
297
|
+
...existing,
|
|
298
|
+
revision: existing.revision + 1,
|
|
299
|
+
status: "acknowledged",
|
|
300
|
+
acknowledgedAt: (input.now ?? new Date()).toISOString(),
|
|
301
|
+
};
|
|
302
|
+
await writeInterruptRequest(input.runDir, next);
|
|
303
|
+
return next;
|
|
304
|
+
}
|
|
305
|
+
export async function settleDagInterrupt(input) {
|
|
306
|
+
const existing = await readInterruptRequest(input.runDir);
|
|
307
|
+
if (!existing)
|
|
308
|
+
return undefined;
|
|
309
|
+
if (existing.status === "settled")
|
|
310
|
+
return existing;
|
|
311
|
+
if (existing.status === "requested") {
|
|
312
|
+
const next = {
|
|
313
|
+
...existing,
|
|
314
|
+
revision: existing.revision + 1,
|
|
315
|
+
status: "needs-reconcile",
|
|
316
|
+
outcome: "runner-did-not-acknowledge",
|
|
317
|
+
};
|
|
318
|
+
await writeInterruptRequest(input.runDir, next);
|
|
319
|
+
return next;
|
|
320
|
+
}
|
|
321
|
+
const next = {
|
|
322
|
+
...existing,
|
|
323
|
+
revision: existing.revision + 1,
|
|
324
|
+
status: "settled",
|
|
325
|
+
settledAt: (input.now ?? new Date()).toISOString(),
|
|
326
|
+
outcome: input.outcome ?? "controller-interrupted",
|
|
327
|
+
};
|
|
328
|
+
await writeInterruptRequest(input.runDir, next);
|
|
329
|
+
return next;
|
|
330
|
+
}
|
|
331
|
+
export async function markInterruptNeedsReconcile(input) {
|
|
332
|
+
const existing = await readInterruptRequest(input.runDir);
|
|
333
|
+
if (!existing)
|
|
334
|
+
return undefined;
|
|
335
|
+
if (existing.status === "settled")
|
|
336
|
+
return existing;
|
|
337
|
+
const next = {
|
|
338
|
+
...existing,
|
|
339
|
+
revision: existing.revision + 1,
|
|
340
|
+
status: "needs-reconcile",
|
|
341
|
+
outcome: input.outcome,
|
|
342
|
+
};
|
|
343
|
+
await writeInterruptRequest(input.runDir, next);
|
|
344
|
+
return next;
|
|
345
|
+
}
|
|
346
|
+
export function projectedInterruptStatus(request) {
|
|
347
|
+
return request?.status ?? "none";
|
|
348
|
+
}
|
|
349
|
+
export function mergeAbortSignals(left, right) {
|
|
350
|
+
if (!left)
|
|
351
|
+
return right;
|
|
352
|
+
if (!right)
|
|
353
|
+
return left;
|
|
354
|
+
const anyFn = AbortSignal.any;
|
|
355
|
+
if (typeof anyFn === "function")
|
|
356
|
+
return anyFn([left, right]);
|
|
357
|
+
const merged = new AbortController();
|
|
358
|
+
const onAbort = () => {
|
|
359
|
+
if (!merged.signal.aborted)
|
|
360
|
+
merged.abort();
|
|
361
|
+
};
|
|
362
|
+
if (left.aborted || right.aborted) {
|
|
363
|
+
merged.abort();
|
|
364
|
+
return merged.signal;
|
|
365
|
+
}
|
|
366
|
+
left.addEventListener("abort", onAbort, { once: true });
|
|
367
|
+
right.addEventListener("abort", onAbort, { once: true });
|
|
368
|
+
return merged.signal;
|
|
369
|
+
}
|
|
370
|
+
export function startInterruptWatcher(input) {
|
|
371
|
+
let stopped = false;
|
|
372
|
+
const tick = () => {
|
|
373
|
+
if (stopped || input.controller.signal.aborted)
|
|
374
|
+
return;
|
|
375
|
+
void (async () => {
|
|
376
|
+
try {
|
|
377
|
+
const existing = await readInterruptRequest(input.runDir);
|
|
378
|
+
if (!existing || existing.runId !== input.runId)
|
|
379
|
+
return;
|
|
380
|
+
if (existing.status !== "requested" && existing.status !== "acknowledged") {
|
|
381
|
+
return;
|
|
382
|
+
}
|
|
383
|
+
const acknowledged = await acknowledgeDagInterrupt({
|
|
384
|
+
runDir: input.runDir,
|
|
385
|
+
runId: input.runId,
|
|
386
|
+
expectedRunnerIdentity: input.expectedRunnerIdentity,
|
|
387
|
+
});
|
|
388
|
+
if (acknowledged &&
|
|
389
|
+
(acknowledged.status === "acknowledged" ||
|
|
390
|
+
acknowledged.status === "settled") &&
|
|
391
|
+
!input.controller.signal.aborted) {
|
|
392
|
+
input.controller.abort(acknowledged.reasonCode);
|
|
393
|
+
}
|
|
394
|
+
}
|
|
395
|
+
catch {
|
|
396
|
+
// Identity drift or corrupt artifact: do not abort. Leave requested
|
|
397
|
+
// for doctor / needs-reconcile at terminal.
|
|
398
|
+
}
|
|
399
|
+
})();
|
|
400
|
+
};
|
|
401
|
+
void mkdir(path.join(input.runDir, ".runtime"), { recursive: true }).then(() => {
|
|
402
|
+
if (!stopped)
|
|
403
|
+
tick();
|
|
404
|
+
}, () => {
|
|
405
|
+
if (!stopped)
|
|
406
|
+
tick();
|
|
407
|
+
});
|
|
408
|
+
const timer = setInterval(tick, 400);
|
|
409
|
+
timer.unref();
|
|
410
|
+
let watcher;
|
|
411
|
+
try {
|
|
412
|
+
watcher = watch(path.join(input.runDir, ".runtime"), () => tick());
|
|
413
|
+
watcher.unref?.();
|
|
414
|
+
}
|
|
415
|
+
catch {
|
|
416
|
+
// Polling is the guaranteed path; fs.watch is best-effort.
|
|
417
|
+
}
|
|
418
|
+
return {
|
|
419
|
+
stop: () => {
|
|
420
|
+
stopped = true;
|
|
421
|
+
clearInterval(timer);
|
|
422
|
+
watcher?.close();
|
|
423
|
+
},
|
|
424
|
+
};
|
|
425
|
+
}
|
|
426
|
+
function readFlag(args, index, name) {
|
|
427
|
+
const arg = args[index.value];
|
|
428
|
+
if (arg === `--${name}`) {
|
|
429
|
+
index.value += 1;
|
|
430
|
+
return args[index.value];
|
|
431
|
+
}
|
|
432
|
+
if (arg?.startsWith(`--${name}=`)) {
|
|
433
|
+
return arg.slice(`--${name}=`.length);
|
|
434
|
+
}
|
|
435
|
+
return undefined;
|
|
436
|
+
}
|
|
437
|
+
export function parseDagRequestInterruptArgs(args) {
|
|
438
|
+
let runId;
|
|
439
|
+
let requestId;
|
|
440
|
+
let targetOperationId;
|
|
441
|
+
let reasonCode;
|
|
442
|
+
let reason;
|
|
443
|
+
let pid;
|
|
444
|
+
let hostname;
|
|
445
|
+
let startedAt;
|
|
446
|
+
let fingerprint;
|
|
447
|
+
for (let i = 0; i < args.length; i += 1) {
|
|
448
|
+
const cursor = { value: i };
|
|
449
|
+
const arg = args[i];
|
|
450
|
+
const runIdValue = readFlag(args, cursor, "run-id");
|
|
451
|
+
if (runIdValue !== undefined) {
|
|
452
|
+
runId = runIdValue;
|
|
453
|
+
i = cursor.value;
|
|
454
|
+
continue;
|
|
455
|
+
}
|
|
456
|
+
const requestIdValue = readFlag(args, cursor, "request-id");
|
|
457
|
+
if (requestIdValue !== undefined) {
|
|
458
|
+
requestId = requestIdValue;
|
|
459
|
+
i = cursor.value;
|
|
460
|
+
continue;
|
|
461
|
+
}
|
|
462
|
+
const opValue = readFlag(args, cursor, "target-operation-id");
|
|
463
|
+
if (opValue !== undefined) {
|
|
464
|
+
targetOperationId = opValue;
|
|
465
|
+
i = cursor.value;
|
|
466
|
+
continue;
|
|
467
|
+
}
|
|
468
|
+
const reasonCodeValue = readFlag(args, cursor, "reason-code");
|
|
469
|
+
if (reasonCodeValue !== undefined) {
|
|
470
|
+
reasonCode = reasonCodeValue;
|
|
471
|
+
i = cursor.value;
|
|
472
|
+
continue;
|
|
473
|
+
}
|
|
474
|
+
const reasonValue = readFlag(args, cursor, "reason");
|
|
475
|
+
if (reasonValue !== undefined) {
|
|
476
|
+
reason = reasonValue;
|
|
477
|
+
i = cursor.value;
|
|
478
|
+
continue;
|
|
479
|
+
}
|
|
480
|
+
const pidValue = readFlag(args, cursor, "expected-runner-pid");
|
|
481
|
+
if (pidValue !== undefined) {
|
|
482
|
+
pid = pidValue;
|
|
483
|
+
i = cursor.value;
|
|
484
|
+
continue;
|
|
485
|
+
}
|
|
486
|
+
const hostValue = readFlag(args, cursor, "expected-runner-hostname");
|
|
487
|
+
if (hostValue !== undefined) {
|
|
488
|
+
hostname = hostValue;
|
|
489
|
+
i = cursor.value;
|
|
490
|
+
continue;
|
|
491
|
+
}
|
|
492
|
+
const startedValue = readFlag(args, cursor, "expected-runner-started-at");
|
|
493
|
+
if (startedValue !== undefined) {
|
|
494
|
+
startedAt = startedValue;
|
|
495
|
+
i = cursor.value;
|
|
496
|
+
continue;
|
|
497
|
+
}
|
|
498
|
+
const fpValue = readFlag(args, cursor, "expected-controller-fingerprint");
|
|
499
|
+
if (fpValue !== undefined) {
|
|
500
|
+
fingerprint = fpValue;
|
|
501
|
+
i = cursor.value;
|
|
502
|
+
continue;
|
|
503
|
+
}
|
|
504
|
+
if (arg === "--json")
|
|
505
|
+
continue;
|
|
506
|
+
if (arg?.startsWith("-")) {
|
|
507
|
+
throw new DagInterruptError("INVALID_INPUT", `unknown dag request-interrupt flag: ${arg}`);
|
|
508
|
+
}
|
|
509
|
+
throw new DagInterruptError("INVALID_INPUT", `unexpected positional argument: ${arg}`);
|
|
510
|
+
}
|
|
511
|
+
const parsedPid = Number(pid);
|
|
512
|
+
if (!runId?.trim() ||
|
|
513
|
+
!requestId?.trim() ||
|
|
514
|
+
!targetOperationId?.trim() ||
|
|
515
|
+
!reasonCode ||
|
|
516
|
+
!reason ||
|
|
517
|
+
!hostname?.trim() ||
|
|
518
|
+
!startedAt?.trim() ||
|
|
519
|
+
!fingerprint?.trim() ||
|
|
520
|
+
!Number.isInteger(parsedPid) ||
|
|
521
|
+
parsedPid <= 0) {
|
|
522
|
+
throw new DagInterruptError("INVALID_INPUT", "dag request-interrupt requires --run-id, --request-id, --target-operation-id, --reason-code, --reason, --expected-runner-pid, --expected-runner-hostname, --expected-runner-started-at, --expected-controller-fingerprint");
|
|
523
|
+
}
|
|
524
|
+
if (!isInterruptReasonCode(reasonCode)) {
|
|
525
|
+
throw new DagInterruptError("INVALID_INPUT", `reason-code must be one of ${INTERRUPT_REASON_CODES.join("|")}`);
|
|
526
|
+
}
|
|
527
|
+
return {
|
|
528
|
+
runId: runId.trim(),
|
|
529
|
+
requestId: requestId.trim(),
|
|
530
|
+
targetOperationId: targetOperationId.trim(),
|
|
531
|
+
reasonCode,
|
|
532
|
+
reason,
|
|
533
|
+
expectedRunnerIdentity: {
|
|
534
|
+
pid: parsedPid,
|
|
535
|
+
hostname: hostname.trim(),
|
|
536
|
+
startedAt: startedAt.trim(),
|
|
537
|
+
controllerFingerprint: fingerprint.trim(),
|
|
538
|
+
},
|
|
539
|
+
};
|
|
540
|
+
}
|
|
541
|
+
export async function executeDagRequestInterrupt(repoRoot, rawArgs) {
|
|
542
|
+
const parsed = parseDagRequestInterruptArgs(rawArgs);
|
|
543
|
+
return requestDagInterrupt({
|
|
544
|
+
cwd: repoRoot,
|
|
545
|
+
runId: parsed.runId,
|
|
546
|
+
requestId: parsed.requestId,
|
|
547
|
+
targetOperationId: parsed.targetOperationId,
|
|
548
|
+
reasonCode: parsed.reasonCode,
|
|
549
|
+
reasonDetail: parsed.reason,
|
|
550
|
+
expectedRunnerIdentity: parsed.expectedRunnerIdentity,
|
|
551
|
+
requestedBy: { source: "cli" },
|
|
552
|
+
});
|
|
553
|
+
}
|
|
554
|
+
export function defaultLocalHostname() {
|
|
555
|
+
return localHostname();
|
|
556
|
+
}
|
|
557
|
+
export function getActiveRunDir(cwd, runId) {
|
|
558
|
+
return getDagRunDir(cwd, "active", runId);
|
|
559
|
+
}
|
|
@@ -743,7 +743,6 @@ function findDoctorFailureNode(state) {
|
|
|
743
743
|
nodeId: state.pausedByNodeId,
|
|
744
744
|
status: node?.status,
|
|
745
745
|
rawFailureCategory: node?.failureCategory,
|
|
746
|
-
skippedReason: node?.skippedReason,
|
|
747
746
|
};
|
|
748
747
|
}
|
|
749
748
|
const errorEntry = Object.entries(state.nodes).find(([, node]) => node.status === "ERROR");
|
|
@@ -759,7 +758,6 @@ function findDoctorFailureNode(state) {
|
|
|
759
758
|
nodeId: selected[0],
|
|
760
759
|
status: selected[1].status,
|
|
761
760
|
rawFailureCategory: selected[1].failureCategory,
|
|
762
|
-
skippedReason: selected[1].skippedReason,
|
|
763
761
|
};
|
|
764
762
|
}
|
|
765
763
|
async function readRunOwnedBackendTestClassification(runDir) {
|
|
@@ -788,7 +786,6 @@ async function resolveDoctorFailureRouting(input) {
|
|
|
788
786
|
normalizedFailureCategory: "unknown",
|
|
789
787
|
nodeId: input.nodeId,
|
|
790
788
|
productLineFailureCategory: classifiedCategory,
|
|
791
|
-
skippedReason: input.skippedReason,
|
|
792
789
|
});
|
|
793
790
|
}
|
|
794
791
|
return routeDagFailure(input);
|
|
@@ -810,7 +807,6 @@ async function formatDagDoctorMarkdown(repoRoot, runId) {
|
|
|
810
807
|
rawFailureCategory,
|
|
811
808
|
normalizedFailureCategory: normalizedCategory,
|
|
812
809
|
nodeId: failure.nodeId,
|
|
813
|
-
skippedReason: failure.skippedReason,
|
|
814
810
|
});
|
|
815
811
|
const evidence = failure.nodeId
|
|
816
812
|
? path.join(located.runDir, failure.nodeId, "result.summary.md")
|