@tea-agent/loop-agent 0.20.1-beta.0 → 0.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +50 -0
- package/dist/application/dag/args.js +29 -0
- package/dist/application/dag/run-dag.js +3 -1
- package/dist/cli/command-definitions.js +15 -1
- package/dist/cli/program.js +11 -1
- package/dist/commands/dag-rerun-task.js +19 -0
- package/dist/commands/dag-rerun.js +111 -0
- package/dist/commands/init.js +7 -0
- package/dist/executors/dag-pi-executor.js +24 -0
- package/dist/executors/pi-executor.js +111 -36
- package/dist/executors/pi-sdk-executor.js +105 -29
- package/dist/executors/shell-executor.js +54 -11
- package/dist/shared/operator/capabilities.js +54 -0
- package/dist/worker/console/index.js +1 -1
- package/dist/worker/console/inspect-split.js +82 -0
- package/dist/worker/console/operation-runner.js +3 -1
- package/dist/worker/console/operation-store.js +1 -0
- package/dist/worker/console/operator-actions.js +153 -2
- package/dist/worker/console/operator-user-error.js +10 -0
- package/dist/worker/console/pi-readiness.js +4 -0
- package/dist/worker/console/recovery-cta.js +116 -5
- package/dist/worker/console/recovery-selection.js +107 -0
- package/dist/worker/console/resolve-dag-run-for-task.js +50 -11
- package/dist/worker/console/routes.js +20 -0
- package/dist/worker/console/sibling-controller.js +12 -7
- package/dist/worker/console/static/assets/index-CUDke82y.js +18 -0
- package/dist/worker/console/static/assets/index-wSEksVSO.css +1 -0
- package/dist/worker/console/static/index.html +2 -2
- package/dist/worker/loop-agent/loop-agent-client.js +43 -9
- package/dist/worker/observability/read-model.js +67 -1
- package/dist/worker/observe/static/constants.js +5 -0
- package/dist/worker/observe/static/format-pool.js +22 -3
- package/dist/worker/observe/static/index.html +1 -1
- package/dist/worker/observe/static/styles.css +32 -3
- package/dist/worker/observe/static/views/dag-inspector.js +2 -2
- package/dist/worker/run-task/run-task.js +23 -6
- package/dist/workflows/dag/backend-test-markdown-workflow.js +291 -97
- package/dist/workflows/dag/backend-test-result-contract.js +10 -4
- package/dist/workflows/dag/frontend-test-result-contract.js +64 -0
- package/dist/workflows/dag/init-hybrid.js +117 -64
- package/dist/workflows/dag/lifecycle.js +60 -4
- package/dist/workflows/dag/liveness-policy.js +250 -0
- package/dist/workflows/dag/node-execution.js +89 -6
- package/dist/workflows/dag/output-protocol.js +76 -0
- package/dist/workflows/dag/rerun-plan.js +611 -0
- package/dist/workflows/dag/rerun-run.js +497 -0
- package/dist/workflows/dag/rerun-task.js +284 -0
- package/dist/workflows/dag/retry-policy.js +20 -1
- package/dist/workflows/dag/runner.js +71 -1
- package/dist/workflows/dag/skill-snapshot.js +22 -3
- package/dist/workflows/dag/types.js +12 -0
- package/dist/workflows/dag/validate.js +11 -0
- package/dist/workflows/dag/workspace-checkpoint.js +163 -0
- package/docs/README.md +5 -5
- package/docs/architecture/dag-execution.md +11 -0
- package/docs/architecture/facts-and-state.md +1 -0
- package/docs/architecture/worker-and-feature.md +10 -0
- package/docs/templates/agent-dag.schema.json +17 -2
- package/docs/templates/backend-test-dag.generate-pytest.prompt.md +5 -2
- package/docs/templates/backend-test-dag.json +15 -15
- package/docs/templates/frontend-test-case-checklist.md +16 -1
- package/docs/templates/frontend-test-dag.generate-cases.prompt.md +26 -2
- package/docs/templates/frontend-test-dag.json +65 -6
- package/docs/templates/frontend-test-dag.retrieve-context.prompt.md +4 -1
- package/harness.json +1 -1
- package/package.json +1 -1
- package/skills/loop-agent/references/command-reference.md +5 -0
- package/skills/loop-agent/references/hybrid-dag.md +2 -2
- package/skills/playwright-cli-case-generator/SKILL.md +35 -7
- package/dist/worker/console/static/assets/index-3vsjZJHq.js +0 -16
- package/dist/worker/console/static/assets/index-i1wV4LrY.css +0 -1
|
@@ -187,10 +187,38 @@ export function assessDagRunLiveness(input) {
|
|
|
187
187
|
return { status: "stale", runnerAlive: true };
|
|
188
188
|
}
|
|
189
189
|
const activeNode = Object.values(input.state.nodes).find((node) => node.status === "RUNNING");
|
|
190
|
-
|
|
191
|
-
|
|
190
|
+
if (!activeNode)
|
|
191
|
+
return { status: "active", runnerAlive: true };
|
|
192
|
+
// Prefer persisted adaptive projection when present.
|
|
193
|
+
if (activeNode.livenessStatus === "needs-attention") {
|
|
194
|
+
return { status: "needs-attention", runnerAlive: true };
|
|
195
|
+
}
|
|
196
|
+
if (activeNode.livenessStatus === "suspected-stall"
|
|
197
|
+
|| activeNode.livenessStatus === "probing") {
|
|
198
|
+
return { status: "suspected-stall", runnerAlive: true };
|
|
199
|
+
}
|
|
200
|
+
if (activeNode.livenessStatus === "quiet") {
|
|
192
201
|
return { status: "node-quiet", runnerAlive: true };
|
|
193
202
|
}
|
|
203
|
+
// Fall back to meaningful progress clocks (never use runner lease as progress).
|
|
204
|
+
const meaningfulAt = activeNode.lastMeaningfulProgressAt
|
|
205
|
+
?? activeNode.lastProviderActivityAt
|
|
206
|
+
?? activeNode.lastToolActivityAt
|
|
207
|
+
?? activeNode.lastOutputActivityAt
|
|
208
|
+
?? activeNode.lastActivityAt
|
|
209
|
+
?? activeNode.startedAt;
|
|
210
|
+
const nodeActivityMs = Date.parse(meaningfulAt ?? "");
|
|
211
|
+
if (!Number.isNaN(nodeActivityMs)) {
|
|
212
|
+
const idleMs = nowMs - nodeActivityMs;
|
|
213
|
+
const stallMs = input.nodeStallThresholdMs ?? 900_000;
|
|
214
|
+
const quietMs = input.nodeQuietThresholdMs ?? 300_000;
|
|
215
|
+
if (idleMs > stallMs) {
|
|
216
|
+
return { status: "suspected-stall", runnerAlive: true };
|
|
217
|
+
}
|
|
218
|
+
if (idleMs > quietMs) {
|
|
219
|
+
return { status: "node-quiet", runnerAlive: true };
|
|
220
|
+
}
|
|
221
|
+
}
|
|
194
222
|
return { status: "active", runnerAlive: true };
|
|
195
223
|
}
|
|
196
224
|
export function deriveDagRunEffectiveStatus(input) {
|
|
@@ -210,6 +238,10 @@ export function deriveDagRunEffectiveStatus(input) {
|
|
|
210
238
|
}
|
|
211
239
|
if (input.liveness === "orphaned" || input.liveness === "stale")
|
|
212
240
|
return "interrupted";
|
|
241
|
+
if (input.liveness === "needs-attention")
|
|
242
|
+
return "needs-attention";
|
|
243
|
+
if (input.liveness === "suspected-stall")
|
|
244
|
+
return "running-suspected-stall";
|
|
213
245
|
if (input.liveness === "node-quiet")
|
|
214
246
|
return "running-quiet";
|
|
215
247
|
if (input.liveness === "unknown-host")
|
|
@@ -232,7 +264,15 @@ export function assessDagRunRecoveryEligibility(input) {
|
|
|
232
264
|
reasons.push("run-already-terminal");
|
|
233
265
|
}
|
|
234
266
|
if (input.lifecycle === "active"
|
|
235
|
-
&& [
|
|
267
|
+
&& [
|
|
268
|
+
"active",
|
|
269
|
+
"node-quiet",
|
|
270
|
+
"suspected-stall",
|
|
271
|
+
"needs-attention",
|
|
272
|
+
"stale",
|
|
273
|
+
"unknown-host",
|
|
274
|
+
"unknown",
|
|
275
|
+
].includes(input.liveness)) {
|
|
236
276
|
canReconcile = false;
|
|
237
277
|
reasons.push("runner-not-proven-dead-or-stopped");
|
|
238
278
|
}
|
|
@@ -358,10 +398,26 @@ export async function detectDagRunHealthIssues(input) {
|
|
|
358
398
|
issues.push({
|
|
359
399
|
code: "node-activity-quiet",
|
|
360
400
|
severity: "warning",
|
|
361
|
-
message: "Runner heartbeat is fresh but the current RUNNING node has produced no
|
|
401
|
+
message: "Runner heartbeat is fresh but the current RUNNING node has produced no meaningful activity for more than 5 minutes",
|
|
362
402
|
advisoryAction: "Inspect the node session events and executor logs before deciding whether to wait or abort.",
|
|
363
403
|
});
|
|
364
404
|
}
|
|
405
|
+
else if (liveness.status === "suspected-stall") {
|
|
406
|
+
issues.push({
|
|
407
|
+
code: "node-activity-suspected-stall",
|
|
408
|
+
severity: "warning",
|
|
409
|
+
message: "Runner heartbeat is fresh but the current RUNNING node has no meaningful Provider/tool/output activity for more than 15 minutes",
|
|
410
|
+
advisoryAction: "Treat as suspected network/provider stall; only retry read-only nodes after the attempt is proven finished. Writers must fail closed to reconcile.",
|
|
411
|
+
});
|
|
412
|
+
}
|
|
413
|
+
else if (liveness.status === "needs-attention") {
|
|
414
|
+
issues.push({
|
|
415
|
+
code: "node-needs-attention",
|
|
416
|
+
severity: "error",
|
|
417
|
+
message: "Runner or node process identity cannot be proven healthy, or absolute max wall clock was exceeded",
|
|
418
|
+
advisoryAction: "Do not forge timed-out/cancelled. Inspect process identity and artifacts; reconcile only when exit is proven.",
|
|
419
|
+
});
|
|
420
|
+
}
|
|
365
421
|
if (lifecycle === "active" &&
|
|
366
422
|
state.status === "running" &&
|
|
367
423
|
state.humanDecisionNodeId) {
|
|
@@ -0,0 +1,250 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
/**
|
|
3
|
+
* Adaptive liveness policy for DAG/Pi supervision.
|
|
4
|
+
*
|
|
5
|
+
* Four clocks must not be mixed:
|
|
6
|
+
* - runner lease (synthetic heartbeat) — not meaningful progress
|
|
7
|
+
* - provider/transport activity
|
|
8
|
+
* - tool activity
|
|
9
|
+
* - output / meaningful progress
|
|
10
|
+
*/
|
|
11
|
+
export const DEFAULT_LIVENESS_POLICY = {
|
|
12
|
+
heartbeatIntervalMs: 15_000,
|
|
13
|
+
runnerStaleMs: 90_000,
|
|
14
|
+
quietMs: 300_000,
|
|
15
|
+
stallProbeMs: 900_000,
|
|
16
|
+
abortGraceMs: 30_000,
|
|
17
|
+
/** 4h absolute max wall clock — cannot be renewed by empty heartbeats. */
|
|
18
|
+
absoluteMaxWallClockMs: 14_400_000,
|
|
19
|
+
};
|
|
20
|
+
export const dagLivenessPolicySchema = z
|
|
21
|
+
.object({
|
|
22
|
+
heartbeatIntervalMs: z.number().int().min(1_000).optional(),
|
|
23
|
+
runnerStaleMs: z.number().int().positive().optional(),
|
|
24
|
+
quietMs: z.number().int().positive().optional(),
|
|
25
|
+
stallProbeMs: z.number().int().positive().optional(),
|
|
26
|
+
abortGraceMs: z.number().int().positive().optional(),
|
|
27
|
+
absoluteMaxWallClockMs: z.number().int().positive().optional(),
|
|
28
|
+
})
|
|
29
|
+
.strict()
|
|
30
|
+
.superRefine((value, ctx) => {
|
|
31
|
+
const resolved = { ...DEFAULT_LIVENESS_POLICY, ...value };
|
|
32
|
+
if (resolved.runnerStaleMs <= resolved.heartbeatIntervalMs) {
|
|
33
|
+
ctx.addIssue({
|
|
34
|
+
code: z.ZodIssueCode.custom,
|
|
35
|
+
path: ["runnerStaleMs"],
|
|
36
|
+
message: "runnerStaleMs must be greater than heartbeatIntervalMs",
|
|
37
|
+
});
|
|
38
|
+
}
|
|
39
|
+
if (resolved.stallProbeMs <= resolved.quietMs) {
|
|
40
|
+
ctx.addIssue({
|
|
41
|
+
code: z.ZodIssueCode.custom,
|
|
42
|
+
path: ["stallProbeMs"],
|
|
43
|
+
message: "stallProbeMs must be greater than quietMs",
|
|
44
|
+
});
|
|
45
|
+
}
|
|
46
|
+
if (resolved.absoluteMaxWallClockMs <= resolved.stallProbeMs) {
|
|
47
|
+
ctx.addIssue({
|
|
48
|
+
code: z.ZodIssueCode.custom,
|
|
49
|
+
path: ["absoluteMaxWallClockMs"],
|
|
50
|
+
message: "absoluteMaxWallClockMs must be greater than stallProbeMs",
|
|
51
|
+
});
|
|
52
|
+
}
|
|
53
|
+
})
|
|
54
|
+
.optional();
|
|
55
|
+
/**
|
|
56
|
+
* Resolve policy from node/defaults partials. Missing fields use conservative defaults.
|
|
57
|
+
*/
|
|
58
|
+
export function resolveLivenessPolicy(...sources) {
|
|
59
|
+
const merged = {};
|
|
60
|
+
for (const source of sources) {
|
|
61
|
+
if (!source)
|
|
62
|
+
continue;
|
|
63
|
+
for (const key of Object.keys(DEFAULT_LIVENESS_POLICY)) {
|
|
64
|
+
const value = source[key];
|
|
65
|
+
if (typeof value === "number" && Number.isFinite(value) && value > 0) {
|
|
66
|
+
merged[key] = Math.trunc(value);
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
return {
|
|
71
|
+
heartbeatIntervalMs: merged.heartbeatIntervalMs ?? DEFAULT_LIVENESS_POLICY.heartbeatIntervalMs,
|
|
72
|
+
runnerStaleMs: merged.runnerStaleMs ?? DEFAULT_LIVENESS_POLICY.runnerStaleMs,
|
|
73
|
+
quietMs: merged.quietMs ?? DEFAULT_LIVENESS_POLICY.quietMs,
|
|
74
|
+
stallProbeMs: merged.stallProbeMs ?? DEFAULT_LIVENESS_POLICY.stallProbeMs,
|
|
75
|
+
abortGraceMs: merged.abortGraceMs ?? DEFAULT_LIVENESS_POLICY.abortGraceMs,
|
|
76
|
+
absoluteMaxWallClockMs: merged.absoluteMaxWallClockMs ??
|
|
77
|
+
DEFAULT_LIVENESS_POLICY.absoluteMaxWallClockMs,
|
|
78
|
+
};
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* Synthetic timer heartbeats never count as meaningful Pi progress.
|
|
82
|
+
*/
|
|
83
|
+
export function isMeaningfulActivityKind(kind) {
|
|
84
|
+
return kind === "provider" || kind === "tool" || kind === "output";
|
|
85
|
+
}
|
|
86
|
+
/**
|
|
87
|
+
* Attempt fence: late activity from a previous attempt must not overwrite the current one.
|
|
88
|
+
*/
|
|
89
|
+
export function shouldAcceptActivity(currentAttempt, activityAttempt) {
|
|
90
|
+
if (!Number.isInteger(currentAttempt) || currentAttempt < 1)
|
|
91
|
+
return false;
|
|
92
|
+
if (!Number.isInteger(activityAttempt) || activityAttempt < 1)
|
|
93
|
+
return false;
|
|
94
|
+
return activityAttempt === currentAttempt;
|
|
95
|
+
}
|
|
96
|
+
export function classifySessionEventActivity(event) {
|
|
97
|
+
if (!event || typeof event !== "object")
|
|
98
|
+
return "provider";
|
|
99
|
+
const type = typeof event.type === "string"
|
|
100
|
+
? String(event.type)
|
|
101
|
+
: "";
|
|
102
|
+
if (type === "tool_start" ||
|
|
103
|
+
type === "tool_end" ||
|
|
104
|
+
type === "tool_execution_start" ||
|
|
105
|
+
type === "tool_execution_end") {
|
|
106
|
+
return "tool";
|
|
107
|
+
}
|
|
108
|
+
if (type === "thinking_delta" || type === "message_update") {
|
|
109
|
+
// Noise — do not treat as progress when callers filter via shouldPersistSessionEvent.
|
|
110
|
+
return null;
|
|
111
|
+
}
|
|
112
|
+
return "provider";
|
|
113
|
+
}
|
|
114
|
+
function parseMs(value) {
|
|
115
|
+
if (!value)
|
|
116
|
+
return NaN;
|
|
117
|
+
const ms = Date.parse(value);
|
|
118
|
+
return Number.isFinite(ms) ? ms : NaN;
|
|
119
|
+
}
|
|
120
|
+
function latestMeaningfulActivityMs(snapshot) {
|
|
121
|
+
const candidates = [
|
|
122
|
+
parseMs(snapshot.lastMeaningfulProgressAt),
|
|
123
|
+
parseMs(snapshot.lastProviderActivityAt),
|
|
124
|
+
parseMs(snapshot.lastToolActivityAt),
|
|
125
|
+
parseMs(snapshot.lastOutputActivityAt),
|
|
126
|
+
// Compatibility: lastActivityAt is treated as meaningful when richer clocks absent.
|
|
127
|
+
parseMs(snapshot.lastActivityAt),
|
|
128
|
+
parseMs(snapshot.startedAt),
|
|
129
|
+
].filter((value) => !Number.isNaN(value));
|
|
130
|
+
if (candidates.length === 0)
|
|
131
|
+
return NaN;
|
|
132
|
+
return Math.max(...candidates);
|
|
133
|
+
}
|
|
134
|
+
/**
|
|
135
|
+
* Pure node liveness evaluator. No I/O.
|
|
136
|
+
*
|
|
137
|
+
* Transitions:
|
|
138
|
+
* - real provider/tool/output activity → active
|
|
139
|
+
* - no activity ≥ quietMs and runner lease fresh → quiet
|
|
140
|
+
* - no activity ≥ stallProbeMs → suspected-stall
|
|
141
|
+
* - probing flag → probing
|
|
142
|
+
* - identity mismatch / unprovable survival → needs-attention
|
|
143
|
+
* - wall clock ≥ absoluteMax → needs-attention (controlled abort path upstream)
|
|
144
|
+
*/
|
|
145
|
+
export function evaluateNodeLiveness(input) {
|
|
146
|
+
const policy = resolveLivenessPolicy(input.policy);
|
|
147
|
+
const nowMs = typeof input.now === "number"
|
|
148
|
+
? input.now
|
|
149
|
+
: (input.now ?? new Date()).getTime();
|
|
150
|
+
const startedMs = parseMs(input.node.startedAt);
|
|
151
|
+
const wallClockMs = !Number.isNaN(startedMs)
|
|
152
|
+
? Math.max(0, nowMs - startedMs)
|
|
153
|
+
: 0;
|
|
154
|
+
const exceededAbsoluteMax = wallClockMs >= policy.absoluteMaxWallClockMs;
|
|
155
|
+
if (input.node.needsAttentionReason || exceededAbsoluteMax) {
|
|
156
|
+
return {
|
|
157
|
+
status: "needs-attention",
|
|
158
|
+
lastMeaningfulProgressAt: input.node.lastMeaningfulProgressAt ?? input.node.lastActivityAt,
|
|
159
|
+
idleMs: 0,
|
|
160
|
+
wallClockMs,
|
|
161
|
+
exceededAbsoluteMax,
|
|
162
|
+
};
|
|
163
|
+
}
|
|
164
|
+
if (input.node.probing) {
|
|
165
|
+
return {
|
|
166
|
+
status: "probing",
|
|
167
|
+
lastMeaningfulProgressAt: input.node.lastMeaningfulProgressAt ?? input.node.lastActivityAt,
|
|
168
|
+
idleMs: 0,
|
|
169
|
+
wallClockMs,
|
|
170
|
+
exceededAbsoluteMax,
|
|
171
|
+
};
|
|
172
|
+
}
|
|
173
|
+
const meaningfulMs = latestMeaningfulActivityMs(input.node);
|
|
174
|
+
const idleMs = !Number.isNaN(meaningfulMs)
|
|
175
|
+
? Math.max(0, nowMs - meaningfulMs)
|
|
176
|
+
: wallClockMs;
|
|
177
|
+
const leaseFresh = input.runnerLeaseFresh ??
|
|
178
|
+
(() => {
|
|
179
|
+
const heartbeatMs = parseMs(input.runnerHeartbeatAt);
|
|
180
|
+
if (Number.isNaN(heartbeatMs))
|
|
181
|
+
return true;
|
|
182
|
+
return nowMs - heartbeatMs <= policy.runnerStaleMs;
|
|
183
|
+
})();
|
|
184
|
+
if (idleMs >= policy.stallProbeMs) {
|
|
185
|
+
return {
|
|
186
|
+
status: "suspected-stall",
|
|
187
|
+
lastMeaningfulProgressAt: input.node.lastMeaningfulProgressAt ?? input.node.lastActivityAt,
|
|
188
|
+
idleMs,
|
|
189
|
+
wallClockMs,
|
|
190
|
+
exceededAbsoluteMax,
|
|
191
|
+
};
|
|
192
|
+
}
|
|
193
|
+
if (idleMs >= policy.quietMs && leaseFresh) {
|
|
194
|
+
return {
|
|
195
|
+
status: "quiet",
|
|
196
|
+
lastMeaningfulProgressAt: input.node.lastMeaningfulProgressAt ?? input.node.lastActivityAt,
|
|
197
|
+
idleMs,
|
|
198
|
+
wallClockMs,
|
|
199
|
+
exceededAbsoluteMax,
|
|
200
|
+
};
|
|
201
|
+
}
|
|
202
|
+
// Active tool presence keeps us from escalating beyond quiet solely on idle output.
|
|
203
|
+
if ((input.node.activeToolCount ?? 0) > 0 && idleMs < policy.stallProbeMs) {
|
|
204
|
+
return {
|
|
205
|
+
status: idleMs >= policy.quietMs ? "quiet" : "active",
|
|
206
|
+
lastMeaningfulProgressAt: input.node.lastMeaningfulProgressAt ?? input.node.lastActivityAt,
|
|
207
|
+
idleMs,
|
|
208
|
+
wallClockMs,
|
|
209
|
+
exceededAbsoluteMax,
|
|
210
|
+
};
|
|
211
|
+
}
|
|
212
|
+
return {
|
|
213
|
+
status: "active",
|
|
214
|
+
lastMeaningfulProgressAt: input.node.lastMeaningfulProgressAt ?? input.node.lastActivityAt,
|
|
215
|
+
idleMs,
|
|
216
|
+
wallClockMs,
|
|
217
|
+
exceededAbsoluteMax,
|
|
218
|
+
};
|
|
219
|
+
}
|
|
220
|
+
/**
|
|
221
|
+
* Apply a fenced activity event onto a mutable node snapshot (pure field updates).
|
|
222
|
+
* Returns whether the write was accepted.
|
|
223
|
+
*/
|
|
224
|
+
export function applyNodeActivity(input) {
|
|
225
|
+
if (!shouldAcceptActivity(input.currentAttempt, input.activityAttempt)) {
|
|
226
|
+
return false;
|
|
227
|
+
}
|
|
228
|
+
const at = input.at ?? new Date().toISOString();
|
|
229
|
+
if (input.kind === "lease" || input.kind === "synthetic-heartbeat") {
|
|
230
|
+
input.node.lastLeaseAt = at;
|
|
231
|
+
// Never treat synthetic lease as meaningful progress.
|
|
232
|
+
return true;
|
|
233
|
+
}
|
|
234
|
+
if (!isMeaningfulActivityKind(input.kind)) {
|
|
235
|
+
return false;
|
|
236
|
+
}
|
|
237
|
+
if (input.kind === "provider") {
|
|
238
|
+
input.node.lastProviderActivityAt = at;
|
|
239
|
+
}
|
|
240
|
+
else if (input.kind === "tool") {
|
|
241
|
+
input.node.lastToolActivityAt = at;
|
|
242
|
+
}
|
|
243
|
+
else if (input.kind === "output") {
|
|
244
|
+
input.node.lastOutputActivityAt = at;
|
|
245
|
+
}
|
|
246
|
+
input.node.lastMeaningfulProgressAt = at;
|
|
247
|
+
input.node.lastActivityAt = at;
|
|
248
|
+
input.node.livenessStatus = "active";
|
|
249
|
+
return true;
|
|
250
|
+
}
|
|
@@ -8,6 +8,8 @@ import { resolveContextPolicy } from "./context-policy.js";
|
|
|
8
8
|
import { buildDagNodePromptEnvelope } from "./prompt.js";
|
|
9
9
|
import { persistLongNodeOutputArtifacts } from "./upstream-artifacts.js";
|
|
10
10
|
import { computeBackoffDelayMs, isRetryablePiFailureCategory, isSafeReadOnlyPiRetryCandidate, } from "./retry-policy.js";
|
|
11
|
+
import { applyNodeActivity, evaluateNodeLiveness, resolveLivenessPolicy, } from "./liveness-policy.js";
|
|
12
|
+
import { buildProtocolRetryInstruction, validateOutputProtocol, } from "./output-protocol.js";
|
|
11
13
|
import { writeDagNodeJsonArtifact } from "../../infrastructure/harness/artifact-store.js";
|
|
12
14
|
import { buildProjectGovernanceContext, readCompletedWriterChangeManifests, writeProjectGovernanceContext, } from "./project-governance-context.js";
|
|
13
15
|
import { assertSkillSnapshotCoversSpec, buildNodePromptFromSnapshot, isDagSkillSnapshotIntegrityError, readSkillSnapshot, } from "./skill-snapshot.js";
|
|
@@ -25,9 +27,19 @@ export function buildNodePrompt(spec, task, upstream, options) {
|
|
|
25
27
|
projectGovernanceContext: options?.projectGovernanceContext,
|
|
26
28
|
});
|
|
27
29
|
}
|
|
28
|
-
function buildAttemptPrompt(task, basePrompt, attemptNumber, previousFailureCategory) {
|
|
29
|
-
if (attemptNumber <= 1
|
|
30
|
-
|
|
30
|
+
function buildAttemptPrompt(task, basePrompt, attemptNumber, previousFailureCategory, previousProtocolReason) {
|
|
31
|
+
if (attemptNumber <= 1)
|
|
32
|
+
return basePrompt;
|
|
33
|
+
if (previousFailureCategory === "protocol-invalid" &&
|
|
34
|
+
task.outputProtocol &&
|
|
35
|
+
previousProtocolReason) {
|
|
36
|
+
return [
|
|
37
|
+
basePrompt,
|
|
38
|
+
"",
|
|
39
|
+
buildProtocolRetryInstruction(task.outputProtocol, previousProtocolReason),
|
|
40
|
+
].join("\n");
|
|
41
|
+
}
|
|
42
|
+
if (task.outputMode !== "structured-required" ||
|
|
31
43
|
previousFailureCategory !== "output-too-large") {
|
|
32
44
|
return basePrompt;
|
|
33
45
|
}
|
|
@@ -224,6 +236,9 @@ export async function executeDagNode(input) {
|
|
|
224
236
|
node.status = "RUNNING";
|
|
225
237
|
node.startedAt = new Date().toISOString();
|
|
226
238
|
node.lastActivityAt = node.startedAt;
|
|
239
|
+
node.lastMeaningfulProgressAt = node.startedAt;
|
|
240
|
+
node.livenessStatus = "active";
|
|
241
|
+
node.currentAttempt = 1;
|
|
227
242
|
if (task.shell?.verifyEvidence) {
|
|
228
243
|
node.verifyEvidence = task.shell.verifyEvidence;
|
|
229
244
|
}
|
|
@@ -299,11 +314,48 @@ export async function executeDagNode(input) {
|
|
|
299
314
|
const maxAttempts = retryPolicy?.maxAttempts ?? 1;
|
|
300
315
|
const attempts = [];
|
|
301
316
|
let totalBackoffMs = 0;
|
|
317
|
+
const livenessPolicy = resolveLivenessPolicy(spec.defaults?.livenessPolicy, task.livenessPolicy);
|
|
318
|
+
/**
|
|
319
|
+
* Attempt-fenced, throttled activity sink. Late events from a previous
|
|
320
|
+
* attempt are no-ops. Persistence is best-effort and never throws to the
|
|
321
|
+
* executor path.
|
|
322
|
+
*/
|
|
323
|
+
const ACTIVITY_PERSIST_MIN_INTERVAL_MS = 2_000;
|
|
324
|
+
let lastActivityPersistMs = 0;
|
|
325
|
+
const reportActivity = (activity) => {
|
|
326
|
+
const accepted = applyNodeActivity({
|
|
327
|
+
node,
|
|
328
|
+
currentAttempt: node.currentAttempt ?? 1,
|
|
329
|
+
activityAttempt: activity.attempt,
|
|
330
|
+
kind: activity.kind,
|
|
331
|
+
at: activity.at,
|
|
332
|
+
});
|
|
333
|
+
if (!accepted)
|
|
334
|
+
return;
|
|
335
|
+
const evaluation = evaluateNodeLiveness({
|
|
336
|
+
node,
|
|
337
|
+
policy: livenessPolicy,
|
|
338
|
+
runnerHeartbeatAt: state.runner?.heartbeatAt,
|
|
339
|
+
});
|
|
340
|
+
node.livenessStatus = evaluation.status;
|
|
341
|
+
const nowMs = Date.now();
|
|
342
|
+
if (nowMs - lastActivityPersistMs < ACTIVITY_PERSIST_MIN_INTERVAL_MS) {
|
|
343
|
+
return;
|
|
344
|
+
}
|
|
345
|
+
lastActivityPersistMs = nowMs;
|
|
346
|
+
// Canonical mid-call activity lives in active state.json. Queue through the
|
|
347
|
+
// runner's serialized state writer; per-node records remain terminal/attempt
|
|
348
|
+
// evidence so a late best-effort write cannot recreate an archived run dir.
|
|
349
|
+
void input.persistState().catch(() => { });
|
|
350
|
+
};
|
|
302
351
|
let terminalResult;
|
|
303
352
|
let previousFailureCategory;
|
|
353
|
+
let previousProtocolReason;
|
|
304
354
|
for (let attemptNumber = 1; attemptNumber <= maxAttempts; attemptNumber += 1) {
|
|
305
355
|
const attemptStartedAt = new Date().toISOString();
|
|
306
356
|
const attemptStarted = Date.now();
|
|
357
|
+
node.currentAttempt = attemptNumber;
|
|
358
|
+
node.livenessStatus = "active";
|
|
307
359
|
let result;
|
|
308
360
|
try {
|
|
309
361
|
validateRepairArtifactGateBeforeShell({
|
|
@@ -315,7 +367,12 @@ export async function executeDagNode(input) {
|
|
|
315
367
|
task,
|
|
316
368
|
cwd,
|
|
317
369
|
model,
|
|
318
|
-
prompt: buildAttemptPrompt(task, prompt, attemptNumber, previousFailureCategory),
|
|
370
|
+
prompt: buildAttemptPrompt(task, prompt, attemptNumber, previousFailureCategory, previousProtocolReason),
|
|
371
|
+
attempt: attemptNumber,
|
|
372
|
+
reportActivity,
|
|
373
|
+
timeoutMs: livenessPolicy.absoluteMaxWallClockMs,
|
|
374
|
+
stallTimeoutMs: livenessPolicy.stallProbeMs,
|
|
375
|
+
abortGraceMs: livenessPolicy.abortGraceMs,
|
|
319
376
|
});
|
|
320
377
|
}
|
|
321
378
|
catch (error) {
|
|
@@ -326,6 +383,26 @@ export async function executeDagNode(input) {
|
|
|
326
383
|
durationMs: Date.now() - attemptStarted,
|
|
327
384
|
};
|
|
328
385
|
}
|
|
386
|
+
// R0: executor ok=true still fails closed when outputProtocol is violated.
|
|
387
|
+
// Valid semantic results (e.g. VERDICT: request-revision) pass validation.
|
|
388
|
+
if (result.ok && task.outputProtocol) {
|
|
389
|
+
const protocolText = `${result.assistantText ?? ""}\n${result.stdout ?? ""}`;
|
|
390
|
+
const protocolCheck = validateOutputProtocol(task.outputProtocol, protocolText);
|
|
391
|
+
if (!protocolCheck.ok) {
|
|
392
|
+
result = {
|
|
393
|
+
...result,
|
|
394
|
+
ok: false,
|
|
395
|
+
failureCategory: protocolCheck.failureCategory,
|
|
396
|
+
stderr: [result.stderr, protocolCheck.reason]
|
|
397
|
+
.filter(Boolean)
|
|
398
|
+
.join("\n"),
|
|
399
|
+
};
|
|
400
|
+
previousProtocolReason = protocolCheck.reason;
|
|
401
|
+
}
|
|
402
|
+
else {
|
|
403
|
+
previousProtocolReason = undefined;
|
|
404
|
+
}
|
|
405
|
+
}
|
|
329
406
|
const attemptFinishedAt = new Date().toISOString();
|
|
330
407
|
const attemptRecord = {
|
|
331
408
|
attempt: attemptNumber,
|
|
@@ -373,6 +450,10 @@ export async function executeDagNode(input) {
|
|
|
373
450
|
? result.parsedEvents
|
|
374
451
|
: sumAttemptMetric(attempts, (attempt) => attempt.parsedEvents);
|
|
375
452
|
node.lastActivityAt = attemptFinishedAt;
|
|
453
|
+
if (result.failureCategory === "termination-unconfirmed") {
|
|
454
|
+
node.needsAttentionReason = "attempt-termination-unconfirmed";
|
|
455
|
+
node.livenessStatus = "needs-attention";
|
|
456
|
+
}
|
|
376
457
|
if (retryPolicy !== undefined) {
|
|
377
458
|
state.nodes[nodeId].nodeRecordPath = path.join(runDir, `${nodeId}.json`);
|
|
378
459
|
await writeNodeRecord(runDir, nodeId, state.nodes[nodeId]);
|
|
@@ -384,9 +465,11 @@ export async function executeDagNode(input) {
|
|
|
384
465
|
break;
|
|
385
466
|
const canRetry = retryPolicy !== undefined && attemptNumber < maxAttempts;
|
|
386
467
|
const retryable = retryPolicy !== undefined &&
|
|
387
|
-
isRetryablePiFailureCategory(result.failureCategory, {
|
|
468
|
+
(isRetryablePiFailureCategory(result.failureCategory, {
|
|
388
469
|
retryCategories: retryPolicy.retryCategories,
|
|
389
|
-
})
|
|
470
|
+
}) ||
|
|
471
|
+
(result.failureCategory === "protocol-invalid" &&
|
|
472
|
+
task.outputProtocol?.retryOnInvalid === true));
|
|
390
473
|
if (!canRetry || !retryable)
|
|
391
474
|
break;
|
|
392
475
|
const delayMs = computeBackoffDelayMs(attemptNumber + 1, retryPolicy);
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
/**
|
|
3
|
+
* Machine-readable output protocol for safe read-only Pi nodes.
|
|
4
|
+
*
|
|
5
|
+
* Phase 1 only supports first-line-enum (e.g. VERDICT lines). Structured JSON
|
|
6
|
+
* continues to use existing structured-required / deterministic gates.
|
|
7
|
+
*/
|
|
8
|
+
export const dagOutputProtocolSchema = z
|
|
9
|
+
.object({
|
|
10
|
+
type: z.literal("first-line-enum"),
|
|
11
|
+
validLines: z.array(z.string().min(1)).min(1),
|
|
12
|
+
retryOnInvalid: z.boolean().default(true),
|
|
13
|
+
})
|
|
14
|
+
.strict();
|
|
15
|
+
/** Reviewer VERDICT protocol used by reviewed/supervised DAGs. */
|
|
16
|
+
export const REVIEW_VERDICT_OUTPUT_PROTOCOL = {
|
|
17
|
+
type: "first-line-enum",
|
|
18
|
+
validLines: ["VERDICT: pass", "VERDICT: request-revision"],
|
|
19
|
+
retryOnInvalid: true,
|
|
20
|
+
};
|
|
21
|
+
/**
|
|
22
|
+
* Extract the first non-empty line from assistant/stdout text.
|
|
23
|
+
*/
|
|
24
|
+
export function firstNonEmptyLine(text) {
|
|
25
|
+
for (const line of text.split("\n")) {
|
|
26
|
+
const trimmed = line.trim();
|
|
27
|
+
if (trimmed)
|
|
28
|
+
return trimmed;
|
|
29
|
+
}
|
|
30
|
+
return undefined;
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* Validate node output against an explicit outputProtocol.
|
|
34
|
+
* Pure function — does not mutate run facts.
|
|
35
|
+
*/
|
|
36
|
+
export function validateOutputProtocol(protocol, text) {
|
|
37
|
+
if (protocol.type !== "first-line-enum") {
|
|
38
|
+
return {
|
|
39
|
+
ok: false,
|
|
40
|
+
failureCategory: "protocol-invalid",
|
|
41
|
+
reason: `unsupported outputProtocol.type: ${protocol.type ?? "unknown"}`,
|
|
42
|
+
};
|
|
43
|
+
}
|
|
44
|
+
const first = firstNonEmptyLine(text);
|
|
45
|
+
if (!first) {
|
|
46
|
+
return {
|
|
47
|
+
ok: false,
|
|
48
|
+
failureCategory: "protocol-invalid",
|
|
49
|
+
reason: `missing first non-empty line; expected one of: ${protocol.validLines.map((l) => JSON.stringify(l)).join(" or ")}`,
|
|
50
|
+
};
|
|
51
|
+
}
|
|
52
|
+
if (!protocol.validLines.includes(first)) {
|
|
53
|
+
return {
|
|
54
|
+
ok: false,
|
|
55
|
+
failureCategory: "protocol-invalid",
|
|
56
|
+
reason: `first non-empty line ${JSON.stringify(first)} is not a valid protocol line; expected one of: ${protocol.validLines.map((l) => JSON.stringify(l)).join(" or ")}`,
|
|
57
|
+
firstNonEmptyLine: first,
|
|
58
|
+
};
|
|
59
|
+
}
|
|
60
|
+
return { ok: true, matchedLine: first };
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* Correction instruction appended on protocol-invalid retry attempts.
|
|
64
|
+
*/
|
|
65
|
+
export function buildProtocolRetryInstruction(protocol, reason) {
|
|
66
|
+
const expected = protocol.validLines
|
|
67
|
+
.map((line) => JSON.stringify(line))
|
|
68
|
+
.join(" or ");
|
|
69
|
+
return [
|
|
70
|
+
"<retry_instruction>",
|
|
71
|
+
"Previous attempt violated the output protocol:",
|
|
72
|
+
reason,
|
|
73
|
+
`Return one valid protocol line before any explanation. Expected first non-empty line: ${expected}.`,
|
|
74
|
+
"</retry_instruction>",
|
|
75
|
+
].join("\n");
|
|
76
|
+
}
|