@tea-agent/loop-agent 0.20.1 → 0.21.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (31) hide show
  1. package/CHANGELOG.md +26 -0
  2. package/dist/commands/init.js +7 -0
  3. package/dist/executors/dag-pi-executor.js +24 -0
  4. package/dist/executors/pi-executor.js +111 -36
  5. package/dist/executors/pi-sdk-executor.js +105 -29
  6. package/dist/executors/shell-executor.js +54 -11
  7. package/dist/worker/loop-agent/loop-agent-client.js +43 -9
  8. package/dist/worker/observability/read-model.js +7 -1
  9. package/dist/worker/observe/static/constants.js +5 -0
  10. package/dist/worker/observe/static/format-pool.js +22 -3
  11. package/dist/worker/observe/static/styles.css +32 -3
  12. package/dist/worker/observe/static/views/dag-inspector.js +2 -2
  13. package/dist/worker/run-task/run-task.js +16 -6
  14. package/dist/workflows/dag/backend-test-markdown-workflow.js +291 -97
  15. package/dist/workflows/dag/backend-test-result-contract.js +10 -4
  16. package/dist/workflows/dag/init-hybrid.js +27 -16
  17. package/dist/workflows/dag/lifecycle.js +60 -4
  18. package/dist/workflows/dag/liveness-policy.js +250 -0
  19. package/dist/workflows/dag/node-execution.js +49 -0
  20. package/dist/workflows/dag/runner.js +21 -1
  21. package/dist/workflows/dag/types.js +5 -0
  22. package/docs/README.md +5 -6
  23. package/docs/architecture/dag-execution.md +11 -0
  24. package/docs/architecture/facts-and-state.md +1 -0
  25. package/docs/architecture/worker-and-feature.md +10 -0
  26. package/docs/templates/backend-test-dag.generate-pytest.prompt.md +5 -2
  27. package/docs/templates/backend-test-dag.json +15 -15
  28. package/harness.json +1 -1
  29. package/package.json +1 -1
  30. package/skills/loop-agent/references/command-reference.md +2 -0
  31. package/skills/loop-agent/references/hybrid-dag.md +2 -2
@@ -2801,12 +2801,18 @@ function buildBackendTestSemanticReviewNode(sources, options = {}) {
2801
2801
  }
2802
2802
  function collectBackendTestShellEnvAllowlist(sources) {
2803
2803
  const names = new Set();
2804
- for (const verify of sources.taskConfig.verifyCommands) {
2804
+ const collectAssignments = (text) => {
2805
2805
  const assignmentPattern = /(?:^|[\s;&|])([A-Z_][A-Z0-9_]*)\s*=/g;
2806
- for (const match of verify.command.matchAll(assignmentPattern)) {
2806
+ for (const match of text.matchAll(assignmentPattern)) {
2807
2807
  if (match[1])
2808
2808
  names.add(match[1]);
2809
2809
  }
2810
+ };
2811
+ for (const constraint of sources.taskConfig.hardConstraints) {
2812
+ collectAssignments(constraint);
2813
+ }
2814
+ for (const verify of sources.taskConfig.verifyCommands) {
2815
+ collectAssignments(verify.command);
2810
2816
  }
2811
2817
  return [...names].sort();
2812
2818
  }
@@ -3051,7 +3057,7 @@ async function buildBackendTestHybridDag(sources) {
3051
3057
  "Read the upstream environment report. Generate a Markdown-first backend test strategy and cases under testcase/md/**.",
3052
3058
  "Write human-readable content in Simplified Chinese by default. Keep English only for machine-readable IDs and technical literals such as Case/AC/REQ/BR IDs, HTTP methods, paths, field names, enum values, commands, filenames, code symbols and exact source citations.",
3053
3059
  "Create testcase/md/README.md as the concise entry page: test objective, target/environment, isolation/cleanup, module summary and a linked case index table with Case ID, Chinese case name, scenario type, endpoint and expected status/result. Avoid repeating every case body in README.",
3054
- "Write each module as readable case cards. Every case starts with `## BE-<MODULE>-<NNN>|<中文用例名称>` and uses these Chinese headings: `### 测试目的`, `### 验收标准`, `### 需求依据`, `### 前置条件`, optional `### 测试数据`, `### 操作步骤`, `### 预期结果`, and `### 自动化映射`. API metadata may use a compact table under the case heading. The deterministic validator also accepts legacy English headings, but new output should use this Chinese presentation.",
3060
+ "Write each module as readable case cards. Every case starts with `## BE-<MODULE>-<NNN>|<中文用例名称>`. The only sections required by the deterministic validator are `### 前置条件`, `### 操作步骤`, and `### 预期结果` (legacy English aliases remain accepted). Add `测试目的`, `验收标准`, `需求依据`, `测试数据`, and `自动化映射` when useful for human readability; every automatable case should explicitly name its target pytest script under `自动化映射` so traceability can scan only that script.",
3055
3061
  "Place steps and their expected results in a compact readable table when that improves clarity; otherwise keep numbered executable steps and numbered/bulleted independently assertable results. Every result must name the observable HTTP status, response field/value, state transition or membership condition, never vague wording such as ‘符合预期’.",
3056
3062
  "In `自动化映射`, record the planned script path and pytest function name when known. Put implementation-only restrictions in a concise `<details>` block rather than dominating the main case flow. Use only environment-supported fixtures/targets/isolation, record evidence gaps in Chinese, and do not emit JSON, pytest, or execute commands.",
3057
3063
  intake.boundedSourceContext, "## Authoritative reference index", JSON.stringify(intake.referenceIndex, null, 2),
@@ -3065,7 +3071,7 @@ async function buildBackendTestHybridDag(sources) {
3065
3071
  writeSet: ["testcase/md/**"], allowedPaths: ["testcase/md/**"], forbiddenPaths: forbidden,
3066
3072
  outputContract: "Review source fidelity and directly revise only testcase/md/**; return concise Markdown, never JSON.",
3067
3073
  subtask_prompt: [
3068
- "Independently review generated Markdown cases against each case 需求依据 and environment evidence. Treat the files as human-facing test documentation: require a clear Chinese name and scenario/purpose, compact metadata, readable steps/results, and a concise automation mapping while preserving exact machine IDs and technical literals.",
3074
+ "Independently review generated Markdown cases against the task requirements and environment evidence. Treat the files as human-facing test documentation: require clear preconditions, executable steps and assertable expected results; improve names, purpose, metadata and automation mapping where useful while preserving exact machine IDs and technical literals.",
3069
3075
  "Check AC completeness/meaning, endpoint, fields/shape, status/error codes, rules, states, documented boundaries/auth, positive/negative coverage, executable steps and assertable results. Reject avoidable English prose, duplicated bilingual wording, repeated boilerplate, oversized unstructured sections, vague results such as ‘符合预期’, and missing script/function mapping where it can be derived.",
3070
3076
  "Correct testcase/md/** directly: add documented omissions, remove unsupported cases, fix mappings/expectations, merge duplicates, improve navigation/tables/Chinese wording, or record gaps in Chinese. Keep Case IDs, AC/REQ/BR IDs, HTTP methods, paths, fields, enum values, filenames, code symbols and source citations exact. The validator accepts Chinese and legacy English section aliases; retain or converge to the Chinese human-readable headings without losing structure.",
3071
3077
  "Read only precise referenced source paths plus requirement sections needed for uncovered ACs. Do not scan the repository, modify source/**, generate pytest, execute tests, or emit JSON.",
@@ -3073,7 +3079,7 @@ async function buildBackendTestHybridDag(sources) {
3073
3079
  "For each index entry, use `readPath` for Pi read-tool calls and keep `path` as the exact Markdown Source References citation. Bound files under .harness/tasks/<taskId>/source/** are read-only inputs: reading them is allowed even though writing .harness/** is forbidden. Never resolve `path` relative to the repository root, search for substitutes, or fall back to docs/** when a bound read fails.",
3074
3080
  ].join("\n\n"),
3075
3081
  };
3076
- const validateCases = shellNode("validate-backend-md-cases-shell", [reviewCases.id], "markdown-cases", "Fail closed on missing/duplicate IDs, sections, AC coverage, source references, executable steps, assertable results, placeholders or secret-shaped content.", "Run-owned reports/backend-md-case-validation.md proving final Markdown quality and safety.");
3082
+ const validateCases = shellNode("validate-backend-md-cases-shell", [reviewCases.id], "markdown-cases", "Record advisory findings for missing/duplicate IDs, missing core sections (preconditions, steps, expected results), AC coverage, executable steps, assertable results or placeholders. Do not validate source-reference existence. Keep quality findings advisory, but fail closed after writing the report when secret-shaped values are detected so downstream pytest/report nodes cannot consume them.", "Run-owned reports/backend-md-case-validation.md with PASS/FAIL advisory findings; downstream execution continues.");
3077
3083
  const generatePytest = {
3078
3084
  id: "generate-backend-pytest-pi", depends_on: [validateCases.id], role: "implementer",
3079
3085
  executor: "pi", toolProfile: "write", complexity: "HIGH", writePolicy: "exclusive",
@@ -3081,20 +3087,23 @@ async function buildBackendTestHybridDag(sources) {
3081
3087
  allowedPaths: Array.from(new Set([...ro, "testcase/**"])), forbiddenPaths: forbidden,
3082
3088
  outputContract: "Convert every final automatable Markdown case into pytest assets whose actual test function region contains the exact Case ID, preferably in the function name or docstring; no JSON and no pytest execution.",
3083
3089
  subtask_prompt: [
3084
- "Convert validated testcase/md/** to pytest using upstream environment and validation evidence plus only bounded pytest config/conftest.",
3085
- "Ensure every final Markdown Case ID appears in at least one real pytest test function or pytest test class method region, preferably as `test_BE_<MODULE>_<NNN>_<description>` and in that function/method docstring. Module-level functions and class-based pytest methods are both supported. Multiple test functions may cover one Case ID; assertions come only from 预期结果/Expected Results and setup comes only from 前置条件/测试数据/自动化映射 or their legacy English aliases.",
3090
+ "Convert testcase/md/** to pytest using upstream environment and advisory validation evidence plus only bounded pytest config/conftest. A FAIL advisory report does not authorize inventing missing behavior; use the final Markdown facts that are present.",
3091
+ "Ensure every final Markdown Case ID appears in at least one real pytest test function or pytest test class method region, preferably as `test_BE_<MODULE>_<NNN>_<description>` and in that function/method docstring. Module-level functions and class-based pytest methods are both supported. Multiple test functions may cover one Case ID; assertions come only from 预期结果/Expected Results and setup comes only from 前置条件 plus any optional 测试数据/自动化映射 or their legacy English aliases.",
3092
+ "Generate a reusable HTTP logging helper (or equivalent client wrapper) and call it for every interface request. The request log must include method, URL/path, and request parameters (query plus JSON/body/payload summary). The response log must include status code and response result (JSON/text/body summary), and both records must be visible in pytest stdout/stderr without changing assertions.",
3093
+ "Compare timestamps and other semantically equivalent protocol values by parsed meaning, not byte-for-byte serialization. In particular, normalize valid ISO-8601 instants before equality/order assertions so differences such as omitted trailing fractional seconds do not create TestBug failures; preserve exact-string assertions only when the Markdown explicitly requires representation equality.",
3094
+ "Before logging, recursively redact sensitive keys and header values including authorization, proxy-authorization, cookie, set-cookie, token, password, secret, api key and credentials. Never print full Authorization/Cookie values. Apply bounded truncation to serialized request and response bodies (with an explicit truncation marker) so large payloads cannot flood pytest or report artifacts.",
3086
3095
  "Do not read source/**, add cases, reassign ACs, modify conftest/config/production code, use skip/xfail, swallow assertions, execute pytest, or emit JSON. For best-effort cleanup, catch only the narrow transport exception actually raised by the selected HTTP client (for example `requests.RequestException` or `urllib.error.URLError`); never use bare `except`, `Exception`, or `BaseException` with `pass`.",
3087
3096
  ].join("\n\n"),
3088
3097
  };
3089
- const traceability = shellNode("backend-test-traceability-gate-shell", [generatePytest.id], "markdown-traceability", "Fail closed only when a real Markdown case heading has no associated pytest test function or class method. Accept exact Case IDs in the function/method name or its decorator/body/docstring region; report multiple mappings and extra automation Case IDs without blocking. Continue to reject skip/xfail or swallowed exceptions.", "Run-owned reports/backend-test-traceability.md proving every real Markdown Case ID is covered by at least one pytest test function.");
3098
+ const traceability = shellNode("backend-test-traceability-gate-shell", [generatePytest.id], "markdown-traceability", "Record advisory findings when a real Markdown case heading has no associated pytest test function or class method in the script explicitly mapped by that Markdown case, or when a mapped HTTP test script lacks request parameters logging, response result logging, recursive redaction or bounded truncation evidence. Accept exact Case IDs in the function/method name or its decorator/body/docstring region. Do not scan unrelated test_*.py files and do not block pytest execution.", "Run-owned reports/backend-test-traceability.md with PASS/FAIL advisory findings for Markdown Case to mapped pytest script/symbol coverage.");
3090
3099
  const pytestCommand = [
3091
3100
  'mkdir -p "${HARNESS_DAG_RUN_DIR}/reports"',
3092
- 'PYTHONUTF8=1 PYTHONIOENCODING=utf-8 PYTHONDONTWRITEBYTECODE=1 python -m pytest testcase/ -v -p no:cacheprovider --junitxml="${HARNESS_DAG_RUN_DIR}/reports/backend-test.junit.xml"',
3093
- "STATUS=$?", 'printf "%s" "${STATUS}" > "${HARNESS_DAG_RUN_DIR}/reports/backend-test-pytest-exit.txt"',
3094
- 'if { [ "${STATUS}" -eq 0 ] || [ "${STATUS}" -eq 1 ]; } && [ -s "${HARNESS_DAG_RUN_DIR}/reports/backend-test.junit.xml" ]; then exit 0; fi',
3095
- 'exit "${STATUS}"',
3101
+ 'echo "pytest targets are resolved at runtime from final Markdown 自动化映射"',
3096
3102
  ].join("; ");
3097
- const execute = shellNode("execute-backend-pytest-and-html-report-shell", [traceability.id], "markdown-execute-html", "Execute pytest exactly once. Validate JUnit, render the primary self-contained Chinese HTML report from the same JUnit plus final Markdown case metadata without rerun, list every case with name/scenario/script/function/result/duration, and preserve failure summaries plus expandable technical details as facts.", "One pytest execution producing valid JUnit, self-contained HTML and reports/backend-test-facts.md; exit 0/1 with valid evidence continues.", [pytestCommand], 300000);
3103
+ const execute = shellNode("execute-backend-pytest-and-html-report-shell", [traceability.id], "markdown-execute-html", "Resolve the final Markdown Automation Notes/自动化映射 to a unique, safe set of testcase/**/test_*.py targets and execute only those scripts exactly once. Validate JUnit, then render the primary self-contained Chinese HTML report from the same JUnit plus final Markdown case metadata without rerun. Keep 测试结论, quality status, failure overview, and a polished per-case result card with concise scenario, automation test name, result, duration, and redacted bounded HTTP request parameters/response results for both passed and failed cases. Do not render a technical/execution evidence section in HTML; retain auditable paths and hashes in facts.", "One scoped pytest execution over Markdown-mapped scripts producing valid JUnit with per-case captured output, self-contained HTML and reports/backend-test-facts.md; exit 0/1 with valid evidence continues.", [pytestCommand], 300000);
3104
+ if (execute.shell) {
3105
+ execute.shell.envAllowlist = collectBackendTestShellEnvAllowlist(sources);
3106
+ }
3098
3107
  const canWriteReport = taskAllowsBackendTestReportWrite(sources);
3099
3108
  const report = {
3100
3109
  id: "backend-test-report-and-l5-pi", depends_on: [execute.id], role: "closeout", executor: "pi", complexity: "MED",
@@ -3104,7 +3113,9 @@ async function buildBackendTestHybridDag(sources) {
3104
3113
  forbiddenPaths: forbidden,
3105
3114
  outputContract: canWriteReport ? "Final Markdown report and L-5 conclusion under docs/test-reports/**; no JSON." : "Final Markdown report and L-5 conclusion in assistant output; no JSON or writes.",
3106
3115
  subtask_prompt: [
3107
- "Generate the final Markdown report from upstream facts and run-owned environment, case-validation, traceability, JUnit and HTML evidence. Do not emit JSON.",
3116
+ "Generate the final Markdown report from upstream facts and run-owned environment, advisory case-validation, advisory traceability, JUnit and HTML evidence. Do not emit JSON.",
3117
+ "Use this exact human-facing section order: 测试结论 → 执行概览 → 质量校验 → 失败分析 → 风险与建议 → 证据与 L-5. Put the decision and key numbers first, use compact tables/bullets, and keep headings concise. Do not paste entire upstream reports, duplicate per-case tables already present in facts, or repeat the same evidence in multiple sections; link to paths/hashes and quote only the findings needed for the conclusion.",
3118
+ "Always state the exact PASS/FAIL status and findings from nodes 4 and 6. Their FAIL status does not block pytest, but it must remain visible as a quality/traceability risk and must never be rewritten as PASS.",
3108
3119
  "Include environment, case quality/review, automation mapping, exact pytest facts, failure classification/analysis, risks, regression recommendations, evidence paths/hashes, coverage/stability availability, and L-5 READY/NOT READY.",
3109
3120
  "Never override Shell/JUnit facts. One run cannot prove FlakyTest. Missing coverage/stability is Unavailable. L-5 requires pass=100%, AC=100%, automation>=90%, stability>=95% n>=5, line>=80%, branch>=70%, skipped=0 and no blocking Critical risk.",
3110
3121
  canWriteReport ? "Write only under docs/test-reports/**." : "Keep the full report in assistant output.",
@@ -3118,9 +3129,9 @@ async function buildBackendTestHybridDag(sources) {
3118
3129
  successCriteria: extractSuccessCriteria(sources.requirementMarkdown, sources.taskId),
3119
3130
  globalConstraints: [
3120
3131
  ...taskConfig.hardConstraints, ...STANDARD_GLOBAL_CONSTRAINTS,
3121
- "backend-test-dag uses exactly 8 real top-level tasks and executes pytest exactly once.",
3132
+ "backend-test-dag uses exactly 8 real top-level tasks and executes pytest exactly once over only the safe scripts explicitly mapped by final Markdown cases.",
3122
3133
  "Model nodes produce Markdown and pytest assets, never backend-test business JSON envelopes.",
3123
- "Environment, Markdown validation, traceability, JUnit, HTML and execution facts are deterministic fail-closed evidence.",
3134
+ "Environment, advisory Markdown validation, advisory traceability, JUnit, HTML and execution facts are deterministic evidence. Nodes 4 and 6 record findings without blocking nodes 5, 7 or 8.",
3124
3135
  "Only Markdown case generation/review may read source facts; pytest generation must not read source/**.",
3125
3136
  "Functional case IDs use BE-<MODULE>-<NNN>; production code/config, skip/xfail, repair and rerun are forbidden.",
3126
3137
  ],
@@ -187,10 +187,38 @@ export function assessDagRunLiveness(input) {
187
187
  return { status: "stale", runnerAlive: true };
188
188
  }
189
189
  const activeNode = Object.values(input.state.nodes).find((node) => node.status === "RUNNING");
190
- const nodeActivityMs = Date.parse(activeNode?.lastActivityAt ?? activeNode?.startedAt ?? "");
191
- if (!Number.isNaN(nodeActivityMs) && nowMs - nodeActivityMs > (input.nodeQuietThresholdMs ?? 300_000)) {
190
+ if (!activeNode)
191
+ return { status: "active", runnerAlive: true };
192
+ // Prefer persisted adaptive projection when present.
193
+ if (activeNode.livenessStatus === "needs-attention") {
194
+ return { status: "needs-attention", runnerAlive: true };
195
+ }
196
+ if (activeNode.livenessStatus === "suspected-stall"
197
+ || activeNode.livenessStatus === "probing") {
198
+ return { status: "suspected-stall", runnerAlive: true };
199
+ }
200
+ if (activeNode.livenessStatus === "quiet") {
192
201
  return { status: "node-quiet", runnerAlive: true };
193
202
  }
203
+ // Fall back to meaningful progress clocks (never use runner lease as progress).
204
+ const meaningfulAt = activeNode.lastMeaningfulProgressAt
205
+ ?? activeNode.lastProviderActivityAt
206
+ ?? activeNode.lastToolActivityAt
207
+ ?? activeNode.lastOutputActivityAt
208
+ ?? activeNode.lastActivityAt
209
+ ?? activeNode.startedAt;
210
+ const nodeActivityMs = Date.parse(meaningfulAt ?? "");
211
+ if (!Number.isNaN(nodeActivityMs)) {
212
+ const idleMs = nowMs - nodeActivityMs;
213
+ const stallMs = input.nodeStallThresholdMs ?? 900_000;
214
+ const quietMs = input.nodeQuietThresholdMs ?? 300_000;
215
+ if (idleMs > stallMs) {
216
+ return { status: "suspected-stall", runnerAlive: true };
217
+ }
218
+ if (idleMs > quietMs) {
219
+ return { status: "node-quiet", runnerAlive: true };
220
+ }
221
+ }
194
222
  return { status: "active", runnerAlive: true };
195
223
  }
196
224
  export function deriveDagRunEffectiveStatus(input) {
@@ -210,6 +238,10 @@ export function deriveDagRunEffectiveStatus(input) {
210
238
  }
211
239
  if (input.liveness === "orphaned" || input.liveness === "stale")
212
240
  return "interrupted";
241
+ if (input.liveness === "needs-attention")
242
+ return "needs-attention";
243
+ if (input.liveness === "suspected-stall")
244
+ return "running-suspected-stall";
213
245
  if (input.liveness === "node-quiet")
214
246
  return "running-quiet";
215
247
  if (input.liveness === "unknown-host")
@@ -232,7 +264,15 @@ export function assessDagRunRecoveryEligibility(input) {
232
264
  reasons.push("run-already-terminal");
233
265
  }
234
266
  if (input.lifecycle === "active"
235
- && ["active", "node-quiet", "stale", "unknown-host", "unknown"].includes(input.liveness)) {
267
+ && [
268
+ "active",
269
+ "node-quiet",
270
+ "suspected-stall",
271
+ "needs-attention",
272
+ "stale",
273
+ "unknown-host",
274
+ "unknown",
275
+ ].includes(input.liveness)) {
236
276
  canReconcile = false;
237
277
  reasons.push("runner-not-proven-dead-or-stopped");
238
278
  }
@@ -358,10 +398,26 @@ export async function detectDagRunHealthIssues(input) {
358
398
  issues.push({
359
399
  code: "node-activity-quiet",
360
400
  severity: "warning",
361
- message: "Runner heartbeat is fresh but the current RUNNING node has produced no state activity for more than 5 minutes",
401
+ message: "Runner heartbeat is fresh but the current RUNNING node has produced no meaningful activity for more than 5 minutes",
362
402
  advisoryAction: "Inspect the node session events and executor logs before deciding whether to wait or abort.",
363
403
  });
364
404
  }
405
+ else if (liveness.status === "suspected-stall") {
406
+ issues.push({
407
+ code: "node-activity-suspected-stall",
408
+ severity: "warning",
409
+ message: "Runner heartbeat is fresh but the current RUNNING node has no meaningful Provider/tool/output activity for more than 15 minutes",
410
+ advisoryAction: "Treat as suspected network/provider stall; only retry read-only nodes after the attempt is proven finished. Writers must fail closed to reconcile.",
411
+ });
412
+ }
413
+ else if (liveness.status === "needs-attention") {
414
+ issues.push({
415
+ code: "node-needs-attention",
416
+ severity: "error",
417
+ message: "Runner or node process identity cannot be proven healthy, or absolute max wall clock was exceeded",
418
+ advisoryAction: "Do not forge timed-out/cancelled. Inspect process identity and artifacts; reconcile only when exit is proven.",
419
+ });
420
+ }
365
421
  if (lifecycle === "active" &&
366
422
  state.status === "running" &&
367
423
  state.humanDecisionNodeId) {
@@ -0,0 +1,250 @@
1
+ import { z } from "zod";
2
+ /**
3
+ * Adaptive liveness policy for DAG/Pi supervision.
4
+ *
5
+ * Four clocks must not be mixed:
6
+ * - runner lease (synthetic heartbeat) — not meaningful progress
7
+ * - provider/transport activity
8
+ * - tool activity
9
+ * - output / meaningful progress
10
+ */
11
+ export const DEFAULT_LIVENESS_POLICY = {
12
+ heartbeatIntervalMs: 15_000,
13
+ runnerStaleMs: 90_000,
14
+ quietMs: 300_000,
15
+ stallProbeMs: 900_000,
16
+ abortGraceMs: 30_000,
17
+ /** 4h absolute max wall clock — cannot be renewed by empty heartbeats. */
18
+ absoluteMaxWallClockMs: 14_400_000,
19
+ };
20
+ export const dagLivenessPolicySchema = z
21
+ .object({
22
+ heartbeatIntervalMs: z.number().int().min(1_000).optional(),
23
+ runnerStaleMs: z.number().int().positive().optional(),
24
+ quietMs: z.number().int().positive().optional(),
25
+ stallProbeMs: z.number().int().positive().optional(),
26
+ abortGraceMs: z.number().int().positive().optional(),
27
+ absoluteMaxWallClockMs: z.number().int().positive().optional(),
28
+ })
29
+ .strict()
30
+ .superRefine((value, ctx) => {
31
+ const resolved = { ...DEFAULT_LIVENESS_POLICY, ...value };
32
+ if (resolved.runnerStaleMs <= resolved.heartbeatIntervalMs) {
33
+ ctx.addIssue({
34
+ code: z.ZodIssueCode.custom,
35
+ path: ["runnerStaleMs"],
36
+ message: "runnerStaleMs must be greater than heartbeatIntervalMs",
37
+ });
38
+ }
39
+ if (resolved.stallProbeMs <= resolved.quietMs) {
40
+ ctx.addIssue({
41
+ code: z.ZodIssueCode.custom,
42
+ path: ["stallProbeMs"],
43
+ message: "stallProbeMs must be greater than quietMs",
44
+ });
45
+ }
46
+ if (resolved.absoluteMaxWallClockMs <= resolved.stallProbeMs) {
47
+ ctx.addIssue({
48
+ code: z.ZodIssueCode.custom,
49
+ path: ["absoluteMaxWallClockMs"],
50
+ message: "absoluteMaxWallClockMs must be greater than stallProbeMs",
51
+ });
52
+ }
53
+ })
54
+ .optional();
55
+ /**
56
+ * Resolve policy from node/defaults partials. Missing fields use conservative defaults.
57
+ */
58
+ export function resolveLivenessPolicy(...sources) {
59
+ const merged = {};
60
+ for (const source of sources) {
61
+ if (!source)
62
+ continue;
63
+ for (const key of Object.keys(DEFAULT_LIVENESS_POLICY)) {
64
+ const value = source[key];
65
+ if (typeof value === "number" && Number.isFinite(value) && value > 0) {
66
+ merged[key] = Math.trunc(value);
67
+ }
68
+ }
69
+ }
70
+ return {
71
+ heartbeatIntervalMs: merged.heartbeatIntervalMs ?? DEFAULT_LIVENESS_POLICY.heartbeatIntervalMs,
72
+ runnerStaleMs: merged.runnerStaleMs ?? DEFAULT_LIVENESS_POLICY.runnerStaleMs,
73
+ quietMs: merged.quietMs ?? DEFAULT_LIVENESS_POLICY.quietMs,
74
+ stallProbeMs: merged.stallProbeMs ?? DEFAULT_LIVENESS_POLICY.stallProbeMs,
75
+ abortGraceMs: merged.abortGraceMs ?? DEFAULT_LIVENESS_POLICY.abortGraceMs,
76
+ absoluteMaxWallClockMs: merged.absoluteMaxWallClockMs ??
77
+ DEFAULT_LIVENESS_POLICY.absoluteMaxWallClockMs,
78
+ };
79
+ }
80
+ /**
81
+ * Synthetic timer heartbeats never count as meaningful Pi progress.
82
+ */
83
+ export function isMeaningfulActivityKind(kind) {
84
+ return kind === "provider" || kind === "tool" || kind === "output";
85
+ }
86
+ /**
87
+ * Attempt fence: late activity from a previous attempt must not overwrite the current one.
88
+ */
89
+ export function shouldAcceptActivity(currentAttempt, activityAttempt) {
90
+ if (!Number.isInteger(currentAttempt) || currentAttempt < 1)
91
+ return false;
92
+ if (!Number.isInteger(activityAttempt) || activityAttempt < 1)
93
+ return false;
94
+ return activityAttempt === currentAttempt;
95
+ }
96
+ export function classifySessionEventActivity(event) {
97
+ if (!event || typeof event !== "object")
98
+ return "provider";
99
+ const type = typeof event.type === "string"
100
+ ? String(event.type)
101
+ : "";
102
+ if (type === "tool_start" ||
103
+ type === "tool_end" ||
104
+ type === "tool_execution_start" ||
105
+ type === "tool_execution_end") {
106
+ return "tool";
107
+ }
108
+ if (type === "thinking_delta" || type === "message_update") {
109
+ // Noise — do not treat as progress when callers filter via shouldPersistSessionEvent.
110
+ return null;
111
+ }
112
+ return "provider";
113
+ }
114
+ function parseMs(value) {
115
+ if (!value)
116
+ return NaN;
117
+ const ms = Date.parse(value);
118
+ return Number.isFinite(ms) ? ms : NaN;
119
+ }
120
+ function latestMeaningfulActivityMs(snapshot) {
121
+ const candidates = [
122
+ parseMs(snapshot.lastMeaningfulProgressAt),
123
+ parseMs(snapshot.lastProviderActivityAt),
124
+ parseMs(snapshot.lastToolActivityAt),
125
+ parseMs(snapshot.lastOutputActivityAt),
126
+ // Compatibility: lastActivityAt is treated as meaningful when richer clocks absent.
127
+ parseMs(snapshot.lastActivityAt),
128
+ parseMs(snapshot.startedAt),
129
+ ].filter((value) => !Number.isNaN(value));
130
+ if (candidates.length === 0)
131
+ return NaN;
132
+ return Math.max(...candidates);
133
+ }
134
+ /**
135
+ * Pure node liveness evaluator. No I/O.
136
+ *
137
+ * Transitions:
138
+ * - real provider/tool/output activity → active
139
+ * - no activity ≥ quietMs and runner lease fresh → quiet
140
+ * - no activity ≥ stallProbeMs → suspected-stall
141
+ * - probing flag → probing
142
+ * - identity mismatch / unprovable survival → needs-attention
143
+ * - wall clock ≥ absoluteMax → needs-attention (controlled abort path upstream)
144
+ */
145
+ export function evaluateNodeLiveness(input) {
146
+ const policy = resolveLivenessPolicy(input.policy);
147
+ const nowMs = typeof input.now === "number"
148
+ ? input.now
149
+ : (input.now ?? new Date()).getTime();
150
+ const startedMs = parseMs(input.node.startedAt);
151
+ const wallClockMs = !Number.isNaN(startedMs)
152
+ ? Math.max(0, nowMs - startedMs)
153
+ : 0;
154
+ const exceededAbsoluteMax = wallClockMs >= policy.absoluteMaxWallClockMs;
155
+ if (input.node.needsAttentionReason || exceededAbsoluteMax) {
156
+ return {
157
+ status: "needs-attention",
158
+ lastMeaningfulProgressAt: input.node.lastMeaningfulProgressAt ?? input.node.lastActivityAt,
159
+ idleMs: 0,
160
+ wallClockMs,
161
+ exceededAbsoluteMax,
162
+ };
163
+ }
164
+ if (input.node.probing) {
165
+ return {
166
+ status: "probing",
167
+ lastMeaningfulProgressAt: input.node.lastMeaningfulProgressAt ?? input.node.lastActivityAt,
168
+ idleMs: 0,
169
+ wallClockMs,
170
+ exceededAbsoluteMax,
171
+ };
172
+ }
173
+ const meaningfulMs = latestMeaningfulActivityMs(input.node);
174
+ const idleMs = !Number.isNaN(meaningfulMs)
175
+ ? Math.max(0, nowMs - meaningfulMs)
176
+ : wallClockMs;
177
+ const leaseFresh = input.runnerLeaseFresh ??
178
+ (() => {
179
+ const heartbeatMs = parseMs(input.runnerHeartbeatAt);
180
+ if (Number.isNaN(heartbeatMs))
181
+ return true;
182
+ return nowMs - heartbeatMs <= policy.runnerStaleMs;
183
+ })();
184
+ if (idleMs >= policy.stallProbeMs) {
185
+ return {
186
+ status: "suspected-stall",
187
+ lastMeaningfulProgressAt: input.node.lastMeaningfulProgressAt ?? input.node.lastActivityAt,
188
+ idleMs,
189
+ wallClockMs,
190
+ exceededAbsoluteMax,
191
+ };
192
+ }
193
+ if (idleMs >= policy.quietMs && leaseFresh) {
194
+ return {
195
+ status: "quiet",
196
+ lastMeaningfulProgressAt: input.node.lastMeaningfulProgressAt ?? input.node.lastActivityAt,
197
+ idleMs,
198
+ wallClockMs,
199
+ exceededAbsoluteMax,
200
+ };
201
+ }
202
+ // Active tool presence keeps us from escalating beyond quiet solely on idle output.
203
+ if ((input.node.activeToolCount ?? 0) > 0 && idleMs < policy.stallProbeMs) {
204
+ return {
205
+ status: idleMs >= policy.quietMs ? "quiet" : "active",
206
+ lastMeaningfulProgressAt: input.node.lastMeaningfulProgressAt ?? input.node.lastActivityAt,
207
+ idleMs,
208
+ wallClockMs,
209
+ exceededAbsoluteMax,
210
+ };
211
+ }
212
+ return {
213
+ status: "active",
214
+ lastMeaningfulProgressAt: input.node.lastMeaningfulProgressAt ?? input.node.lastActivityAt,
215
+ idleMs,
216
+ wallClockMs,
217
+ exceededAbsoluteMax,
218
+ };
219
+ }
220
+ /**
221
+ * Apply a fenced activity event onto a mutable node snapshot (pure field updates).
222
+ * Returns whether the write was accepted.
223
+ */
224
+ export function applyNodeActivity(input) {
225
+ if (!shouldAcceptActivity(input.currentAttempt, input.activityAttempt)) {
226
+ return false;
227
+ }
228
+ const at = input.at ?? new Date().toISOString();
229
+ if (input.kind === "lease" || input.kind === "synthetic-heartbeat") {
230
+ input.node.lastLeaseAt = at;
231
+ // Never treat synthetic lease as meaningful progress.
232
+ return true;
233
+ }
234
+ if (!isMeaningfulActivityKind(input.kind)) {
235
+ return false;
236
+ }
237
+ if (input.kind === "provider") {
238
+ input.node.lastProviderActivityAt = at;
239
+ }
240
+ else if (input.kind === "tool") {
241
+ input.node.lastToolActivityAt = at;
242
+ }
243
+ else if (input.kind === "output") {
244
+ input.node.lastOutputActivityAt = at;
245
+ }
246
+ input.node.lastMeaningfulProgressAt = at;
247
+ input.node.lastActivityAt = at;
248
+ input.node.livenessStatus = "active";
249
+ return true;
250
+ }
@@ -8,6 +8,7 @@ import { resolveContextPolicy } from "./context-policy.js";
8
8
  import { buildDagNodePromptEnvelope } from "./prompt.js";
9
9
  import { persistLongNodeOutputArtifacts } from "./upstream-artifacts.js";
10
10
  import { computeBackoffDelayMs, isRetryablePiFailureCategory, isSafeReadOnlyPiRetryCandidate, } from "./retry-policy.js";
11
+ import { applyNodeActivity, evaluateNodeLiveness, resolveLivenessPolicy, } from "./liveness-policy.js";
11
12
  import { buildProtocolRetryInstruction, validateOutputProtocol, } from "./output-protocol.js";
12
13
  import { writeDagNodeJsonArtifact } from "../../infrastructure/harness/artifact-store.js";
13
14
  import { buildProjectGovernanceContext, readCompletedWriterChangeManifests, writeProjectGovernanceContext, } from "./project-governance-context.js";
@@ -235,6 +236,9 @@ export async function executeDagNode(input) {
235
236
  node.status = "RUNNING";
236
237
  node.startedAt = new Date().toISOString();
237
238
  node.lastActivityAt = node.startedAt;
239
+ node.lastMeaningfulProgressAt = node.startedAt;
240
+ node.livenessStatus = "active";
241
+ node.currentAttempt = 1;
238
242
  if (task.shell?.verifyEvidence) {
239
243
  node.verifyEvidence = task.shell.verifyEvidence;
240
244
  }
@@ -310,12 +314,48 @@ export async function executeDagNode(input) {
310
314
  const maxAttempts = retryPolicy?.maxAttempts ?? 1;
311
315
  const attempts = [];
312
316
  let totalBackoffMs = 0;
317
+ const livenessPolicy = resolveLivenessPolicy(spec.defaults?.livenessPolicy, task.livenessPolicy);
318
+ /**
319
+ * Attempt-fenced, throttled activity sink. Late events from a previous
320
+ * attempt are no-ops. Persistence is best-effort and never throws to the
321
+ * executor path.
322
+ */
323
+ const ACTIVITY_PERSIST_MIN_INTERVAL_MS = 2_000;
324
+ let lastActivityPersistMs = 0;
325
+ const reportActivity = (activity) => {
326
+ const accepted = applyNodeActivity({
327
+ node,
328
+ currentAttempt: node.currentAttempt ?? 1,
329
+ activityAttempt: activity.attempt,
330
+ kind: activity.kind,
331
+ at: activity.at,
332
+ });
333
+ if (!accepted)
334
+ return;
335
+ const evaluation = evaluateNodeLiveness({
336
+ node,
337
+ policy: livenessPolicy,
338
+ runnerHeartbeatAt: state.runner?.heartbeatAt,
339
+ });
340
+ node.livenessStatus = evaluation.status;
341
+ const nowMs = Date.now();
342
+ if (nowMs - lastActivityPersistMs < ACTIVITY_PERSIST_MIN_INTERVAL_MS) {
343
+ return;
344
+ }
345
+ lastActivityPersistMs = nowMs;
346
+ // Canonical mid-call activity lives in active state.json. Queue through the
347
+ // runner's serialized state writer; per-node records remain terminal/attempt
348
+ // evidence so a late best-effort write cannot recreate an archived run dir.
349
+ void input.persistState().catch(() => { });
350
+ };
313
351
  let terminalResult;
314
352
  let previousFailureCategory;
315
353
  let previousProtocolReason;
316
354
  for (let attemptNumber = 1; attemptNumber <= maxAttempts; attemptNumber += 1) {
317
355
  const attemptStartedAt = new Date().toISOString();
318
356
  const attemptStarted = Date.now();
357
+ node.currentAttempt = attemptNumber;
358
+ node.livenessStatus = "active";
319
359
  let result;
320
360
  try {
321
361
  validateRepairArtifactGateBeforeShell({
@@ -328,6 +368,11 @@ export async function executeDagNode(input) {
328
368
  cwd,
329
369
  model,
330
370
  prompt: buildAttemptPrompt(task, prompt, attemptNumber, previousFailureCategory, previousProtocolReason),
371
+ attempt: attemptNumber,
372
+ reportActivity,
373
+ timeoutMs: livenessPolicy.absoluteMaxWallClockMs,
374
+ stallTimeoutMs: livenessPolicy.stallProbeMs,
375
+ abortGraceMs: livenessPolicy.abortGraceMs,
331
376
  });
332
377
  }
333
378
  catch (error) {
@@ -405,6 +450,10 @@ export async function executeDagNode(input) {
405
450
  ? result.parsedEvents
406
451
  : sumAttemptMetric(attempts, (attempt) => attempt.parsedEvents);
407
452
  node.lastActivityAt = attemptFinishedAt;
453
+ if (result.failureCategory === "termination-unconfirmed") {
454
+ node.needsAttentionReason = "attempt-termination-unconfirmed";
455
+ node.livenessStatus = "needs-attention";
456
+ }
408
457
  if (retryPolicy !== undefined) {
409
458
  state.nodes[nodeId].nodeRecordPath = path.join(runDir, `${nodeId}.json`);
410
459
  await writeNodeRecord(runDir, nodeId, state.nodes[nodeId]);
@@ -8,6 +8,7 @@ import { CANONICAL_TASK_ID_PATTERN, formatLocalCompactDate, } from "../../task/r
8
8
  import { assertFrozenBudget, initRunBudgetLedger, preflightBudgetOrBreach, recordFinishedNodeBudget, writeBudgetLedgerArtifacts, } from "./budget-enforcement.js";
9
9
  import { getDagRunDir, isTerminalDagRunStatus, locateDagRun, readHumanApprovalArtifact, requireActiveDagRun, } from "./lifecycle.js";
10
10
  import { moveToCompletedRunDir, moveToPausedRunDir, prepareActiveRunDir, writeRunSpec, writeRunState, } from "./run-store.js";
11
+ import { evaluateNodeLiveness, resolveLivenessPolicy, } from "./liveness-policy.js";
11
12
  import { createDagNodeExecutor } from "./executor-registry.js";
12
13
  import { executeDagPiNode } from "../../executors/dag-pi-executor.js";
13
14
  import { assertValidDagSpec } from "./validate.js";
@@ -406,12 +407,31 @@ async function executeDagCheckpoint(input) {
406
407
  stateWriteQueue = stateWriteQueue.then(() => writeRunState(runDir, state, options));
407
408
  await stateWriteQueue;
408
409
  };
410
+ const runnerLivenessPolicy = resolveLivenessPolicy(spec.defaults?.livenessPolicy);
409
411
  const heartbeatTimer = setInterval(() => {
410
412
  if (!state.runner)
411
413
  return;
414
+ // Runner lease only — never counts as meaningful Pi progress.
412
415
  state.runner.heartbeatAt = new Date().toISOString();
416
+ const tasksByIdForPolicy = new Map(spec.tasks.map((task) => [task.id, task]));
417
+ for (const node of Object.values(state.nodes)) {
418
+ if (node.status !== "RUNNING")
419
+ continue;
420
+ // Lease clock is separate from meaningful activity.
421
+ node.lastLeaseAt = state.runner.heartbeatAt;
422
+ const task = tasksByIdForPolicy.get(node.id);
423
+ const policy = resolveLivenessPolicy(spec.defaults?.livenessPolicy, task?.livenessPolicy);
424
+ const evaluation = evaluateNodeLiveness({
425
+ node,
426
+ policy,
427
+ runnerHeartbeatAt: state.runner.heartbeatAt,
428
+ });
429
+ node.livenessStatus = evaluation.status;
430
+ // Intentionally do NOT refresh lastActivityAt / lastMeaningfulProgressAt
431
+ // from the runner lease timer.
432
+ }
413
433
  void persistState().catch(() => { });
414
- }, 15_000);
434
+ }, runnerLivenessPolicy.heartbeatIntervalMs);
415
435
  heartbeatTimer.unref();
416
436
  try {
417
437
  const tasksById = new Map(spec.tasks.map((task) => [task.id, task]));
@@ -2,6 +2,7 @@ import { z } from "zod";
2
2
  import { campaignBudgetSchema, } from "../../application/evaluation/budget.js";
3
3
  import { assertDagPromptSourceRule } from "./prompt-source.js";
4
4
  import { dagRetryPolicySchema } from "./retry-policy.js";
5
+ import { dagLivenessPolicySchema, } from "./liveness-policy.js";
5
6
  import { dagOutputProtocolSchema } from "./output-protocol.js";
6
7
  export const dagComplexitySchema = z.enum(["HIGH", "MED", "LOW"]);
7
8
  export const dagNodeExecutorSchema = z.enum(["pi", "shell", "static"]);
@@ -194,6 +195,8 @@ export const dagDefaultsSchema = z
194
195
  contextPolicyId: contextPolicyIdSchema.optional(),
195
196
  skills: z.array(z.string()).optional(),
196
197
  writePolicy: dagWritePolicySchema.optional(),
198
+ /** Adaptive liveness thresholds for Pi node supervision. */
199
+ livenessPolicy: dagLivenessPolicySchema,
197
200
  })
198
201
  .optional();
199
202
  export const dagNodeStatusSchema = z.enum([
@@ -400,6 +403,8 @@ export const dagTaskSchema = z.object({
400
403
  forbiddenPaths: z.array(z.string()).optional().default([]),
401
404
  decisionGate: dagDecisionGateSchema.optional(),
402
405
  retryPolicy: dagRetryPolicySchema.optional(),
406
+ /** Optional per-node adaptive liveness override (merged over defaults). */
407
+ livenessPolicy: dagLivenessPolicySchema,
403
408
  dynamicExpansion: dagDynamicExpansionSchema.optional(),
404
409
  dynamicReduction: dagDynamicReductionSchema.optional(),
405
410
  dynamicCondition: dagDynamicConditionSchema.optional(),