@tea-agent/loop-agent 0.16.1-beta.2 → 0.16.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +4 -8
- package/CHANGELOG.md +76 -18
- package/README.md +76 -299
- package/dist/application/evaluation/alias.js +184 -0
- package/dist/application/evaluation/budget.js +192 -0
- package/dist/application/evaluation/campaign-hash.js +47 -0
- package/dist/application/evaluation/campaign-matrix.js +372 -0
- package/dist/application/evaluation/campaign-scorecard.js +135 -0
- package/dist/application/evaluation/campaign.js +370 -0
- package/dist/application/evaluation/candidate.js +23 -6
- package/dist/application/evaluation/corpus-hash.js +38 -0
- package/dist/application/evaluation/corpus.js +56 -0
- package/dist/application/evaluation/experiment.js +294 -0
- package/dist/application/evaluation/ignition.js +198 -0
- package/dist/application/evaluation/integrity-audit.js +162 -0
- package/dist/application/evaluation/outer-loop.js +132 -0
- package/dist/application/evaluation/pi-cell-executor.js +39 -0
- package/dist/application/evaluation/private-verifier.js +46 -0
- package/dist/application/evaluation/promotion-policy.js +151 -0
- package/dist/application/evaluation/proposer.js +98 -0
- package/dist/application/evaluation/types.js +522 -0
- package/dist/cli/command-definitions.js +19 -3
- package/dist/commands/dag-reconcile-run.js +3 -116
- package/dist/commands/eval.js +1176 -13
- package/dist/commands/init.js +7 -1
- package/dist/executors/dag-pi-executor.js +8 -46
- package/dist/executors/pi-sdk-executor.js +66 -3
- package/dist/executors/shell-executor.js +213 -30
- package/dist/executors/shell-presets.js +12 -2
- package/dist/executors/shell-write-guard.js +20 -1
- package/dist/infrastructure/evaluation/alias-store.js +199 -0
- package/dist/infrastructure/evaluation/campaign-store.js +154 -0
- package/dist/infrastructure/evaluation/corpus-store.js +181 -0
- package/dist/infrastructure/evaluation/experiment-store.js +124 -0
- package/dist/infrastructure/evaluation/ignition-store.js +82 -0
- package/dist/infrastructure/evaluation/private-verifier-store.js +145 -0
- package/dist/infrastructure/evaluation/proposer-store.js +78 -0
- package/dist/records/promotion.js +3 -1
- package/dist/shared/git-progress.js +9 -2
- package/dist/worker/cli.js +83 -0
- package/dist/worker/delivery/git-transaction.js +75 -0
- package/dist/worker/delivery/verification-bundle.js +13 -2
- package/dist/worker/feature/review.js +3 -2
- package/dist/worker/observability/read-model.js +56 -0
- package/dist/worker/observe/server.js +6 -3
- package/dist/worker/observe/static/dag-helpers.js +0 -62
- package/dist/worker/observe/static/styles.css +18 -55
- package/dist/worker/observe/static/views/dag.js +13 -5
- package/dist/worker/outcomes/adapters.js +4 -1
- package/dist/worker/outcomes/declared-artifacts.js +103 -0
- package/dist/worker/outcomes/evidence-tokens.js +29 -0
- package/dist/worker/outcomes/gate.js +10 -11
- package/dist/worker/outcomes/projector.js +30 -4
- package/dist/worker/outcomes/types.js +3 -0
- package/dist/worker/pool/reconcile.js +285 -0
- package/dist/worker/run-task/run-task.js +81 -4
- package/dist/worker/runner/run-ready.js +25 -2
- package/dist/worker/task-graph/ready-planner.js +14 -8
- package/dist/worker/task-graph/task-graph-schema.js +5 -3
- package/dist/workflows/dag/backend-test-analysis-contract.js +87 -30
- package/dist/workflows/dag/backend-test-case-manifest.js +71 -8
- package/dist/workflows/dag/backend-test-execution-contract.js +63 -11
- package/dist/workflows/dag/backend-test-repair-contract.js +94 -0
- package/dist/workflows/dag/backend-test-result-contract.js +6 -4
- package/dist/workflows/dag/backend-test-semantic-review-contract.js +36 -0
- package/dist/workflows/dag/budget-enforcement.js +67 -0
- package/dist/workflows/dag/context-policy.js +137 -0
- package/dist/workflows/dag/dynamic-runtime/condition.js +1 -1
- package/dist/workflows/dag/dynamic-runtime/shared.js +42 -0
- package/dist/workflows/dag/failure-routing.js +8 -1
- package/dist/workflows/dag/frontend-implementation-contract.js +32 -93
- package/dist/workflows/dag/init-hybrid.js +624 -172
- package/dist/workflows/dag/knowledge-curator.js +3 -0
- package/dist/workflows/dag/lifecycle.js +33 -2
- package/dist/workflows/dag/node-execution.js +11 -4
- package/dist/workflows/dag/prompt.js +1 -1
- package/dist/workflows/dag/reconcile-run.js +121 -0
- package/dist/workflows/dag/report.js +12 -0
- package/dist/workflows/dag/runner.js +43 -16
- package/dist/workflows/dag/scheduler.js +87 -17
- package/dist/workflows/dag/skill-snapshot.js +11 -7
- package/dist/workflows/dag/types.js +49 -1
- package/dist/workflows/dag/validate.js +35 -15
- package/docs/README.md +3 -1
- package/docs/architecture/runtime-boundaries.md +3 -2
- package/docs/init-surface.manifest.json +4 -0
- package/docs/local-development-environment.md +52 -0
- package/docs/templates/agent-dag.schema.json +25 -7
- package/docs/templates/agent-dag.supervised-implementation.json +23 -4
- package/docs/templates/backend-test-analysis.schema.json +9 -16
- package/docs/templates/backend-test-dag.json +493 -197
- package/docs/templates/backend-test-dag.review-cases.prompt.md +10 -4
- package/docs/templates/backend-test-execution.schema.json +6 -1
- package/docs/templates/branch-merge-report.md +14 -0
- package/docs/templates/evaluation/campaign-budget-v1.json +12 -0
- package/docs/templates/evaluation/campaign-dogfood-v0.json +24 -0
- package/docs/templates/evaluation/campaign-evidence-v1.json +44 -0
- package/docs/templates/evaluation/context-policy-baseline-v1.json +17 -0
- package/docs/templates/evaluation/context-policy-role-specialized-v1.json +28 -0
- package/docs/templates/evaluation/corpus-dogfood-v0.manifest.json +118 -0
- package/docs/templates/evaluation/matrix-dag-dry-run-v1.json +21 -0
- package/docs/templates/evaluation/matrix-fixture-v1.json +10 -0
- package/docs/templates/evaluation/private-verifier-dogfood-v0.json +16 -0
- package/docs/templates/product-line/AGENTS.md +1 -0
- package/docs/templates/product-line/README.md +17 -0
- package/docs/templates/product-line/acceptance.yaml +9 -0
- package/docs/templates/product-line/feature.yaml +11 -0
- package/docs/templates/product-line/task-graph.yaml +8 -0
- package/docs/templates/product-line/task.yaml +4 -0
- package/package.json +2 -1
- package/skills/frontend-implementation/references/node-contracts.md +3 -3
- package/skills/loop-agent/references/command-reference.md +5 -0
- package/skills/loop-agent/references/hybrid-dag.md +7 -4
package/dist/commands/init.js
CHANGED
|
@@ -859,6 +859,12 @@ export function buildManagedGitignoreBlock() {
|
|
|
859
859
|
".harness/*.log",
|
|
860
860
|
".harness/init-surface.json",
|
|
861
861
|
"",
|
|
862
|
+
"# Eval Lab runtime (candidates, campaigns, scorecards, dogfood scratch)",
|
|
863
|
+
".harness/evaluation/",
|
|
864
|
+
"",
|
|
865
|
+
"# fullstack / worker dogfood evidence (local campaign scratch)",
|
|
866
|
+
".harness/dogfood-evidence/",
|
|
867
|
+
"",
|
|
862
868
|
"# worker task pool",
|
|
863
869
|
".harness/task-pool/*",
|
|
864
870
|
".task-pool/",
|
|
@@ -2089,7 +2095,7 @@ export function buildInitInstructions(input) {
|
|
|
2089
2095
|
"- Enrich the root `README.md`: keep the deterministic project title and the loop-agent managed block intact, and fill the human-authored sections (项目概览, 技术栈与目录结构, 开发与验证) from the target project's actual files. The root README must serve both as a human-first project entry and as an agent work entry; replace the initialization-model supplement comments when the project files provide the information.",
|
|
2090
2096
|
`- Populate \`${governanceRoot}/verification-matrix.md\` with the target project's actual quick, standard, and full verification commands derived from its real language and toolchain, keeping the governance rows intact.`,
|
|
2091
2097
|
"- Project repo-local skills live in `.agents/skills/`. Do not create a root `skills/` directory in the target project; the package's bundled `skills/` remains the built-in fallback.",
|
|
2092
|
-
"- Merge a loop-agent managed block into `.gitignore` that ignores harness runtime facts (tasks, dag-runs, runs, live, cache, init-surface.json, .harness/task-pool, legacy .task-pool residue) while keeping prompts and directory placeholders shareable.",
|
|
2098
|
+
"- Merge a loop-agent managed block into `.gitignore` that ignores harness runtime facts (tasks, dag-runs, runs, evaluation, dogfood-evidence, live, cache, init-surface.json, .harness/task-pool, legacy .task-pool residue) while keeping prompts and directory placeholders shareable.",
|
|
2093
2099
|
"- Do not copy examples by default; examples stay bundled in the tool and are available through `loop-agent examples`.",
|
|
2094
2100
|
"- Add or update a loop-agent managed block in AGENTS.md.",
|
|
2095
2101
|
"- The generated AGENTS.md must include documentation convergence and structured DAG write-boundary rules so target projects keep the same working discipline as this repository.",
|
|
@@ -272,7 +272,7 @@ export async function executeDagPiNode(input, meta, piStepFn = executePiStep) {
|
|
|
272
272
|
persona,
|
|
273
273
|
step,
|
|
274
274
|
});
|
|
275
|
-
const mapped = mapPiResultToDagNodeResult(result
|
|
275
|
+
const mapped = mapPiResultToDagNodeResult(result);
|
|
276
276
|
if (!isWriteTask) {
|
|
277
277
|
return mapped;
|
|
278
278
|
}
|
|
@@ -292,9 +292,11 @@ export async function executeDagPiNode(input, meta, piStepFn = executePiStep) {
|
|
|
292
292
|
writeGuardOk = guard.ok;
|
|
293
293
|
writeGuardViolations = guard.violations;
|
|
294
294
|
}
|
|
295
|
-
catch {
|
|
295
|
+
catch (error) {
|
|
296
296
|
writeGuardOk = false;
|
|
297
|
-
writeGuardViolations = [
|
|
297
|
+
writeGuardViolations = [
|
|
298
|
+
`git status unavailable: ${error instanceof Error ? error.message : String(error)}`,
|
|
299
|
+
];
|
|
298
300
|
}
|
|
299
301
|
}
|
|
300
302
|
if (writeGuardOk) {
|
|
@@ -313,57 +315,17 @@ export async function executeDagPiNode(input, meta, piStepFn = executePiStep) {
|
|
|
313
315
|
durationMs: mapped.durationMs || Date.now() - started,
|
|
314
316
|
};
|
|
315
317
|
}
|
|
316
|
-
export function mapPiResultToDagNodeResult(result
|
|
317
|
-
const assistantText = canonicalizeProtocolFirstLine(result.assistantText, firstProtocolLine);
|
|
318
|
+
export function mapPiResultToDagNodeResult(result) {
|
|
318
319
|
return {
|
|
319
320
|
ok: result.ok,
|
|
320
|
-
stdout: assistantText || result.stdout,
|
|
321
|
+
stdout: result.assistantText || result.stdout,
|
|
321
322
|
stderr: result.stderr,
|
|
322
323
|
failureCategory: result.failureCategory,
|
|
323
324
|
durationMs: result.durationMs,
|
|
324
|
-
assistantText,
|
|
325
|
+
assistantText: result.assistantText,
|
|
325
326
|
backend: result.backend,
|
|
326
327
|
sdkAttempted: result.sdkAttempted,
|
|
327
328
|
tokensUsed: result.tokensUsed,
|
|
328
329
|
parsedEvents: result.parsedEvents,
|
|
329
330
|
};
|
|
330
331
|
}
|
|
331
|
-
function canonicalizeProtocolFirstLine(assistantText, firstProtocolLine) {
|
|
332
|
-
if (!assistantText || !firstProtocolLine)
|
|
333
|
-
return assistantText;
|
|
334
|
-
const lines = assistantText.split(/\r?\n/);
|
|
335
|
-
let protocolIndex = -1;
|
|
336
|
-
let protocolLine = "";
|
|
337
|
-
for (const [index, line] of lines.entries()) {
|
|
338
|
-
const normalized = normalizeProtocolLine(line);
|
|
339
|
-
if (normalized.startsWith(firstProtocolLine)) {
|
|
340
|
-
protocolIndex = index;
|
|
341
|
-
protocolLine = normalized;
|
|
342
|
-
break;
|
|
343
|
-
}
|
|
344
|
-
}
|
|
345
|
-
if (protocolIndex < 0)
|
|
346
|
-
return assistantText;
|
|
347
|
-
if (protocolIndex === 0) {
|
|
348
|
-
return [protocolLine, ...lines.slice(1)].join("\n");
|
|
349
|
-
}
|
|
350
|
-
const before = lines.slice(0, protocolIndex);
|
|
351
|
-
const after = lines.slice(protocolIndex + 1);
|
|
352
|
-
while (before.at(-1)?.trim() === "" &&
|
|
353
|
-
after.at(0)?.trim() === "") {
|
|
354
|
-
after.shift();
|
|
355
|
-
}
|
|
356
|
-
const bodyLines = [...before, ...after];
|
|
357
|
-
while (bodyLines.at(0)?.trim() === "")
|
|
358
|
-
bodyLines.shift();
|
|
359
|
-
while (bodyLines.at(-1)?.trim() === "")
|
|
360
|
-
bodyLines.pop();
|
|
361
|
-
return bodyLines.length > 0
|
|
362
|
-
? `${protocolLine}\n\n${bodyLines.join("\n")}`
|
|
363
|
-
: protocolLine;
|
|
364
|
-
}
|
|
365
|
-
function normalizeProtocolLine(line) {
|
|
366
|
-
const trimmed = line.trim();
|
|
367
|
-
const emphasized = trimmed.match(/^(\*{1,3})\s*(.*?)\s*\1$/);
|
|
368
|
-
return (emphasized?.[2] ?? trimmed).trim();
|
|
369
|
-
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { appendFile, mkdir } from 'node:fs/promises';
|
|
2
2
|
import path from 'node:path';
|
|
3
|
-
import { classifyPiFailure, DEFAULT_TIMEOUT_MS, extractAssistantTextFromPiJson,
|
|
3
|
+
import { classifyPiFailure, DEFAULT_TIMEOUT_MS, extractAssistantTextFromPiJson, } from './pi-executor.js';
|
|
4
4
|
import { serializeSessionEvent } from './pi-event-serializer.js';
|
|
5
5
|
let sdkSessionFactoryOverride;
|
|
6
6
|
let sdkImportOverrideForTests;
|
|
@@ -202,6 +202,63 @@ async function resolveSdkSessionFactory(reuseScope) {
|
|
|
202
202
|
return createSdkSession(sdk, input, shared);
|
|
203
203
|
};
|
|
204
204
|
}
|
|
205
|
+
function isRecord(value) {
|
|
206
|
+
return typeof value === 'object' && value !== null;
|
|
207
|
+
}
|
|
208
|
+
function readUsageNumber(record, keys) {
|
|
209
|
+
for (const key of keys) {
|
|
210
|
+
const value = record[key];
|
|
211
|
+
if (typeof value === 'number' && Number.isFinite(value) && value >= 0) {
|
|
212
|
+
return Math.trunc(value);
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
return undefined;
|
|
216
|
+
}
|
|
217
|
+
function extractSdkUsageSample(event) {
|
|
218
|
+
const message = isRecord(event.message) ? event.message : undefined;
|
|
219
|
+
const usageCandidates = [event.usage, message?.usage, event.tokenUsage];
|
|
220
|
+
let tokens;
|
|
221
|
+
for (const candidate of usageCandidates) {
|
|
222
|
+
if (!isRecord(candidate))
|
|
223
|
+
continue;
|
|
224
|
+
const input = readUsageNumber(candidate, ['input_tokens', 'inputTokens', 'prompt_tokens', 'promptTokens']);
|
|
225
|
+
const output = readUsageNumber(candidate, ['output_tokens', 'outputTokens', 'completion_tokens', 'completionTokens']);
|
|
226
|
+
const total = readUsageNumber(candidate, ['total_tokens', 'totalTokens']);
|
|
227
|
+
if (input !== undefined && output !== undefined) {
|
|
228
|
+
tokens = input + output;
|
|
229
|
+
break;
|
|
230
|
+
}
|
|
231
|
+
if (total !== undefined) {
|
|
232
|
+
tokens = total;
|
|
233
|
+
break;
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
if (tokens === undefined)
|
|
237
|
+
return undefined;
|
|
238
|
+
const responseKeyCandidates = [
|
|
239
|
+
event.responseId,
|
|
240
|
+
event.messageId,
|
|
241
|
+
message?.responseId,
|
|
242
|
+
message?.id,
|
|
243
|
+
];
|
|
244
|
+
const responseKey = responseKeyCandidates.find((value) => typeof value === 'string' && value.length > 0);
|
|
245
|
+
return responseKey ? { responseKey, tokens } : { tokens };
|
|
246
|
+
}
|
|
247
|
+
function aggregateSdkTokenUsage(samples) {
|
|
248
|
+
const identified = new Map();
|
|
249
|
+
let anonymousMaximum = 0;
|
|
250
|
+
for (const sample of samples) {
|
|
251
|
+
if (sample.responseKey) {
|
|
252
|
+
identified.set(sample.responseKey, Math.max(identified.get(sample.responseKey) ?? 0, sample.tokens));
|
|
253
|
+
}
|
|
254
|
+
else {
|
|
255
|
+
// SDK lifecycle events may repeat cumulative usage without a response id.
|
|
256
|
+
// Treat anonymous samples as snapshots for this attempt rather than increments.
|
|
257
|
+
anonymousMaximum = Math.max(anonymousMaximum, sample.tokens);
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
return anonymousMaximum + Array.from(identified.values()).reduce((sum, tokens) => sum + tokens, 0);
|
|
261
|
+
}
|
|
205
262
|
/**
|
|
206
263
|
* Execute a single Pi step via the SDK.
|
|
207
264
|
* When reuseScope is active, only shared auth/model resources are reused; each attempt still
|
|
@@ -227,6 +284,7 @@ export async function executeSingleSdkAttempt(options) {
|
|
|
227
284
|
let timedOut = false;
|
|
228
285
|
let stderr = '';
|
|
229
286
|
const stdoutLines = [];
|
|
287
|
+
const usageSamples = [];
|
|
230
288
|
let session;
|
|
231
289
|
let timeoutHandle;
|
|
232
290
|
const sessionEventAppender = options.sessionEventsPath
|
|
@@ -243,11 +301,16 @@ export async function executeSingleSdkAttempt(options) {
|
|
|
243
301
|
thinking: modelConfig.thinking,
|
|
244
302
|
});
|
|
245
303
|
const unsubscribe = session.subscribe((event) => {
|
|
304
|
+
const usageSample = extractSdkUsageSample(event);
|
|
305
|
+
if (usageSample)
|
|
306
|
+
usageSamples.push(usageSample);
|
|
246
307
|
if (!shouldPersistSessionEvent(event))
|
|
247
308
|
return;
|
|
248
309
|
const line = serializeSessionEvent(event);
|
|
249
310
|
stdoutLines.push(line);
|
|
250
|
-
sessionEventAppender
|
|
311
|
+
if (sessionEventAppender) {
|
|
312
|
+
sessionEventAppender.append(line, event);
|
|
313
|
+
}
|
|
251
314
|
});
|
|
252
315
|
const filePrefix = options.attachedFiles.map((file) => `@${file}`).join(' ');
|
|
253
316
|
const promptMessage = filePrefix
|
|
@@ -305,7 +368,7 @@ export async function executeSingleSdkAttempt(options) {
|
|
|
305
368
|
const stdout = stdoutLines.join('\n');
|
|
306
369
|
const durationMs = Date.now() - startedAt;
|
|
307
370
|
const { assistantText, parsedEvents } = extractAssistantTextFromPiJson(stdout);
|
|
308
|
-
const tokensUsed =
|
|
371
|
+
const tokensUsed = aggregateSdkTokenUsage(usageSamples);
|
|
309
372
|
const failureCategory = classifyPiFailure({
|
|
310
373
|
assistantText,
|
|
311
374
|
exitCode: timedOut ? 1 : stderr ? 1 : 0,
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { spawn } from "node:child_process";
|
|
2
2
|
import { appendFileSync, existsSync, mkdirSync, writeFileSync } from "node:fs";
|
|
3
|
+
import { readFile, writeFile } from "node:fs/promises";
|
|
3
4
|
import path from "node:path";
|
|
4
5
|
import { writeDagNodeTextArtifact } from "../infrastructure/harness/artifact-store.js";
|
|
5
6
|
import { truncateOutput } from "../shared/output-truncation.js";
|
|
@@ -11,7 +12,9 @@ import { formatFrontendFailureAssessStdout, formatFrontendRepairContractStdout,
|
|
|
11
12
|
import { formatTraceabilityGateStdout, materializeBackendTestCaseManifest, runBackendTestTraceabilityGate, } from "../workflows/dag/backend-test-case-manifest.js";
|
|
12
13
|
import { materializeBackendTestExecutionContract } from "../workflows/dag/backend-test-execution-contract.js";
|
|
13
14
|
import { materializeBackendTestResultFromRunDir } from "../workflows/dag/backend-test-result-contract.js";
|
|
14
|
-
import {
|
|
15
|
+
import { buildBackendTestEffectiveResultSelectorShellSnippet, buildBackendTestRepairEligibilityShellSnippet, buildBackendTestRepairSafetyShellSnippet, materializeBackendTestClassification, } from "../workflows/dag/backend-test-repair-contract.js";
|
|
16
|
+
import { materializeBackendTestSemanticReview } from "../workflows/dag/backend-test-semantic-review-contract.js";
|
|
17
|
+
import { pathsChangedDuringRun, readGitStatusPorcelain, snapshotGitStatusPorcelain, validateShellWriteGuard, } from "./shell-write-guard.js";
|
|
15
18
|
import { buildShellProcessEnv } from "./shell-verification.js";
|
|
16
19
|
const DEFAULT_SHELL_TIMEOUT_MS = 300_000;
|
|
17
20
|
const SUMMARY_STDOUT_MAX = 4_000;
|
|
@@ -63,32 +66,6 @@ export function resolveShellCommands(shell) {
|
|
|
63
66
|
...fromRequirementCoverageGate,
|
|
64
67
|
];
|
|
65
68
|
}
|
|
66
|
-
async function readGitStatusPorcelain(cwd) {
|
|
67
|
-
return new Promise((resolve, reject) => {
|
|
68
|
-
const child = spawn("git", ["status", "--porcelain=v1", "--untracked-files=all"], {
|
|
69
|
-
cwd,
|
|
70
|
-
stdio: ["ignore", "pipe", "pipe"],
|
|
71
|
-
});
|
|
72
|
-
let stdout = "";
|
|
73
|
-
let stderr = "";
|
|
74
|
-
child.stdout.setEncoding("utf-8");
|
|
75
|
-
child.stderr.setEncoding("utf-8");
|
|
76
|
-
child.stdout.on("data", (chunk) => {
|
|
77
|
-
stdout += chunk;
|
|
78
|
-
});
|
|
79
|
-
child.stderr.on("data", (chunk) => {
|
|
80
|
-
stderr += chunk;
|
|
81
|
-
});
|
|
82
|
-
child.on("error", reject);
|
|
83
|
-
child.on("close", (code) => {
|
|
84
|
-
if (code === 0) {
|
|
85
|
-
resolve(stdout);
|
|
86
|
-
return;
|
|
87
|
-
}
|
|
88
|
-
reject(new Error(`git status failed: ${stderr.trim() || stdout.trim()}`));
|
|
89
|
-
});
|
|
90
|
-
});
|
|
91
|
-
}
|
|
92
69
|
export async function executeShellCommand(input) {
|
|
93
70
|
return new Promise((resolve) => {
|
|
94
71
|
const startedAt = Date.now();
|
|
@@ -98,7 +75,7 @@ export async function executeShellCommand(input) {
|
|
|
98
75
|
HARNESS_DAG_RUN_ID: input.dagRunMeta.runId,
|
|
99
76
|
}
|
|
100
77
|
: undefined;
|
|
101
|
-
const child = spawn(resolveBashExecutable(), ["-
|
|
78
|
+
const child = spawn(resolveBashExecutable(), ["-c", input.command], {
|
|
102
79
|
cwd: input.cwd,
|
|
103
80
|
env: buildShellProcessEnv(input.envAllowlist, injectedEnv),
|
|
104
81
|
stdio: ["ignore", "pipe", "pipe"],
|
|
@@ -297,8 +274,194 @@ async function runShellWriteGuard(input) {
|
|
|
297
274
|
forbiddenPaths: input.task.forbiddenPaths,
|
|
298
275
|
});
|
|
299
276
|
}
|
|
277
|
+
async function executeBackendTestPipeline(input, meta) {
|
|
278
|
+
const pipeline = input.task.shell?.backendTestPipeline;
|
|
279
|
+
const started = Date.now();
|
|
280
|
+
try {
|
|
281
|
+
if (!pipeline)
|
|
282
|
+
throw new Error("missing backend-test pipeline id");
|
|
283
|
+
const outputs = [];
|
|
284
|
+
if (pipeline === "contracts") {
|
|
285
|
+
const wrapperPath = path.join(meta.runDir, "analyze-and-discover-backend-test-pi.json");
|
|
286
|
+
const wrapper = JSON.parse(await readFile(wrapperPath, "utf8"));
|
|
287
|
+
const raw = wrapper.assistantText?.trim() || wrapper.stdout?.trim() || "";
|
|
288
|
+
const fenced = raw.match(/^```json\s*([\s\S]*?)\s*```$/i);
|
|
289
|
+
const envelope = JSON.parse(fenced ? fenced[1] : raw);
|
|
290
|
+
if (!envelope.analysis || !envelope.execution)
|
|
291
|
+
throw new Error("backend-test contract envelope requires analysis and execution");
|
|
292
|
+
await writeFile(path.join(meta.runDir, "backend-test-analysis-envelope.json"), JSON.stringify({ assistantText: JSON.stringify(envelope.analysis) }));
|
|
293
|
+
await writeFile(path.join(meta.runDir, "backend-test-execution-envelope.json"), JSON.stringify({ assistantText: JSON.stringify(envelope.execution) }));
|
|
294
|
+
const analysis = await materializeBackendTestAnalysisContract({
|
|
295
|
+
runDir: meta.runDir,
|
|
296
|
+
fromNodeId: "backend-test-analysis-envelope",
|
|
297
|
+
artifactName: "backend-test-analysis.json",
|
|
298
|
+
outputDir: "contracts",
|
|
299
|
+
sourceBinding: meta.spec.sourceBinding,
|
|
300
|
+
});
|
|
301
|
+
const execution = await materializeBackendTestExecutionContract({
|
|
302
|
+
runDir: meta.runDir,
|
|
303
|
+
fromNodeId: "backend-test-execution-envelope",
|
|
304
|
+
artifactName: "backend-test-execution.json",
|
|
305
|
+
outputDir: "contracts",
|
|
306
|
+
});
|
|
307
|
+
outputs.push(`analysis=${analysis.path}`, `execution=${execution.path}`);
|
|
308
|
+
}
|
|
309
|
+
else if (pipeline === "semantic-initial" || pipeline === "semantic-final") {
|
|
310
|
+
const fromNodeId = pipeline === "semantic-final"
|
|
311
|
+
? "review-generated-backend-pytest-final-pi"
|
|
312
|
+
: "review-generated-backend-pytest-pi";
|
|
313
|
+
const artifactName = pipeline === "semantic-final"
|
|
314
|
+
? "backend-test-semantic-review-final.json"
|
|
315
|
+
: "backend-test-semantic-review.json";
|
|
316
|
+
const review = await materializeBackendTestSemanticReview({
|
|
317
|
+
runDir: meta.runDir,
|
|
318
|
+
fromNodeId,
|
|
319
|
+
artifactName,
|
|
320
|
+
outputDir: "contracts",
|
|
321
|
+
});
|
|
322
|
+
const trace = await runBackendTestTraceabilityGate({
|
|
323
|
+
runDir: meta.runDir,
|
|
324
|
+
workspaceRoot: input.cwd,
|
|
325
|
+
});
|
|
326
|
+
outputs.push(`semantic=${review.path}`, formatTraceabilityGateStdout(trace));
|
|
327
|
+
const parsed = JSON.parse(await readFile(review.path, "utf8"));
|
|
328
|
+
if (pipeline === "semantic-initial") {
|
|
329
|
+
return { ok: true, stdout: JSON.stringify(parsed), stderr: "", failureCategory: "success", durationMs: Date.now() - started };
|
|
330
|
+
}
|
|
331
|
+
if (pipeline === "semantic-final") {
|
|
332
|
+
if (parsed.verdict !== "pass")
|
|
333
|
+
throw new Error("backend pytest semantic review did not pass");
|
|
334
|
+
}
|
|
335
|
+
}
|
|
336
|
+
else if (pipeline === "execute-parse-initial") {
|
|
337
|
+
const results = await executePipelineCommands(input, meta);
|
|
338
|
+
if (!results.every((result) => result.ok)) {
|
|
339
|
+
const failure = results.find((result) => !result.ok);
|
|
340
|
+
return { ok: false, stdout: results.map((result) => result.stdout).join("\n"), stderr: failure.stderr, failureCategory: failure.failureCategory, durationMs: Date.now() - started };
|
|
341
|
+
}
|
|
342
|
+
const artifact = await materializeBackendTestResultFromRunDir({
|
|
343
|
+
runDir: meta.runDir,
|
|
344
|
+
fromNodeId: input.task.id,
|
|
345
|
+
artifactName: "backend-test-result-initial.json",
|
|
346
|
+
outputDir: "contracts",
|
|
347
|
+
junitRelativePath: "reports/backend-test-initial-junit.xml",
|
|
348
|
+
});
|
|
349
|
+
outputs.push(...results.map((result) => result.stdout), `result=${artifact.path}`);
|
|
350
|
+
}
|
|
351
|
+
else if (pipeline === "classification-eligibility") {
|
|
352
|
+
const classification = await materializeBackendTestClassification({
|
|
353
|
+
runDir: meta.runDir,
|
|
354
|
+
fromNodeId: "classify-backend-test-result-pi",
|
|
355
|
+
artifactName: "backend-test-classification.json",
|
|
356
|
+
outputDir: "contracts",
|
|
357
|
+
});
|
|
358
|
+
const results = await executePipelineCommands(input, meta, [buildBackendTestRepairEligibilityShellSnippet()]);
|
|
359
|
+
if (!results.every((result) => result.ok))
|
|
360
|
+
throw new Error(results.find((result) => !result.ok)?.stderr || "repair eligibility failed");
|
|
361
|
+
const eligibility = JSON.parse(await readFile(path.join(meta.runDir, "contracts", "backend-test-repair-eligibility.json"), "utf8"));
|
|
362
|
+
return { ok: true, stdout: JSON.stringify(eligibility), stderr: "", failureCategory: "success", durationMs: Date.now() - started };
|
|
363
|
+
}
|
|
364
|
+
else if (pipeline === "repair-safety-traceability") {
|
|
365
|
+
const results = await executePipelineCommands(input, meta, [buildBackendTestRepairSafetyShellSnippet()]);
|
|
366
|
+
if (!results.every((result) => result.ok))
|
|
367
|
+
throw new Error(results.find((result) => !result.ok)?.stderr || "repair safety failed");
|
|
368
|
+
const trace = await runBackendTestTraceabilityGate({ runDir: meta.runDir, workspaceRoot: input.cwd });
|
|
369
|
+
outputs.push(...results.map((result) => result.stdout), formatTraceabilityGateStdout(trace));
|
|
370
|
+
}
|
|
371
|
+
else if (pipeline === "finalize-effective-result") {
|
|
372
|
+
const eligibilityPath = path.join(meta.runDir, "contracts", "backend-test-repair-eligibility.json");
|
|
373
|
+
const eligibility = existsSync(eligibilityPath)
|
|
374
|
+
? JSON.parse(await readFile(eligibilityPath, "utf8"))
|
|
375
|
+
: { eligible: false };
|
|
376
|
+
const finalNeeded = eligibility.eligible === true;
|
|
377
|
+
if (finalNeeded) {
|
|
378
|
+
const results = await executePipelineCommands(input, meta);
|
|
379
|
+
if (!results.every((result) => result.ok))
|
|
380
|
+
return { ok: false, stdout: results.map((result) => result.stdout).join("\n"), stderr: results.find((result) => !result.ok)?.stderr ?? "final pytest failed", failureCategory: results.find((result) => !result.ok)?.failureCategory ?? "nonzero-exit", durationMs: Date.now() - started };
|
|
381
|
+
const artifact = await materializeBackendTestResultFromRunDir({ runDir: meta.runDir, fromNodeId: input.task.id, artifactName: "backend-test-result-final.json", outputDir: "contracts", junitRelativePath: "reports/backend-test-final-junit.xml" });
|
|
382
|
+
outputs.push(...results.map((result) => result.stdout), `final=${artifact.path}`);
|
|
383
|
+
}
|
|
384
|
+
const selected = await executePipelineCommands(input, meta, [buildBackendTestEffectiveResultSelectorShellSnippet()]);
|
|
385
|
+
if (!selected.every((result) => result.ok))
|
|
386
|
+
throw new Error(selected.find((result) => !result.ok)?.stderr || "effective result selection failed");
|
|
387
|
+
outputs.push(...selected.map((result) => result.stdout));
|
|
388
|
+
}
|
|
389
|
+
else {
|
|
390
|
+
throw new Error(`unsupported backend-test pipeline: ${pipeline}`);
|
|
391
|
+
}
|
|
392
|
+
return { ok: true, stdout: outputs.filter(Boolean).join("\n"), stderr: "", failureCategory: "success", durationMs: Date.now() - started };
|
|
393
|
+
}
|
|
394
|
+
catch (error) {
|
|
395
|
+
return { ok: false, stdout: "", stderr: error instanceof Error ? error.message : String(error), failureCategory: "invalid-output", durationMs: Date.now() - started };
|
|
396
|
+
}
|
|
397
|
+
}
|
|
398
|
+
async function executeBackendTestPipelineWithWriteGuard(input, meta) {
|
|
399
|
+
let beforeStatus;
|
|
400
|
+
try {
|
|
401
|
+
beforeStatus = await readGitStatusPorcelain(input.cwd);
|
|
402
|
+
}
|
|
403
|
+
catch {
|
|
404
|
+
// Non-git cwd: keep the same documented limitation as normal shell nodes.
|
|
405
|
+
beforeStatus = undefined;
|
|
406
|
+
}
|
|
407
|
+
const result = await executeBackendTestPipeline(input, meta);
|
|
408
|
+
if (beforeStatus === undefined)
|
|
409
|
+
return result;
|
|
410
|
+
try {
|
|
411
|
+
const guard = await runShellWriteGuard({
|
|
412
|
+
rootCwd: input.cwd,
|
|
413
|
+
task: input.task,
|
|
414
|
+
beforeStatus,
|
|
415
|
+
});
|
|
416
|
+
if (guard.ok)
|
|
417
|
+
return result;
|
|
418
|
+
const detail = `write guard failed: ${guard.violations.join(", ")}`;
|
|
419
|
+
return {
|
|
420
|
+
...result,
|
|
421
|
+
ok: false,
|
|
422
|
+
stderr: [result.stderr, detail].filter(Boolean).join("\n\n"),
|
|
423
|
+
failureCategory: "write-guard",
|
|
424
|
+
};
|
|
425
|
+
}
|
|
426
|
+
catch (error) {
|
|
427
|
+
const detail = `write guard failed: git status unavailable: ${error instanceof Error ? error.message : String(error)}`;
|
|
428
|
+
return {
|
|
429
|
+
...result,
|
|
430
|
+
ok: false,
|
|
431
|
+
stderr: [result.stderr, detail].filter(Boolean).join("\n\n"),
|
|
432
|
+
failureCategory: "write-guard",
|
|
433
|
+
};
|
|
434
|
+
}
|
|
435
|
+
}
|
|
436
|
+
async function executePipelineCommands(input, meta, overrideCommands) {
|
|
437
|
+
const shell = input.task.shell;
|
|
438
|
+
const cwd = resolveShellCwd(input.cwd, shell.cwd);
|
|
439
|
+
const commands = overrideCommands ?? shell.commands;
|
|
440
|
+
const results = [];
|
|
441
|
+
for (const command of commands) {
|
|
442
|
+
const commandNumber = results.length + 1;
|
|
443
|
+
const result = await executeShellCommand({
|
|
444
|
+
command,
|
|
445
|
+
cwd,
|
|
446
|
+
timeoutMs: shell.timeoutMs ?? DEFAULT_SHELL_TIMEOUT_MS,
|
|
447
|
+
envAllowlist: shell.envAllowlist,
|
|
448
|
+
dagRunMeta: { runDir: meta.runDir, runId: meta.runId },
|
|
449
|
+
outputArtifacts: {
|
|
450
|
+
stdoutPath: path.join(meta.runDir, input.task.id, "commands", `${commandNumber}.stdout.txt`),
|
|
451
|
+
stderrPath: path.join(meta.runDir, input.task.id, "commands", `${commandNumber}.stderr.txt`),
|
|
452
|
+
},
|
|
453
|
+
});
|
|
454
|
+
results.push(result);
|
|
455
|
+
if (!result.ok)
|
|
456
|
+
break;
|
|
457
|
+
}
|
|
458
|
+
return results;
|
|
459
|
+
}
|
|
300
460
|
export async function executeDagShellNode(input, meta) {
|
|
301
461
|
const shell = input.task.shell;
|
|
462
|
+
if (shell?.backendTestPipeline) {
|
|
463
|
+
return executeBackendTestPipelineWithWriteGuard(input, meta);
|
|
464
|
+
}
|
|
302
465
|
if (shell?.jsonArtifactGate) {
|
|
303
466
|
const started = Date.now();
|
|
304
467
|
try {
|
|
@@ -306,6 +469,7 @@ export async function executeDagShellNode(input, meta) {
|
|
|
306
469
|
let artifact;
|
|
307
470
|
switch (gate.schemaId) {
|
|
308
471
|
case "backend-test-analysis-v1":
|
|
472
|
+
case "backend-test-analysis-v2":
|
|
309
473
|
artifact = await materializeBackendTestAnalysisContract({
|
|
310
474
|
runDir: meta.runDir,
|
|
311
475
|
fromNodeId: gate.fromNodeId,
|
|
@@ -328,6 +492,23 @@ export async function executeDagShellNode(input, meta) {
|
|
|
328
492
|
fromNodeId: gate.fromNodeId,
|
|
329
493
|
artifactName: gate.artifactName,
|
|
330
494
|
outputDir: gate.outputDir,
|
|
495
|
+
junitRelativePath: gate.junitRelativePath,
|
|
496
|
+
});
|
|
497
|
+
break;
|
|
498
|
+
case "backend-test-classification-v1":
|
|
499
|
+
artifact = await materializeBackendTestClassification({
|
|
500
|
+
runDir: meta.runDir,
|
|
501
|
+
fromNodeId: gate.fromNodeId,
|
|
502
|
+
artifactName: gate.artifactName,
|
|
503
|
+
outputDir: gate.outputDir,
|
|
504
|
+
});
|
|
505
|
+
break;
|
|
506
|
+
case "backend-test-semantic-review-v1":
|
|
507
|
+
artifact = await materializeBackendTestSemanticReview({
|
|
508
|
+
runDir: meta.runDir,
|
|
509
|
+
fromNodeId: gate.fromNodeId,
|
|
510
|
+
artifactName: gate.artifactName,
|
|
511
|
+
outputDir: gate.outputDir,
|
|
331
512
|
});
|
|
332
513
|
break;
|
|
333
514
|
case "backend-test-case-manifest-v1":
|
|
@@ -520,9 +701,11 @@ export async function executeDagShellNode(input, meta) {
|
|
|
520
701
|
writeGuardOk = guard.ok;
|
|
521
702
|
writeGuardViolations = guard.violations;
|
|
522
703
|
}
|
|
523
|
-
catch {
|
|
704
|
+
catch (error) {
|
|
524
705
|
writeGuardOk = false;
|
|
525
|
-
writeGuardViolations = [
|
|
706
|
+
writeGuardViolations = [
|
|
707
|
+
`git status unavailable: ${error instanceof Error ? error.message : String(error)}`,
|
|
708
|
+
];
|
|
526
709
|
}
|
|
527
710
|
}
|
|
528
711
|
const commandsOk = results.every((result) => result.ok);
|
|
@@ -31,10 +31,20 @@ function verdictGateNodeParser(lineMode) {
|
|
|
31
31
|
*/
|
|
32
32
|
export function buildVerdictGateShellCommand(gate) {
|
|
33
33
|
const gateLabel = gate.label ?? `${gate.fromNodeId} verdict`;
|
|
34
|
-
const jsonFile = `${gate.fromNodeId}.json`;
|
|
35
34
|
const lineMode = gate.lineMode ?? "first-non-empty";
|
|
36
35
|
const parser = verdictGateNodeParser(lineMode);
|
|
37
|
-
const
|
|
36
|
+
const candidateNodeIds = [
|
|
37
|
+
gate.fromNodeId,
|
|
38
|
+
...(gate.fallbackFromNodeIds ?? []).filter((id) => id !== gate.fromNodeId),
|
|
39
|
+
];
|
|
40
|
+
const resolveFile = candidateNodeIds.length === 1
|
|
41
|
+
? `FILE="\${HARNESS_DAG_RUN_DIR}/${candidateNodeIds[0]}.json"; test -f "\${FILE}" || { echo "missing ${gateLabel} JSON output: \${FILE}" >&2; exit 1; }`
|
|
42
|
+
: [
|
|
43
|
+
"FILE=\"\"",
|
|
44
|
+
...candidateNodeIds.map((id) => `if test -z "\${FILE}" && test -f "\${HARNESS_DAG_RUN_DIR}/${id}.json"; then FILE="\${HARNESS_DAG_RUN_DIR}/${id}.json"; fi`),
|
|
45
|
+
`test -n "\${FILE}" || { echo "missing ${gateLabel} JSON output (tried: ${candidateNodeIds.join(", ")})" >&2; exit 1; }`,
|
|
46
|
+
].join("; ");
|
|
47
|
+
const preamble = `test -n "\${HARNESS_DAG_RUN_DIR:-}" || { echo "missing HARNESS_DAG_RUN_DIR for ${gateLabel} gate" >&2; exit 1; }; ${resolveFile}; FIRST=$(${parser})`;
|
|
38
48
|
const blockedMessage = `${gateLabel} gate blocked: \${FIRST:-missing VERDICT line}`;
|
|
39
49
|
if (gate.accept.length === 1) {
|
|
40
50
|
const expected = escapeShellSingleQuoted(gate.accept[0]);
|
|
@@ -67,7 +67,26 @@ export function validateShellWriteGuard(input) {
|
|
|
67
67
|
}
|
|
68
68
|
return { ok: violations.length === 0, violations };
|
|
69
69
|
}
|
|
70
|
-
export async function readGitStatusPorcelain(cwd) {
|
|
70
|
+
export async function readGitStatusPorcelain(cwd, options = {}) {
|
|
71
|
+
const attempts = Math.max(1, options.attempts ?? 3);
|
|
72
|
+
const retryDelayMs = Math.max(0, options.retryDelayMs ?? 100);
|
|
73
|
+
let lastError;
|
|
74
|
+
for (let attempt = 1; attempt <= attempts; attempt += 1) {
|
|
75
|
+
try {
|
|
76
|
+
return await readGitStatusPorcelainOnce(cwd);
|
|
77
|
+
}
|
|
78
|
+
catch (error) {
|
|
79
|
+
lastError = error;
|
|
80
|
+
if (attempt < attempts && retryDelayMs > 0) {
|
|
81
|
+
await new Promise((resolve) => setTimeout(resolve, retryDelayMs * attempt));
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
throw lastError instanceof Error
|
|
86
|
+
? lastError
|
|
87
|
+
: new Error(`git status failed after ${attempts} attempts`);
|
|
88
|
+
}
|
|
89
|
+
function readGitStatusPorcelainOnce(cwd) {
|
|
71
90
|
return new Promise((resolve, reject) => {
|
|
72
91
|
const child = spawn("git", ["status", "--porcelain=v1", "--untracked-files=all"], {
|
|
73
92
|
cwd,
|