@tea-agent/loop-agent 0.16.1 → 0.16.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -0
- package/dist/executors/dag-pi-executor.js +4 -2
- package/dist/executors/pi-sdk-executor.js +66 -3
- package/dist/executors/shell-executor.js +212 -29
- package/dist/executors/shell-presets.js +12 -2
- package/dist/executors/shell-write-guard.js +20 -1
- package/dist/shared/git-progress.js +9 -2
- package/dist/worker/observability/read-model.js +56 -0
- package/dist/worker/observe/server.js +6 -3
- package/dist/workflows/dag/backend-test-analysis-contract.js +87 -30
- package/dist/workflows/dag/backend-test-case-manifest.js +71 -8
- package/dist/workflows/dag/backend-test-execution-contract.js +63 -11
- package/dist/workflows/dag/backend-test-repair-contract.js +94 -0
- package/dist/workflows/dag/backend-test-result-contract.js +6 -4
- package/dist/workflows/dag/backend-test-semantic-review-contract.js +36 -0
- package/dist/workflows/dag/dynamic-runtime/condition.js +1 -1
- package/dist/workflows/dag/dynamic-runtime/shared.js +42 -0
- package/dist/workflows/dag/failure-routing.js +1 -1
- package/dist/workflows/dag/frontend-implementation-contract.js +32 -16
- package/dist/workflows/dag/init-hybrid.js +591 -119
- package/dist/workflows/dag/lifecycle.js +33 -2
- package/dist/workflows/dag/scheduler.js +87 -17
- package/dist/workflows/dag/types.js +31 -0
- package/dist/workflows/dag/validate.js +20 -14
- package/docs/templates/agent-dag.schema.json +25 -2
- package/docs/templates/backend-test-analysis.schema.json +9 -16
- package/docs/templates/backend-test-dag.json +493 -197
- package/docs/templates/backend-test-dag.review-cases.prompt.md +10 -4
- package/docs/templates/backend-test-execution.schema.json +6 -1
- package/package.json +1 -1
- package/skills/loop-agent/references/hybrid-dag.md +4 -1
|
@@ -3,50 +3,84 @@ import { readFile } from "node:fs/promises";
|
|
|
3
3
|
import path from "node:path";
|
|
4
4
|
import { z } from "zod";
|
|
5
5
|
import { writeDagRunJsonArtifact } from "../../infrastructure/harness/artifact-store.js";
|
|
6
|
-
export const BACKEND_TEST_ANALYSIS_SCHEMA_ID = "backend-test-analysis-
|
|
6
|
+
export const BACKEND_TEST_ANALYSIS_SCHEMA_ID = "backend-test-analysis-v2";
|
|
7
7
|
const sourceRefSchema = z.string().min(1);
|
|
8
|
-
const
|
|
8
|
+
const fieldV1Schema = z.object({
|
|
9
9
|
name: z.string().min(1),
|
|
10
10
|
type: z.string().min(1).optional(),
|
|
11
11
|
required: z.boolean().optional(),
|
|
12
12
|
description: z.string().optional(),
|
|
13
13
|
}).strict();
|
|
14
|
+
const fieldV2Schema = fieldV1Schema.extend({
|
|
15
|
+
format: z.string().min(1).optional(),
|
|
16
|
+
comparison: z.enum(["exact", "parseable-only", "semantic"]).optional(),
|
|
17
|
+
precision: z.string().min(1).optional(),
|
|
18
|
+
sourceRefs: z.array(sourceRefSchema).default([]),
|
|
19
|
+
}).strict();
|
|
14
20
|
const errorCaseSchema = z.object({
|
|
15
21
|
status: z.number().int().min(400).max(599).optional(),
|
|
16
22
|
code: z.string().min(1).optional(),
|
|
17
23
|
messageField: z.string().min(1).optional(),
|
|
18
24
|
description: z.string().min(1),
|
|
19
25
|
}).strict();
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
26
|
+
const optionalEvidenceDescriptionSchema = z.object({
|
|
27
|
+
name: z.string().min(1).optional(),
|
|
28
|
+
description: z.string().min(1),
|
|
29
|
+
sourceRef: sourceRefSchema.optional(),
|
|
30
|
+
}).strict();
|
|
31
|
+
const sourceBindingSchema = z.object({
|
|
32
|
+
taskId: z.string().min(1),
|
|
33
|
+
requirementPath: z.string().min(1),
|
|
34
|
+
requirementSha256: z.string().regex(/^[a-f0-9]{64}$/),
|
|
35
|
+
referencePaths: z.array(z.string().min(1)),
|
|
36
|
+
requirementIds: z.array(z.string().regex(/^(?:REQ|BR|AC)-[A-Z0-9]+(?:-[A-Z0-9]+)*$/)),
|
|
37
|
+
}).strict();
|
|
38
|
+
const commonShape = {
|
|
39
|
+
sourceBinding: sourceBindingSchema,
|
|
29
40
|
acceptanceCriteria: z.array(z.object({
|
|
30
41
|
id: z.string().regex(/^AC-[A-Z0-9]+(?:-[A-Z0-9]+)*$/),
|
|
31
42
|
text: z.string().min(1),
|
|
32
43
|
sourceRef: sourceRefSchema,
|
|
33
44
|
}).strict()),
|
|
34
|
-
endpoints: z.array(z.object({
|
|
35
|
-
id: z.string().min(1),
|
|
36
|
-
method: z.enum(["GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS"]),
|
|
37
|
-
path: z.string().startsWith("/"),
|
|
38
|
-
requestFields: z.array(fieldSchema),
|
|
39
|
-
responseFields: z.array(fieldSchema),
|
|
40
|
-
successStatuses: z.array(z.number().int().min(100).max(399)),
|
|
41
|
-
errorCases: z.array(errorCaseSchema),
|
|
42
|
-
}).strict()),
|
|
43
45
|
dataModels: z.array(z.object({ id: z.string().min(1), description: z.string().min(1), sourceRef: sourceRefSchema }).strict()),
|
|
44
46
|
businessRules: z.array(z.object({ id: z.string().min(1), text: z.string().min(1), sourceRef: sourceRefSchema }).strict()),
|
|
45
47
|
stateTransitions: z.array(z.object({ from: z.string().min(1), to: z.string().min(1), trigger: z.string().min(1), sourceRef: sourceRefSchema }).strict()),
|
|
46
48
|
boundaryConstraints: z.array(z.object({ field: z.string().min(1), constraint: z.string().min(1), sourceRef: sourceRefSchema }).strict()),
|
|
47
|
-
externalDependencies: z.array(
|
|
48
|
-
risks: z.array(
|
|
49
|
+
externalDependencies: z.array(optionalEvidenceDescriptionSchema),
|
|
50
|
+
risks: z.array(optionalEvidenceDescriptionSchema),
|
|
49
51
|
evidenceGaps: z.array(z.object({ description: z.string().min(1), requirementId: z.string().optional(), sourceRef: sourceRefSchema.optional() }).strict()),
|
|
52
|
+
};
|
|
53
|
+
const endpointBase = {
|
|
54
|
+
id: z.string().min(1),
|
|
55
|
+
method: z.enum(["GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS"]),
|
|
56
|
+
path: z.string().startsWith("/"),
|
|
57
|
+
successStatuses: z.array(z.number().int().min(100).max(399)),
|
|
58
|
+
errorCases: z.array(errorCaseSchema),
|
|
59
|
+
};
|
|
60
|
+
const analysisV1Schema = z.object({
|
|
61
|
+
schemaVersion: z.literal(1),
|
|
62
|
+
...commonShape,
|
|
63
|
+
endpoints: z.array(z.object({
|
|
64
|
+
...endpointBase,
|
|
65
|
+
requestFields: z.array(fieldV1Schema),
|
|
66
|
+
responseFields: z.array(fieldV1Schema),
|
|
67
|
+
}).strict()),
|
|
68
|
+
}).strict();
|
|
69
|
+
export const backendTestAnalysisContractSchema = z.object({
|
|
70
|
+
schemaVersion: z.literal(2),
|
|
71
|
+
...commonShape,
|
|
72
|
+
endpoints: z.array(z.object({
|
|
73
|
+
...endpointBase,
|
|
74
|
+
requestFields: z.array(fieldV2Schema),
|
|
75
|
+
responseFields: z.array(fieldV2Schema),
|
|
76
|
+
responseBody: z.object({
|
|
77
|
+
kind: z.enum(["array", "object", "scalar", "empty", "unknown"]),
|
|
78
|
+
itemSchemaRef: z.string().min(1).optional(),
|
|
79
|
+
ordering: z.enum(["specified", "unspecified", "not-applicable"]),
|
|
80
|
+
description: z.string().min(1).optional(),
|
|
81
|
+
}).strict(),
|
|
82
|
+
sourceRefs: z.array(sourceRefSchema),
|
|
83
|
+
}).strict()),
|
|
50
84
|
}).strict();
|
|
51
85
|
const SECRET_KEY = /(?:password|passwd|secret|token|api[_-]?key|private[_-]?key|credential|authorization)/i;
|
|
52
86
|
const SECRET_VALUE = /(?:-----BEGIN [A-Z ]*PRIVATE KEY-----|\b(?:sk|ghp|github_pat|xox[baprs]|AKIA)[-_A-Za-z0-9]{12,}\b)/;
|
|
@@ -74,13 +108,32 @@ export function extractStrictJsonObject(text) {
|
|
|
74
108
|
}
|
|
75
109
|
return JSON.parse(blocks[0][1]);
|
|
76
110
|
}
|
|
111
|
+
function normalizeAnalysis(value) {
|
|
112
|
+
const v2 = backendTestAnalysisContractSchema.safeParse(value);
|
|
113
|
+
if (v2.success)
|
|
114
|
+
return v2.data;
|
|
115
|
+
const v1 = analysisV1Schema.safeParse(value);
|
|
116
|
+
if (!v1.success) {
|
|
117
|
+
throw new Error(v2.error.issues.map((issue) => `${issue.path.join(".")}: ${issue.message}`).join("; "));
|
|
118
|
+
}
|
|
119
|
+
return backendTestAnalysisContractSchema.parse({
|
|
120
|
+
...v1.data,
|
|
121
|
+
schemaVersion: 2,
|
|
122
|
+
endpoints: v1.data.endpoints.map((endpoint) => ({
|
|
123
|
+
...endpoint,
|
|
124
|
+
requestFields: endpoint.requestFields.map((field) => ({ ...field, sourceRefs: [] })),
|
|
125
|
+
responseFields: endpoint.responseFields.map((field) => ({ ...field, sourceRefs: [] })),
|
|
126
|
+
responseBody: { kind: "unknown", ordering: "unspecified" },
|
|
127
|
+
sourceRefs: [],
|
|
128
|
+
})),
|
|
129
|
+
});
|
|
130
|
+
}
|
|
77
131
|
function assertSourceBinding(contract, binding) {
|
|
78
132
|
const requirement = binding.sources.find((source) => source.kind === "requirement");
|
|
79
133
|
const references = binding.sources.filter((source) => source.kind === "reference").map((source) => source.path).sort();
|
|
80
134
|
const actualReferences = [...contract.sourceBinding.referencePaths].sort();
|
|
81
|
-
if (!requirement || contract.sourceBinding.taskId !== binding.taskId || contract.sourceBinding.requirementPath !== requirement.path || contract.sourceBinding.requirementSha256 !== requirement.sha256)
|
|
135
|
+
if (!requirement || contract.sourceBinding.taskId !== binding.taskId || contract.sourceBinding.requirementPath !== requirement.path || contract.sourceBinding.requirementSha256 !== requirement.sha256)
|
|
82
136
|
throw new Error("analysis source binding does not match DAG requirement source");
|
|
83
|
-
}
|
|
84
137
|
if (JSON.stringify(actualReferences) !== JSON.stringify(references))
|
|
85
138
|
throw new Error("analysis referencePaths do not match DAG source binding");
|
|
86
139
|
if (JSON.stringify(contract.sourceBinding.requirementIds) !== JSON.stringify(binding.requirementIds))
|
|
@@ -109,12 +162,16 @@ export async function materializeBackendTestAnalysisContract(input) {
|
|
|
109
162
|
const secrets = secretIssues(parsed);
|
|
110
163
|
if (secrets.length)
|
|
111
164
|
throw new Error(`invalid-output: ${secrets.join("; ")}`);
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
165
|
+
let result;
|
|
166
|
+
try {
|
|
167
|
+
result = normalizeAnalysis(parsed);
|
|
168
|
+
}
|
|
169
|
+
catch (error) {
|
|
170
|
+
throw new Error(`invalid-output: ${error instanceof Error ? error.message : String(error)}`);
|
|
171
|
+
}
|
|
172
|
+
assertSourceBinding(result, input.sourceBinding);
|
|
116
173
|
const relativePath = path.posix.join(input.outputDir, input.artifactName);
|
|
117
|
-
const artifactPath = await writeDagRunJsonArtifact(input.runDir, relativePath, result
|
|
118
|
-
const normalized = `${JSON.stringify(result
|
|
174
|
+
const artifactPath = await writeDagRunJsonArtifact(input.runDir, relativePath, result);
|
|
175
|
+
const normalized = `${JSON.stringify(result, null, 2)}\n`;
|
|
119
176
|
return { path: artifactPath, sha256: createHash("sha256").update(normalized).digest("hex"), schemaId: BACKEND_TEST_ANALYSIS_SCHEMA_ID };
|
|
120
177
|
}
|
|
@@ -108,8 +108,10 @@ export const backendTestCaseManifestSchema = z
|
|
|
108
108
|
plannedCount: z.number().int().min(0),
|
|
109
109
|
skippedCount: z.number().int().min(0),
|
|
110
110
|
unsupportedCount: z.number().int().min(0),
|
|
111
|
-
/** Deterministic fraction in [0,1]; model must not invent this. */
|
|
112
111
|
acCoverageRatio: z.number().min(0).max(1),
|
|
112
|
+
global: z.object({ total: z.number().int().min(0), covered: z.number().int().min(0), gapped: z.number().int().min(0), ratio: z.number().min(0).max(1) }).strict().optional(),
|
|
113
|
+
inScope: z.object({ total: z.number().int().min(0), covered: z.number().int().min(0), gapped: z.number().int().min(0), ratio: z.number().min(0).max(1) }).strict().optional(),
|
|
114
|
+
crossDomainGapCount: z.number().int().min(0).optional(),
|
|
113
115
|
})
|
|
114
116
|
.strict()
|
|
115
117
|
.optional(),
|
|
@@ -138,10 +140,17 @@ export function extractStrictJsonObject(text) {
|
|
|
138
140
|
return JSON.parse(trimmed);
|
|
139
141
|
}
|
|
140
142
|
const blocks = [...trimmed.matchAll(/```json\s*\n([\s\S]*?)\n```/gi)];
|
|
141
|
-
if (blocks.length !== 1
|
|
142
|
-
throw new Error("manifest output must
|
|
143
|
+
if (blocks.length !== 1) {
|
|
144
|
+
throw new Error("manifest output must contain exactly one fenced json block");
|
|
143
145
|
}
|
|
144
|
-
|
|
146
|
+
// Writer nodes may summarize file changes before the manifest. The single
|
|
147
|
+
// fenced JSON block is authoritative; trailing text remains forbidden.
|
|
148
|
+
const last = blocks[0];
|
|
149
|
+
const trailing = trimmed.slice((last.index ?? 0) + last[0].length).trim();
|
|
150
|
+
if (trailing) {
|
|
151
|
+
throw new Error("manifest output must not contain trailing text after the final fenced json block");
|
|
152
|
+
}
|
|
153
|
+
return JSON.parse(last[1]);
|
|
145
154
|
}
|
|
146
155
|
function assertSourceBinding(manifest, binding) {
|
|
147
156
|
const requirement = binding.sources.find((source) => source.kind === "requirement");
|
|
@@ -195,6 +204,14 @@ export function computeCaseManifestCoverageSummary(manifest) {
|
|
|
195
204
|
const explicitAcCount = explicitAc.length;
|
|
196
205
|
const coveredAcCount = coveredExplicit.length;
|
|
197
206
|
const acCoverageRatio = explicitAcCount === 0 ? 1 : coveredAcCount / explicitAcCount;
|
|
207
|
+
const crossDomainAc = new Set(manifest.evidenceGaps
|
|
208
|
+
.filter((gap) => /(?:frontend|flyway|migration|mvn|browser|e2e|cross[- ]?domain)/i.test(gap.description))
|
|
209
|
+
.map((gap) => gap.acId)
|
|
210
|
+
.filter((id) => Boolean(id)));
|
|
211
|
+
const inScopeIds = explicitAc.filter((id) => !crossDomainAc.has(id));
|
|
212
|
+
const inScopeCovered = inScopeIds.filter((id) => covered.has(id)).length;
|
|
213
|
+
const inScopeGapped = inScopeIds.filter((id) => !covered.has(id) && gapped.has(id)).length;
|
|
214
|
+
const crossDomainGapCount = manifest.evidenceGaps.filter((gap) => /(?:frontend|flyway|migration|mvn|browser|e2e|cross[- ]?domain)/i.test(gap.description)).length;
|
|
198
215
|
return {
|
|
199
216
|
explicitAcCount,
|
|
200
217
|
coveredAcCount,
|
|
@@ -202,6 +219,19 @@ export function computeCaseManifestCoverageSummary(manifest) {
|
|
|
202
219
|
caseCount: manifest.cases.length,
|
|
203
220
|
...statusCounts,
|
|
204
221
|
acCoverageRatio,
|
|
222
|
+
global: {
|
|
223
|
+
total: explicitAcCount,
|
|
224
|
+
covered: coveredAcCount,
|
|
225
|
+
gapped: gappedExplicit.length,
|
|
226
|
+
ratio: acCoverageRatio,
|
|
227
|
+
},
|
|
228
|
+
inScope: {
|
|
229
|
+
total: inScopeIds.length,
|
|
230
|
+
covered: inScopeCovered,
|
|
231
|
+
gapped: inScopeGapped,
|
|
232
|
+
ratio: inScopeIds.length === 0 ? 1 : inScopeCovered / inScopeIds.length,
|
|
233
|
+
},
|
|
234
|
+
crossDomainGapCount,
|
|
205
235
|
};
|
|
206
236
|
}
|
|
207
237
|
export function assertBackendTestCaseManifestInvariants(manifest, binding) {
|
|
@@ -274,7 +304,10 @@ export function assertBackendTestCaseManifestInvariants(manifest, binding) {
|
|
|
274
304
|
const expected = computeCaseManifestCoverageSummary(manifest);
|
|
275
305
|
const actual = manifest.coverageSummary;
|
|
276
306
|
for (const key of Object.keys(expected)) {
|
|
277
|
-
|
|
307
|
+
const same = typeof expected[key] === "object"
|
|
308
|
+
? JSON.stringify(actual[key]) === JSON.stringify(expected[key])
|
|
309
|
+
: actual[key] === expected[key];
|
|
310
|
+
if (!same) {
|
|
278
311
|
throw new Error(`coverageSummary.${key} mismatch: expected ${String(expected[key])}, got ${String(actual[key])}`);
|
|
279
312
|
}
|
|
280
313
|
}
|
|
@@ -475,10 +508,40 @@ export async function assertBackendTestTraceability(input) {
|
|
|
475
508
|
}
|
|
476
509
|
return { ok: true, matched };
|
|
477
510
|
}
|
|
478
|
-
export async function
|
|
511
|
+
export async function deriveBackendTestGeneratedManifest(input) {
|
|
479
512
|
const relative = input.manifestRelativePath ?? "contracts/backend-test-case-manifest.json";
|
|
480
513
|
const manifestPath = path.join(input.runDir, relative);
|
|
481
|
-
const
|
|
514
|
+
const parsed = backendTestCaseManifestSchema.parse(JSON.parse(await readFile(manifestPath, "utf8")));
|
|
515
|
+
const hits = await discoverPytestSymbols(input.workspaceRoot, input.testRoot ?? "testcase");
|
|
516
|
+
const matched = [];
|
|
517
|
+
const cases = parsed.cases.map((item) => {
|
|
518
|
+
if (item.automationStatus === "skipped" || item.automationStatus === "unsupported")
|
|
519
|
+
return item;
|
|
520
|
+
const hit = hits.find((candidate) => symbolMatchesCase(candidate, item.caseId, item.symbol));
|
|
521
|
+
if (!hit)
|
|
522
|
+
return { ...item, automationStatus: "planned", file: undefined, symbol: undefined };
|
|
523
|
+
matched.push({ caseId: item.caseId, file: hit.file, symbol: hit.symbol });
|
|
524
|
+
return { ...item, automationStatus: "generated", file: hit.file, symbol: hit.symbol };
|
|
525
|
+
});
|
|
526
|
+
const derived = {
|
|
527
|
+
...parsed,
|
|
528
|
+
cases,
|
|
529
|
+
coverageSummary: undefined,
|
|
530
|
+
};
|
|
531
|
+
derived.coverageSummary = computeCaseManifestCoverageSummary(derived);
|
|
532
|
+
assertBackendTestCaseManifestInvariants(derived);
|
|
533
|
+
await writeDagRunJsonArtifact(input.runDir, relative, derived);
|
|
534
|
+
return {
|
|
535
|
+
path: manifestPath,
|
|
536
|
+
generatedCount: derived.coverageSummary.generatedCount,
|
|
537
|
+
plannedCount: derived.coverageSummary.plannedCount,
|
|
538
|
+
matched,
|
|
539
|
+
};
|
|
540
|
+
}
|
|
541
|
+
export async function runBackendTestTraceabilityGate(input) {
|
|
542
|
+
const relative = input.manifestRelativePath ?? "contracts/backend-test-case-manifest.json";
|
|
543
|
+
const derived = await deriveBackendTestGeneratedManifest(input);
|
|
544
|
+
const raw = JSON.parse(await readFile(derived.path, "utf8"));
|
|
482
545
|
const parsed = backendTestCaseManifestSchema.safeParse(raw);
|
|
483
546
|
if (!parsed.success) {
|
|
484
547
|
throw new Error(`traceability: invalid manifest: ${parsed.error.issues
|
|
@@ -494,7 +557,7 @@ export async function runBackendTestTraceabilityGate(input) {
|
|
|
494
557
|
return {
|
|
495
558
|
ok: true,
|
|
496
559
|
matchedCount: result.matched.length,
|
|
497
|
-
manifestPath,
|
|
560
|
+
manifestPath: derived.path,
|
|
498
561
|
};
|
|
499
562
|
}
|
|
500
563
|
/** Shell-friendly: build a one-liner that invokes node -e would be heavy; export for shell-executor special command. */
|
|
@@ -31,9 +31,62 @@ const fixtureSchema = z
|
|
|
31
31
|
kind: z.string().min(1),
|
|
32
32
|
})
|
|
33
33
|
.strict();
|
|
34
|
+
/**
|
|
35
|
+
* HTTP URL path for service readiness probes (e.g. `/api/health`).
|
|
36
|
+
* Not a repository file path — do not reuse relativePosixPathSchema.
|
|
37
|
+
*/
|
|
38
|
+
export function normalizeHttpReadinessPath(value) {
|
|
39
|
+
const trimmed = value.trim();
|
|
40
|
+
if (!trimmed)
|
|
41
|
+
throw new Error("readiness.path must be non-empty");
|
|
42
|
+
if (trimmed.includes("\\")) {
|
|
43
|
+
throw new Error("readiness.path must use posix separators");
|
|
44
|
+
}
|
|
45
|
+
// Windows treats paths starting with "/" as absolute; only reject drive-letter / UNC filesystem forms here.
|
|
46
|
+
if (/^[a-zA-Z]:/.test(trimmed) || /^[a-zA-Z]:[\\/]/.test(trimmed)) {
|
|
47
|
+
throw new Error("readiness.path must not be a filesystem absolute path");
|
|
48
|
+
}
|
|
49
|
+
if (trimmed.includes("://") || trimmed.startsWith("//")) {
|
|
50
|
+
throw new Error("readiness.path must be a URL path, not a full URL");
|
|
51
|
+
}
|
|
52
|
+
if (!trimmed.startsWith("/")) {
|
|
53
|
+
throw new Error("readiness.path must start with / (HTTP URL path)");
|
|
54
|
+
}
|
|
55
|
+
if (trimmed.includes("?") || trimmed.includes("#")) {
|
|
56
|
+
throw new Error("readiness.path must not include query or fragment");
|
|
57
|
+
}
|
|
58
|
+
const segments = trimmed.split("/");
|
|
59
|
+
// first segment is empty because path starts with /
|
|
60
|
+
for (const segment of segments.slice(1)) {
|
|
61
|
+
if (segment === "..")
|
|
62
|
+
throw new Error("readiness.path traversal is forbidden");
|
|
63
|
+
if (segment === ".")
|
|
64
|
+
throw new Error("readiness.path must not contain '.' segments");
|
|
65
|
+
}
|
|
66
|
+
// collapse duplicate slashes except keep single leading /
|
|
67
|
+
const body = segments
|
|
68
|
+
.slice(1)
|
|
69
|
+
.filter((part) => part !== "")
|
|
70
|
+
.join("/");
|
|
71
|
+
return body ? `/${body}` : "/";
|
|
72
|
+
}
|
|
73
|
+
const httpReadinessPathSchema = z
|
|
74
|
+
.string()
|
|
75
|
+
.min(1)
|
|
76
|
+
.refine((value) => {
|
|
77
|
+
try {
|
|
78
|
+
normalizeHttpReadinessPath(value);
|
|
79
|
+
return true;
|
|
80
|
+
}
|
|
81
|
+
catch {
|
|
82
|
+
return false;
|
|
83
|
+
}
|
|
84
|
+
}, {
|
|
85
|
+
message: "readiness.path must be an HTTP URL path (e.g. /api/health), not a repo-relative file path",
|
|
86
|
+
});
|
|
34
87
|
const readinessSchema = z
|
|
35
88
|
.object({
|
|
36
|
-
path:
|
|
89
|
+
path: httpReadinessPathSchema,
|
|
37
90
|
description: z.string().min(1),
|
|
38
91
|
})
|
|
39
92
|
.strict();
|
|
@@ -194,7 +247,7 @@ function normalizeContractPaths(contract) {
|
|
|
194
247
|
})),
|
|
195
248
|
readiness: readiness.map((item) => ({
|
|
196
249
|
...item,
|
|
197
|
-
path:
|
|
250
|
+
path: normalizeHttpReadinessPath(item.path),
|
|
198
251
|
})),
|
|
199
252
|
requiredEnvNames,
|
|
200
253
|
evidenceGaps,
|
|
@@ -258,12 +311,11 @@ export function assertBackendTestExecutionPreflight(input) {
|
|
|
258
311
|
errors.push("managed-command is fail-closed without managedCommand.sourceRef task-source evidence");
|
|
259
312
|
}
|
|
260
313
|
}
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
}
|
|
314
|
+
// evidenceGaps are informational for greenfield backend-test (generate may fill
|
|
315
|
+
// missing test_*.py later). Hard env/targetMode/fixture checks remain below.
|
|
264
316
|
if (contract.targetMode === "in-process" &&
|
|
265
317
|
(contract.existingFixtures?.length ?? 0) === 0) {
|
|
266
|
-
errors.push("in-process targetMode requires at least one existingFixtures entry
|
|
318
|
+
errors.push("in-process targetMode requires at least one existingFixtures entry");
|
|
267
319
|
}
|
|
268
320
|
return {
|
|
269
321
|
ok: errors.length === 0,
|
|
@@ -298,10 +350,11 @@ export async function materializeBackendTestExecutionContract(input) {
|
|
|
298
350
|
.join("; ")}`);
|
|
299
351
|
}
|
|
300
352
|
const normalized = normalizeContractPaths(result.data);
|
|
301
|
-
//
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
353
|
+
// evidenceGaps may record greenfield/incomplete discovery (no test_*.py yet,
|
|
354
|
+
// missing pytest.ini, projected schema path, etc.). Do not block materialize:
|
|
355
|
+
// generate-functional-cases / generate-pytest are expected to fill automation.
|
|
356
|
+
// Hard fail-closed remains: schema/secrets, in-process fixtures, managed sourceRef
|
|
357
|
+
// (schema superRefine), and execute preflight env/targetMode checks.
|
|
305
358
|
if (normalized.targetMode === "in-process" &&
|
|
306
359
|
(normalized.existingFixtures?.length ?? 0) === 0) {
|
|
307
360
|
throw new Error("invalid-output: in-process targetMode requires existingFixtures evidence");
|
|
@@ -335,7 +388,6 @@ export function buildBackendTestExecutionPreflightShellSnippet(options) {
|
|
|
335
388
|
`if(testRoot.replace(/\\/+$/,"")!==expected.replace(/\\/+$/,"")) errors.push("testRoot mismatch vs frozen command: "+testRoot+" !== "+expected);`,
|
|
336
389
|
`const rootAbs=path.resolve(process.cwd(),testRoot);`,
|
|
337
390
|
`if(!fs.existsSync(rootAbs)) errors.push("testRoot does not exist: "+testRoot);`,
|
|
338
|
-
`if(Array.isArray(contract.evidenceGaps)&&contract.evidenceGaps.length) errors.push("evidenceGaps present: "+contract.evidenceGaps.length);`,
|
|
339
391
|
`if(contract.targetMode==="in-process"&&!(Array.isArray(contract.existingFixtures)&&contract.existingFixtures.length)) errors.push("in-process requires existingFixtures");`,
|
|
340
392
|
`for (const name of (contract.requiredEnvNames||[])) { if(!process.env[name]) errors.push("required env missing: "+name); }`,
|
|
341
393
|
`if(contract.targetMode==="external-running-service"){ const n=contract.baseUrlEnvName; if(!n||!process.env[n]) errors.push("external base URL env missing: "+String(n||"<empty>")); }`,
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { readFile } from "node:fs/promises";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
import { z } from "zod";
|
|
5
|
+
import { writeDagRunJsonArtifact } from "../../infrastructure/harness/artifact-store.js";
|
|
6
|
+
import { BACKEND_TEST_CLASSIFICATION_CATEGORIES, backendTestResultContractSchema, } from "./backend-test-result-contract.js";
|
|
7
|
+
export const BACKEND_TEST_CLASSIFICATION_SCHEMA_ID = "backend-test-classification-v1";
|
|
8
|
+
export const backendTestClassificationContractSchema = z
|
|
9
|
+
.object({
|
|
10
|
+
category: z.enum(BACKEND_TEST_CLASSIFICATION_CATEGORIES),
|
|
11
|
+
evidence: z.array(z.string().min(1)).min(1),
|
|
12
|
+
confidence: z.number().min(0).max(1),
|
|
13
|
+
notes: z.string().min(1),
|
|
14
|
+
})
|
|
15
|
+
.strict();
|
|
16
|
+
function extractJsonObject(text) {
|
|
17
|
+
const trimmed = text.trim();
|
|
18
|
+
const fenced = trimmed.match(/^```json\s*([\s\S]*?)\s*```$/i);
|
|
19
|
+
return JSON.parse(fenced ? fenced[1] : trimmed);
|
|
20
|
+
}
|
|
21
|
+
export async function materializeBackendTestClassification(input) {
|
|
22
|
+
const wrapperPath = path.join(input.runDir, `${input.fromNodeId}.json`);
|
|
23
|
+
const wrapper = JSON.parse(await readFile(wrapperPath, "utf8"));
|
|
24
|
+
const raw = wrapper.assistantText?.trim() || wrapper.stdout?.trim() || "";
|
|
25
|
+
if (!raw)
|
|
26
|
+
throw new Error("backend-test classification output is empty");
|
|
27
|
+
const classification = backendTestClassificationContractSchema.parse(extractJsonObject(raw));
|
|
28
|
+
const relativePath = path.posix.join(input.outputDir, input.artifactName);
|
|
29
|
+
const artifactPath = await writeDagRunJsonArtifact(input.runDir, relativePath, classification);
|
|
30
|
+
const serialized = `${JSON.stringify(classification, null, 2)}\n`;
|
|
31
|
+
return {
|
|
32
|
+
path: artifactPath,
|
|
33
|
+
sha256: createHash("sha256").update(serialized).digest("hex"),
|
|
34
|
+
schemaId: BACKEND_TEST_CLASSIFICATION_SCHEMA_ID,
|
|
35
|
+
};
|
|
36
|
+
}
|
|
37
|
+
export function assessBackendTestRepairEligibility(input) {
|
|
38
|
+
const result = backendTestResultContractSchema.parse(input.result);
|
|
39
|
+
const classification = backendTestClassificationContractSchema.parse(input.classification);
|
|
40
|
+
const attempt = input.revisionAttempt ?? 0;
|
|
41
|
+
let reason = "eligible TestBug assertion failure";
|
|
42
|
+
let eligible = true;
|
|
43
|
+
if (attempt !== 0) {
|
|
44
|
+
eligible = false;
|
|
45
|
+
reason = "repair attempt already consumed";
|
|
46
|
+
}
|
|
47
|
+
else if (classification.category !== "TestBug") {
|
|
48
|
+
eligible = false;
|
|
49
|
+
reason = `classification ${classification.category} is not TestBug`;
|
|
50
|
+
}
|
|
51
|
+
else if (classification.confidence < 0.7) {
|
|
52
|
+
eligible = false;
|
|
53
|
+
reason = "TestBug confidence is below 0.7";
|
|
54
|
+
}
|
|
55
|
+
else if (result.executionStatus !== "completed" ||
|
|
56
|
+
result.outcome !== "completed-with-failures") {
|
|
57
|
+
eligible = false;
|
|
58
|
+
reason = "initial pytest did not complete with assertion failures";
|
|
59
|
+
}
|
|
60
|
+
else if (result.failed <= 0 || result.error !== 0) {
|
|
61
|
+
eligible = false;
|
|
62
|
+
reason = "repair requires failed>0 and error=0";
|
|
63
|
+
}
|
|
64
|
+
else if (result.failures.length === 0) {
|
|
65
|
+
eligible = false;
|
|
66
|
+
reason = "repair requires concrete failure summaries";
|
|
67
|
+
}
|
|
68
|
+
return {
|
|
69
|
+
schemaVersion: 1,
|
|
70
|
+
eligible,
|
|
71
|
+
reason,
|
|
72
|
+
revisionAttempt: attempt === 0 ? 0 : 1,
|
|
73
|
+
category: classification.category,
|
|
74
|
+
confidence: classification.confidence,
|
|
75
|
+
};
|
|
76
|
+
}
|
|
77
|
+
export function buildBackendTestRepairEligibilityShellSnippet() {
|
|
78
|
+
return [
|
|
79
|
+
'test -n "${HARNESS_DAG_RUN_DIR:-}" || { echo "missing HARNESS_DAG_RUN_DIR for backend-test repair eligibility" >&2; exit 2; }',
|
|
80
|
+
`node -e 'const fs=require("fs"),path=require("path");const root=process.env.HARNESS_DAG_RUN_DIR;const r=JSON.parse(fs.readFileSync(path.join(root,"contracts","backend-test-result-initial.json"),"utf8"));const c=JSON.parse(fs.readFileSync(path.join(root,"contracts","backend-test-classification.json"),"utf8"));let eligible=true,reason="eligible TestBug assertion failure";if(c.category!=="TestBug"){eligible=false;reason="classification "+c.category+" is not TestBug";}else if(Number(c.confidence)<0.7){eligible=false;reason="TestBug confidence is below 0.7";}else if(r.executionStatus!=="completed"||r.outcome!=="completed-with-failures"){eligible=false;reason="initial pytest did not complete with assertion failures";}else if(Number(r.failed)<=0||Number(r.error)!==0){eligible=false;reason="repair requires failed>0 and error=0";}else if(!Array.isArray(r.failures)||r.failures.length===0){eligible=false;reason="repair requires concrete failure summaries";}const out={schemaVersion:1,eligible,reason,revisionAttempt:0,category:c.category,confidence:c.confidence};fs.writeFileSync(path.join(root,"contracts","backend-test-repair-eligibility.json"),JSON.stringify(out,null,2)+"\\n");process.stdout.write(JSON.stringify(out));'`,
|
|
81
|
+
].join("; ");
|
|
82
|
+
}
|
|
83
|
+
export function buildBackendTestEffectiveResultSelectorShellSnippet() {
|
|
84
|
+
return [
|
|
85
|
+
'test -n "${HARNESS_DAG_RUN_DIR:-}" || { echo "missing HARNESS_DAG_RUN_DIR for backend-test effective result" >&2; exit 2; }',
|
|
86
|
+
`node -e 'const fs=require("fs"),path=require("path"),crypto=require("crypto");const root=process.env.HARNESS_DAG_RUN_DIR;const contracts=path.join(root,"contracts");const finalPath=path.join(contracts,"backend-test-result-final.json");const initialPath=path.join(contracts,"backend-test-result-initial.json");const source=fs.existsSync(finalPath)?finalPath:initialPath;if(!fs.existsSync(source))throw new Error("missing initial/final backend-test result");const result=JSON.parse(fs.readFileSync(source,"utf8"));const target=path.join(contracts,"backend-test-result.json");const body=JSON.stringify(result,null,2)+"\\n";fs.writeFileSync(target,body);process.stdout.write(JSON.stringify({effectiveResult:path.basename(source),revisionAttempt:fs.existsSync(finalPath)?1:0,outcome:result.outcome,sha256:crypto.createHash("sha256").update(body).digest("hex")}));'`,
|
|
87
|
+
].join("; ");
|
|
88
|
+
}
|
|
89
|
+
export function buildBackendTestRepairSafetyShellSnippet() {
|
|
90
|
+
return [
|
|
91
|
+
'test -d testcase || { echo "missing testcase directory after TestBug repair" >&2; exit 1; }',
|
|
92
|
+
`node -e 'const fs=require("fs"),path=require("path");const root="testcase";const bad=[];function walk(p){for(const e of fs.readdirSync(p,{withFileTypes:true})){const f=path.join(p,e.name);if(e.isDirectory())walk(f);else if(/\\.py$/.test(e.name)){const s=fs.readFileSync(f,"utf8");if(/@pytest\\.mark\\.(?:skip|skipif|xfail)\\b|pytest\\.(?:skip|xfail)\\s*\\(/.test(s))bad.push(f+": skip/xfail is forbidden in automatic repair");if(/except\\s+(?:AssertionError|Exception|BaseException)\\s*:\\s*(?:pass|return)\\b/.test(s))bad.push(f+": broad failure swallowing is forbidden");}}}walk(root);if(bad.length){console.error(bad.join("\\n"));process.exit(1);}console.log("backend-test repair safety gate: pass");'`,
|
|
93
|
+
].join("; ");
|
|
94
|
+
}
|
|
@@ -479,8 +479,8 @@ export function classifyCategoryHints(result) {
|
|
|
479
479
|
confidenceCap: 0.75,
|
|
480
480
|
};
|
|
481
481
|
}
|
|
482
|
-
async function readPytestExitCode(runDir, fromNodeId) {
|
|
483
|
-
const exitPath = path.join(runDir, "
|
|
482
|
+
async function readPytestExitCode(runDir, fromNodeId, exitRelativePath = "reports/backend-test-pytest-exit.txt") {
|
|
483
|
+
const exitPath = path.join(runDir, ...exitRelativePath.split("/"));
|
|
484
484
|
try {
|
|
485
485
|
const raw = (await readFile(exitPath, "utf8")).trim();
|
|
486
486
|
const code = Number(raw);
|
|
@@ -531,8 +531,10 @@ export async function materializeBackendTestResultFromRunDir(input) {
|
|
|
531
531
|
if (!junitXml.trim()) {
|
|
532
532
|
throw new Error("invalid junit xml: empty report");
|
|
533
533
|
}
|
|
534
|
-
const pytestExitCode = await readPytestExitCode(input.runDir, input.fromNodeId)
|
|
535
|
-
|
|
534
|
+
const pytestExitCode = await readPytestExitCode(input.runDir, input.fromNodeId, input.junitRelativePath?.endsWith("-junit.xml")
|
|
535
|
+
? input.junitRelativePath.replace(/-junit\.xml$/, "-pytest-exit.txt")
|
|
536
|
+
: "reports/backend-test-pytest-exit.txt");
|
|
537
|
+
const commandSummary = `PYTHONUTF8=1 PYTHONIOENCODING=utf-8 PYTHONDONTWRITEBYTECODE=1 python -m pytest testcase/ -v -p no:cacheprovider --junitxml=${junitRelativePath}`;
|
|
536
538
|
let result;
|
|
537
539
|
try {
|
|
538
540
|
result = deriveBackendTestResult({
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { readFile } from "node:fs/promises";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
import { z } from "zod";
|
|
5
|
+
import { writeDagRunJsonArtifact } from "../../infrastructure/harness/artifact-store.js";
|
|
6
|
+
export const BACKEND_TEST_SEMANTIC_REVIEW_SCHEMA_ID = "backend-test-semantic-review-v1";
|
|
7
|
+
export const backendTestSemanticReviewSchema = z.object({
|
|
8
|
+
verdict: z.enum(["pass", "request-revision"]),
|
|
9
|
+
findings: z.array(z.object({
|
|
10
|
+
severity: z.enum(["Critical", "Important", "Informational"]),
|
|
11
|
+
caseId: z.string().regex(/^BE-[A-Z0-9]+(?:-[A-Z0-9]+)*-\d{3}$/),
|
|
12
|
+
testFile: z.string().min(1),
|
|
13
|
+
testSymbol: z.string().min(1),
|
|
14
|
+
contractRefs: z.array(z.string().min(1)).min(1),
|
|
15
|
+
issue: z.string().min(1),
|
|
16
|
+
requiredChange: z.string().min(1),
|
|
17
|
+
})).default([]),
|
|
18
|
+
summary: z.string().min(1),
|
|
19
|
+
}).strict().superRefine((value, ctx) => {
|
|
20
|
+
if (value.verdict === "pass" && value.findings.some((item) => item.severity === "Critical")) {
|
|
21
|
+
ctx.addIssue({ code: z.ZodIssueCode.custom, message: "pass verdict cannot contain Critical findings", path: ["verdict"] });
|
|
22
|
+
}
|
|
23
|
+
if (value.verdict === "request-revision" && value.findings.length === 0) {
|
|
24
|
+
ctx.addIssue({ code: z.ZodIssueCode.custom, message: "request-revision requires findings", path: ["findings"] });
|
|
25
|
+
}
|
|
26
|
+
});
|
|
27
|
+
export async function materializeBackendTestSemanticReview(input) {
|
|
28
|
+
const wrapper = JSON.parse(await readFile(path.join(input.runDir, `${input.fromNodeId}.json`), "utf8"));
|
|
29
|
+
const raw = wrapper.assistantText?.trim() || wrapper.stdout?.trim() || "";
|
|
30
|
+
const fenced = raw.match(/^```json\s*([\s\S]*?)\s*```$/i);
|
|
31
|
+
const parsed = backendTestSemanticReviewSchema.parse(JSON.parse(fenced ? fenced[1] : raw));
|
|
32
|
+
const relativePath = path.posix.join(input.outputDir, input.artifactName);
|
|
33
|
+
const artifactPath = await writeDagRunJsonArtifact(input.runDir, relativePath, parsed);
|
|
34
|
+
const normalized = `${JSON.stringify(parsed, null, 2)}\n`;
|
|
35
|
+
return { path: artifactPath, sha256: createHash("sha256").update(normalized).digest("hex"), schemaId: BACKEND_TEST_SEMANTIC_REVIEW_SCHEMA_ID };
|
|
36
|
+
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { parseConditionLiteral, resolveOutputSelector } from "./shared.js";
|
|
2
|
-
function evaluateConditionExpression(expression, state) {
|
|
2
|
+
export function evaluateConditionExpression(expression, state) {
|
|
3
3
|
const equality = expression.match(/^\s*(\$\..+?)\s*==\s*(.+?)\s*$/);
|
|
4
4
|
if (!equality?.[1]) {
|
|
5
5
|
throw new Error(`unsupported condition expression: ${expression}`);
|
|
@@ -28,7 +28,49 @@ export function getNodeOutputAsJson(state, nodeId) {
|
|
|
28
28
|
throw new Error(`itemsFrom upstream node "${nodeId}" output is not valid JSON: ${error instanceof Error ? error.message : String(error)}`);
|
|
29
29
|
}
|
|
30
30
|
}
|
|
31
|
+
/** Normalize a line the same way verdict gates do (strip whole-line Markdown emphasis). */
|
|
32
|
+
export function normalizeVerdictCandidateLine(value) {
|
|
33
|
+
const trimmed = String(value).trim();
|
|
34
|
+
const emphasized = trimmed.match(/^\*{1,3}\s*(VERDICT:[^*]+?)\s*\*{1,3}$/);
|
|
35
|
+
return (emphasized ? emphasized[1] : trimmed).trim();
|
|
36
|
+
}
|
|
37
|
+
/** First line matching /^VERDICT:/ from node assistantText/stdout (markdown emphasis OK). */
|
|
38
|
+
export function extractFirstVerdictLineFromNodeText(text) {
|
|
39
|
+
for (const line of String(text).split(/\r?\n/)) {
|
|
40
|
+
const normalized = normalizeVerdictCandidateLine(line);
|
|
41
|
+
if (/^VERDICT:/.test(normalized))
|
|
42
|
+
return normalized;
|
|
43
|
+
}
|
|
44
|
+
return "";
|
|
45
|
+
}
|
|
46
|
+
function getNodeRawText(state, nodeId) {
|
|
47
|
+
const record = state.nodes[nodeId];
|
|
48
|
+
if (!record || record.status !== "FINISHED") {
|
|
49
|
+
throw new Error(`selector upstream node "${nodeId}" is not finished`);
|
|
50
|
+
}
|
|
51
|
+
return record.assistantText?.trim() || record.stdout?.trim() || "";
|
|
52
|
+
}
|
|
31
53
|
export function resolveOutputSelector(selector, state) {
|
|
54
|
+
const firstVerdictMatch = selector.match(/^\$\.nodes\[['"]([^'"]+)['"]\]\.firstVerdictLine\s*$/);
|
|
55
|
+
if (firstVerdictMatch?.[1]) {
|
|
56
|
+
const nodeId = firstVerdictMatch[1];
|
|
57
|
+
return extractFirstVerdictLineFromNodeText(getNodeRawText(state, nodeId));
|
|
58
|
+
}
|
|
59
|
+
const jsonFieldMatch = selector.match(/^\$\.nodes\[['"]([^'"]+)['"]\]\.json\.([A-Za-z0-9_.-]+)\s*$/);
|
|
60
|
+
if (jsonFieldMatch?.[1] && jsonFieldMatch[2]) {
|
|
61
|
+
let current = parseJsonFromText(getNodeRawText(state, jsonFieldMatch[1]));
|
|
62
|
+
if (current === undefined) {
|
|
63
|
+
throw new Error(`selector upstream node "${jsonFieldMatch[1]}" output is not valid JSON`);
|
|
64
|
+
}
|
|
65
|
+
for (const segment of jsonFieldMatch[2].split(".")) {
|
|
66
|
+
if (current && typeof current === "object" && segment in current) {
|
|
67
|
+
current = current[segment];
|
|
68
|
+
continue;
|
|
69
|
+
}
|
|
70
|
+
throw new Error(`output selector path not found: ${selector} (missing "${segment}")`);
|
|
71
|
+
}
|
|
72
|
+
return current;
|
|
73
|
+
}
|
|
32
74
|
const match = selector.match(/^\$\.nodes\[['"]([^'"]+)['"]\]\.output(?:\.(.+))?$/);
|
|
33
75
|
if (!match?.[1]) {
|
|
34
76
|
throw new Error(`unsupported itemsFrom selector: ${selector}`);
|
|
@@ -86,7 +86,7 @@ function routeToProductLine(input) {
|
|
|
86
86
|
}
|
|
87
87
|
}
|
|
88
88
|
export function routeDagFailure(input) {
|
|
89
|
-
const productLineFailureCategory = routeToProductLine(input);
|
|
89
|
+
const productLineFailureCategory = input.productLineFailureCategory ?? routeToProductLine(input);
|
|
90
90
|
if (!productLineFailureCategory)
|
|
91
91
|
return {};
|
|
92
92
|
const recommendedFollowUp = productLineFailureCategory === "ContractMismatch" &&
|