@tea-agent/loop-agent 0.28.13 → 0.29.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +1 -1
- package/CHANGELOG.md +73 -15
- package/dist/commands/client-recovery.js +56 -1
- package/dist/commands/init-upgrade.js +186 -21
- package/dist/commands/init.js +1 -1
- package/dist/executors/dag-pi-executor.js +52 -3
- package/dist/executors/pi-playwright-cli-tool.js +14 -8
- package/dist/executors/shell-executor.js +288 -0
- package/dist/task/config-types.js +21 -0
- package/dist/worker/console/app-data.js +132 -11
- package/dist/worker/console/chat/pi-runtime.js +24 -42
- package/dist/worker/console/chat/resource-loader.js +11 -20
- package/dist/worker/console/chat/routes.js +7 -8
- package/dist/worker/console/chat/runtime-context.js +1 -1
- package/dist/worker/console/chat/tools.js +67 -54
- package/dist/worker/console/operation-runner.js +15 -1
- package/dist/worker/console/operation-store.js +70 -49
- package/dist/worker/console/operator-actions.js +57 -1
- package/dist/worker/console/static/assets/index-Cwx-ZVEQ.js +29 -0
- package/dist/worker/console/static/favicon.svg +37 -0
- package/dist/worker/console/static/index.html +2 -1
- package/dist/workflows/dag/backend-test-scenario-param.js +846 -0
- package/dist/workflows/dag/backend-test-writer-completeness.js +418 -0
- package/dist/workflows/dag/frontend-test-case-checklist.js +94 -15
- package/dist/workflows/dag/frontend-test-case-manifest.js +104 -0
- package/dist/workflows/dag/frontend-test-html-report.js +106 -24
- package/dist/workflows/dag/frontend-test-result-contract.js +3 -0
- package/dist/workflows/dag/init-hybrid.js +187 -108
- package/dist/workflows/dag/node-execution.js +31 -2
- package/dist/workflows/dag/retry-policy.js +55 -18
- package/dist/workflows/dag/types.js +41 -0
- package/dist/workflows/dag/validate.js +42 -4
- package/docs/operations/README.md +1 -1
- package/docs/templates/README.md +2 -1
- package/docs/templates/agent-dag.schema.json +9 -4
- package/docs/templates/backend-test-dag.json +36 -11
- package/docs/templates/frontend-test-case-checklist.md +1 -1
- package/docs/templates/frontend-test-dag.generate-cases.prompt.md +9 -1
- package/docs/templates/frontend-test-dag.json +125 -267
- package/docs/templates/frontend-test-dag.retrieve-context.prompt.md +7 -1
- package/docs/templates/frontend-test-dag.review-cases.prompt.md +1 -1
- package/docs/templates/frontend-test-standard-scenarios.v1.json +114 -0
- package/docs/templates/init-managed-agents.md +1 -1
- package/harness.json +2 -2
- package/package.json +2 -1
- package/skills/playwright-cli/SKILL.md +1 -1
- package/skills/playwright-cli-case-generator/SKILL.md +1 -1
- package/dist/worker/console/static/assets/index-BfRgtLF4.js +0 -29
|
@@ -0,0 +1,418 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { access, mkdir, readFile, readdir, writeFile } from "node:fs/promises";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
import { z } from "zod";
|
|
5
|
+
import { expectedBackendTestPytestScriptForMarkdownModule, normalizeBackendTestModuleStem, } from "./backend-test-markdown-workflow.js";
|
|
6
|
+
export const BACKEND_TEST_WRITER_PROGRESS_SCHEMA_ID = "backend-test-writer-progress-v1";
|
|
7
|
+
export const BACKEND_TEST_OUTPUT_LIMIT_RECOVERY_REPORT = "backend-test-output-limit-recovery.md";
|
|
8
|
+
const progressSchema = z
|
|
9
|
+
.object({
|
|
10
|
+
schemaId: z.literal(BACKEND_TEST_WRITER_PROGRESS_SCHEMA_ID),
|
|
11
|
+
role: z.enum(["md-generate", "pytest-generate", "report"]),
|
|
12
|
+
status: z.enum(["PASS", "RECOVERABLE", "NON_RECOVERABLE"]),
|
|
13
|
+
expectedPaths: z.array(z.string()),
|
|
14
|
+
actualPaths: z.array(z.string()),
|
|
15
|
+
missingPaths: z.array(z.string()),
|
|
16
|
+
brokenPaths: z.array(z.string()),
|
|
17
|
+
targetPaths: z.array(z.string()),
|
|
18
|
+
issues: z.array(z
|
|
19
|
+
.object({
|
|
20
|
+
code: z.enum(["T3", "T4", "T5", "T6"]),
|
|
21
|
+
path: z.string().optional(),
|
|
22
|
+
detail: z.string(),
|
|
23
|
+
recoverable: z.boolean(),
|
|
24
|
+
})
|
|
25
|
+
.strict()),
|
|
26
|
+
})
|
|
27
|
+
.strict();
|
|
28
|
+
async function exists(filePath) {
|
|
29
|
+
try {
|
|
30
|
+
await access(filePath);
|
|
31
|
+
return true;
|
|
32
|
+
}
|
|
33
|
+
catch {
|
|
34
|
+
return false;
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
function orderedUnique(values) {
|
|
38
|
+
return [...new Set(values.filter(Boolean))];
|
|
39
|
+
}
|
|
40
|
+
function hasMarkdownTable(section, headerNeedle) {
|
|
41
|
+
const lines = section
|
|
42
|
+
.replaceAll("\r\n", "\n")
|
|
43
|
+
.replaceAll("\r", "\n")
|
|
44
|
+
.split("\n");
|
|
45
|
+
const headerIndex = lines.findIndex((line) => line.toLowerCase().includes(headerNeedle.toLowerCase()));
|
|
46
|
+
if (headerIndex < 0)
|
|
47
|
+
return false;
|
|
48
|
+
const header = lines[headerIndex] ?? "";
|
|
49
|
+
const separator = lines[headerIndex + 1] ?? "";
|
|
50
|
+
return (header.includes("|") &&
|
|
51
|
+
/-{3,}/.test(separator) &&
|
|
52
|
+
separator.includes("|"));
|
|
53
|
+
}
|
|
54
|
+
function extractModuleStemsFromReadme(readme) {
|
|
55
|
+
const stems = [];
|
|
56
|
+
for (const match of readme.matchAll(/`?testcase\/md\/([A-Za-z0-9_.-]+)\.md`?/g)) {
|
|
57
|
+
const stem = match[1];
|
|
58
|
+
if (stem && stem.toLowerCase() !== "readme")
|
|
59
|
+
stems.push(stem);
|
|
60
|
+
}
|
|
61
|
+
for (const match of readme.matchAll(/\|\s*`?([A-Za-z0-9_.-]+)`?\s*\|\s*`?testcase\/test_/g)) {
|
|
62
|
+
if (match[1])
|
|
63
|
+
stems.push(match[1]);
|
|
64
|
+
}
|
|
65
|
+
for (const match of readme.matchAll(/\[[^\]]+\]\(\.\/([A-Za-z0-9_.-]+)\.md\)/g)) {
|
|
66
|
+
if (match[1] && match[1].toLowerCase() !== "readme")
|
|
67
|
+
stems.push(match[1]);
|
|
68
|
+
}
|
|
69
|
+
return orderedUnique(stems.map((stem) => normalizeBackendTestModuleStem(stem)));
|
|
70
|
+
}
|
|
71
|
+
async function listMarkdownModules(workspaceRoot) {
|
|
72
|
+
const root = path.join(workspaceRoot, "testcase", "md");
|
|
73
|
+
if (!(await exists(root)))
|
|
74
|
+
return [];
|
|
75
|
+
const entries = await readdir(root, { withFileTypes: true });
|
|
76
|
+
return orderedUnique(entries
|
|
77
|
+
.filter((entry) => entry.isFile() &&
|
|
78
|
+
entry.name.toLowerCase().endsWith(".md") &&
|
|
79
|
+
entry.name.toLowerCase() !== "readme.md")
|
|
80
|
+
.map((entry) => path.posix.join("testcase/md", entry.name.replaceAll("\\", "/"))));
|
|
81
|
+
}
|
|
82
|
+
function moduleStructurallyComplete(markdown) {
|
|
83
|
+
if (!markdown.trim())
|
|
84
|
+
return false;
|
|
85
|
+
if (!/^##\s+BE-[A-Z0-9_-]+-\d{2,3}\b/m.test(markdown))
|
|
86
|
+
return false;
|
|
87
|
+
const required = [
|
|
88
|
+
"\u8986\u76d6\u89c4\u5219",
|
|
89
|
+
"\u6d4b\u8bd5\u70b9",
|
|
90
|
+
"\u573a\u666f\u7c7b\u578b",
|
|
91
|
+
"\u524d\u7f6e\u6761\u4ef6",
|
|
92
|
+
"\u64cd\u4f5c\u6b65\u9aa4",
|
|
93
|
+
"\u9884\u671f\u7ed3\u679c",
|
|
94
|
+
"\u81ea\u52a8\u5316\u6620\u5c04"
|
|
95
|
+
];
|
|
96
|
+
return required.every((heading) => markdown.includes(`### ${heading}`));
|
|
97
|
+
}
|
|
98
|
+
function pythonParseable(source) {
|
|
99
|
+
const text = source.replace(/\r\n/g, "\n");
|
|
100
|
+
if (!text.trim())
|
|
101
|
+
return false;
|
|
102
|
+
if (/^\s*(?:def|class|import|from)\b/m.test(text) === false)
|
|
103
|
+
return false;
|
|
104
|
+
const openParens = (text.match(/\(/g) ?? []).length;
|
|
105
|
+
const closeParens = (text.match(/\)/g) ?? []).length;
|
|
106
|
+
const openBrackets = (text.match(/\[/g) ?? []).length;
|
|
107
|
+
const closeBrackets = (text.match(/\]/g) ?? []).length;
|
|
108
|
+
const openBraces = (text.match(/\{/g) ?? []).length;
|
|
109
|
+
const closeBraces = (text.match(/\}/g) ?? []).length;
|
|
110
|
+
if (openParens !== closeParens)
|
|
111
|
+
return false;
|
|
112
|
+
if (openBrackets !== closeBrackets)
|
|
113
|
+
return false;
|
|
114
|
+
if (openBraces !== closeBraces)
|
|
115
|
+
return false;
|
|
116
|
+
if (/("""|''')[\s\S]*$/.test(text)) {
|
|
117
|
+
const triples = text.match(/("""|''')/g) ?? [];
|
|
118
|
+
if (triples.length % 2 !== 0)
|
|
119
|
+
return false;
|
|
120
|
+
}
|
|
121
|
+
if (/\bdef\s+\w+\s*\([^)]*$/m.test(text))
|
|
122
|
+
return false;
|
|
123
|
+
if (/\bpytest\.param\s*\([^)]*$/m.test(text))
|
|
124
|
+
return false;
|
|
125
|
+
return true;
|
|
126
|
+
}
|
|
127
|
+
export function buildOutputLimitRecoveryPrompt(input) {
|
|
128
|
+
const paths = input.targetPaths.length > 0
|
|
129
|
+
? input.targetPaths.map((item) => ` - ${item}`).join("\n")
|
|
130
|
+
: " - (none)";
|
|
131
|
+
return [
|
|
132
|
+
"<retry_instruction>",
|
|
133
|
+
`OUTPUT_LIMIT_RECOVERY attempt=${input.attempt}/${input.maxAttempts}`,
|
|
134
|
+
`reason=${input.reason}`,
|
|
135
|
+
"target_paths_only:",
|
|
136
|
+
paths,
|
|
137
|
+
"Rules:",
|
|
138
|
+
"- Continue or repair ONLY listed paths; do not delete or shrink in-scope cases/rules/TPs.",
|
|
139
|
+
"- One file per write/edit; no chat dump of full bodies.",
|
|
140
|
+
"- Prefer edit/append for truncated files; rewrite a file only if unparseable or empty.",
|
|
141
|
+
"- After targets are complete, short IMPLEMENTATION_OUTCOME only.",
|
|
142
|
+
"- Do not mark already-satisfied if any target_path still missing or structurally broken.",
|
|
143
|
+
"- Preserve quality invariants: no scope shrink, no skip/xfail, no secret-shaped samples.",
|
|
144
|
+
"</retry_instruction>",
|
|
145
|
+
].join("\n");
|
|
146
|
+
}
|
|
147
|
+
export async function assessBackendTestMdWriterCompleteness(workspaceRoot) {
|
|
148
|
+
const issues = [];
|
|
149
|
+
const expectedPaths = ["testcase/md/README.md"];
|
|
150
|
+
const actualPaths = [];
|
|
151
|
+
const missingPaths = [];
|
|
152
|
+
const brokenPaths = [];
|
|
153
|
+
const readmePath = path.join(workspaceRoot, "testcase", "md", "README.md");
|
|
154
|
+
let readme = "";
|
|
155
|
+
if (!(await exists(readmePath))) {
|
|
156
|
+
missingPaths.push("testcase/md/README.md");
|
|
157
|
+
issues.push({
|
|
158
|
+
code: "T3",
|
|
159
|
+
path: "testcase/md/README.md",
|
|
160
|
+
detail: "README.md is missing",
|
|
161
|
+
recoverable: true,
|
|
162
|
+
});
|
|
163
|
+
}
|
|
164
|
+
else {
|
|
165
|
+
actualPaths.push("testcase/md/README.md");
|
|
166
|
+
readme = await readFile(readmePath, "utf8");
|
|
167
|
+
if (!readme.includes("## Coverage Scope")) {
|
|
168
|
+
brokenPaths.push("testcase/md/README.md");
|
|
169
|
+
issues.push({
|
|
170
|
+
code: "T5",
|
|
171
|
+
path: "testcase/md/README.md",
|
|
172
|
+
detail: "README.md is missing ## Coverage Scope",
|
|
173
|
+
recoverable: true,
|
|
174
|
+
});
|
|
175
|
+
}
|
|
176
|
+
if (!readme.includes("## Coverage Matrix") ||
|
|
177
|
+
!hasMarkdownTable(readme, "Rule Key")) {
|
|
178
|
+
if (!brokenPaths.includes("testcase/md/README.md")) {
|
|
179
|
+
brokenPaths.push("testcase/md/README.md");
|
|
180
|
+
}
|
|
181
|
+
issues.push({
|
|
182
|
+
code: "T5",
|
|
183
|
+
path: "testcase/md/README.md",
|
|
184
|
+
detail: "README.md Coverage Matrix table is missing or broken",
|
|
185
|
+
recoverable: true,
|
|
186
|
+
});
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
const indexedStems = extractModuleStemsFromReadme(readme);
|
|
190
|
+
const existingModules = await listMarkdownModules(workspaceRoot);
|
|
191
|
+
const existingByStem = new Map(existingModules.map((rel) => [
|
|
192
|
+
normalizeBackendTestModuleStem(path.posix.basename(rel)),
|
|
193
|
+
rel,
|
|
194
|
+
]));
|
|
195
|
+
for (const stem of indexedStems) {
|
|
196
|
+
const rel = `testcase/md/${stem}.md`;
|
|
197
|
+
expectedPaths.push(rel);
|
|
198
|
+
const existing = existingByStem.get(stem);
|
|
199
|
+
if (!existing) {
|
|
200
|
+
missingPaths.push(rel);
|
|
201
|
+
issues.push({
|
|
202
|
+
code: "T3",
|
|
203
|
+
path: rel,
|
|
204
|
+
detail: `module file missing for README index stem ${stem}`,
|
|
205
|
+
recoverable: true,
|
|
206
|
+
});
|
|
207
|
+
continue;
|
|
208
|
+
}
|
|
209
|
+
actualPaths.push(existing);
|
|
210
|
+
const body = await readFile(path.join(workspaceRoot, existing), "utf8");
|
|
211
|
+
if (!moduleStructurallyComplete(body)) {
|
|
212
|
+
brokenPaths.push(existing);
|
|
213
|
+
issues.push({
|
|
214
|
+
code: "T5",
|
|
215
|
+
path: existing,
|
|
216
|
+
detail: "module file is empty or missing required case sections",
|
|
217
|
+
recoverable: true,
|
|
218
|
+
});
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
for (const rel of existingModules) {
|
|
222
|
+
if (!actualPaths.includes(rel))
|
|
223
|
+
actualPaths.push(rel);
|
|
224
|
+
const body = await readFile(path.join(workspaceRoot, rel), "utf8");
|
|
225
|
+
if (!moduleStructurallyComplete(body) && !brokenPaths.includes(rel)) {
|
|
226
|
+
brokenPaths.push(rel);
|
|
227
|
+
issues.push({
|
|
228
|
+
code: "T5",
|
|
229
|
+
path: rel,
|
|
230
|
+
detail: "module file is empty or missing required case sections",
|
|
231
|
+
recoverable: true,
|
|
232
|
+
});
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
if (existingModules.length === 0 &&
|
|
236
|
+
indexedStems.length === 0 &&
|
|
237
|
+
(await exists(readmePath))) {
|
|
238
|
+
issues.push({
|
|
239
|
+
code: "T3",
|
|
240
|
+
detail: "no module Markdown files were produced",
|
|
241
|
+
recoverable: true,
|
|
242
|
+
});
|
|
243
|
+
}
|
|
244
|
+
const targetPaths = orderedUnique([...missingPaths, ...brokenPaths]);
|
|
245
|
+
const nonRecoverable = issues.some((issue) => !issue.recoverable);
|
|
246
|
+
const status = nonRecoverable
|
|
247
|
+
? "NON_RECOVERABLE"
|
|
248
|
+
: targetPaths.length > 0 || issues.length > 0
|
|
249
|
+
? "RECOVERABLE"
|
|
250
|
+
: "PASS";
|
|
251
|
+
return {
|
|
252
|
+
schemaId: BACKEND_TEST_WRITER_PROGRESS_SCHEMA_ID,
|
|
253
|
+
role: "md-generate",
|
|
254
|
+
status,
|
|
255
|
+
expectedPaths: orderedUnique(expectedPaths),
|
|
256
|
+
actualPaths: orderedUnique(actualPaths),
|
|
257
|
+
missingPaths: orderedUnique(missingPaths),
|
|
258
|
+
brokenPaths: orderedUnique(brokenPaths),
|
|
259
|
+
targetPaths,
|
|
260
|
+
issues,
|
|
261
|
+
};
|
|
262
|
+
}
|
|
263
|
+
export async function assessBackendTestPytestWriterCompleteness(workspaceRoot) {
|
|
264
|
+
const issues = [];
|
|
265
|
+
const expectedPaths = [];
|
|
266
|
+
const actualPaths = [];
|
|
267
|
+
const missingPaths = [];
|
|
268
|
+
const brokenPaths = [];
|
|
269
|
+
const modules = await listMarkdownModules(workspaceRoot);
|
|
270
|
+
for (const moduleRel of modules) {
|
|
271
|
+
const expected = expectedBackendTestPytestScriptForMarkdownModule(path.posix.basename(moduleRel));
|
|
272
|
+
expectedPaths.push(expected);
|
|
273
|
+
const absolute = path.join(workspaceRoot, expected);
|
|
274
|
+
if (!(await exists(absolute))) {
|
|
275
|
+
missingPaths.push(expected);
|
|
276
|
+
issues.push({
|
|
277
|
+
code: "T3",
|
|
278
|
+
path: expected,
|
|
279
|
+
detail: `mapped pytest script missing for ${moduleRel}`,
|
|
280
|
+
recoverable: true,
|
|
281
|
+
});
|
|
282
|
+
continue;
|
|
283
|
+
}
|
|
284
|
+
actualPaths.push(expected);
|
|
285
|
+
const source = await readFile(absolute, "utf8");
|
|
286
|
+
if (!pythonParseable(source)) {
|
|
287
|
+
brokenPaths.push(expected);
|
|
288
|
+
issues.push({
|
|
289
|
+
code: "T5",
|
|
290
|
+
path: expected,
|
|
291
|
+
detail: "pytest script appears truncated or unparseable",
|
|
292
|
+
recoverable: true,
|
|
293
|
+
});
|
|
294
|
+
continue;
|
|
295
|
+
}
|
|
296
|
+
const markdown = await readFile(path.join(workspaceRoot, moduleRel), "utf8");
|
|
297
|
+
const caseIds = orderedUnique([...markdown.matchAll(/\bBE-[A-Z0-9_-]+-\d{2,3}\b/g)].map((item) => item[0]));
|
|
298
|
+
const primaryHints = orderedUnique([
|
|
299
|
+
...markdown.matchAll(/primary\s*symbol\s*[::]\s*`?([A-Za-z0-9_]+)`?/gi),
|
|
300
|
+
].map((item) => item[1]));
|
|
301
|
+
for (const symbol of primaryHints) {
|
|
302
|
+
if (!source.includes(symbol)) {
|
|
303
|
+
brokenPaths.push(expected);
|
|
304
|
+
issues.push({
|
|
305
|
+
code: "T5",
|
|
306
|
+
path: expected,
|
|
307
|
+
detail: `primary symbol ${symbol} not found in mapped script`,
|
|
308
|
+
recoverable: true,
|
|
309
|
+
});
|
|
310
|
+
break;
|
|
311
|
+
}
|
|
312
|
+
}
|
|
313
|
+
if (caseIds.length > 0 &&
|
|
314
|
+
!caseIds.some((caseId) => source.includes(caseId) || source.includes(caseId.replaceAll("-", "_")))) {
|
|
315
|
+
brokenPaths.push(expected);
|
|
316
|
+
issues.push({
|
|
317
|
+
code: "T5",
|
|
318
|
+
path: expected,
|
|
319
|
+
detail: "no Markdown Case ID found in mapped pytest script",
|
|
320
|
+
recoverable: true,
|
|
321
|
+
});
|
|
322
|
+
}
|
|
323
|
+
}
|
|
324
|
+
const targetPaths = orderedUnique([...missingPaths, ...brokenPaths]);
|
|
325
|
+
const status = targetPaths.length > 0 || issues.length > 0 ? "RECOVERABLE" : "PASS";
|
|
326
|
+
return {
|
|
327
|
+
schemaId: BACKEND_TEST_WRITER_PROGRESS_SCHEMA_ID,
|
|
328
|
+
role: "pytest-generate",
|
|
329
|
+
status,
|
|
330
|
+
expectedPaths: orderedUnique(expectedPaths),
|
|
331
|
+
actualPaths: orderedUnique(actualPaths),
|
|
332
|
+
missingPaths: orderedUnique(missingPaths),
|
|
333
|
+
brokenPaths: orderedUnique(brokenPaths),
|
|
334
|
+
targetPaths,
|
|
335
|
+
issues,
|
|
336
|
+
};
|
|
337
|
+
}
|
|
338
|
+
export function classifyBackendTestWriterCompletenessFailure(progress) {
|
|
339
|
+
if (progress.status === "PASS") {
|
|
340
|
+
return {
|
|
341
|
+
failureCategory: "invalid-output",
|
|
342
|
+
reason: "T4",
|
|
343
|
+
recoverable: false,
|
|
344
|
+
};
|
|
345
|
+
}
|
|
346
|
+
if (progress.status === "NON_RECOVERABLE") {
|
|
347
|
+
return {
|
|
348
|
+
failureCategory: "invalid-output",
|
|
349
|
+
reason: "T6",
|
|
350
|
+
recoverable: false,
|
|
351
|
+
};
|
|
352
|
+
}
|
|
353
|
+
const preferred = progress.issues.find((issue) => issue.code === "T5")?.code ??
|
|
354
|
+
progress.issues.find((issue) => issue.code === "T3")?.code ??
|
|
355
|
+
progress.issues.find((issue) => issue.code === "T4")?.code ??
|
|
356
|
+
"T3";
|
|
357
|
+
return {
|
|
358
|
+
failureCategory: "incomplete-write-set",
|
|
359
|
+
reason: preferred,
|
|
360
|
+
recoverable: true,
|
|
361
|
+
};
|
|
362
|
+
}
|
|
363
|
+
export async function writeBackendTestWriterProgressArtifacts(input) {
|
|
364
|
+
const contractsDir = path.join(input.runDir, "contracts");
|
|
365
|
+
await mkdir(contractsDir, { recursive: true });
|
|
366
|
+
const factsPath = path.join(contractsDir, `backend-test-writer-progress-${input.progress.role}.json`);
|
|
367
|
+
const parsed = progressSchema.parse(input.progress);
|
|
368
|
+
await writeFile(factsPath, JSON.stringify(parsed, null, 2), "utf8");
|
|
369
|
+
if (input.progress.status === "PASS") {
|
|
370
|
+
return { factsPath };
|
|
371
|
+
}
|
|
372
|
+
const reportsDir = path.join(input.runDir, "reports");
|
|
373
|
+
await mkdir(reportsDir, { recursive: true });
|
|
374
|
+
const reportPath = path.join(reportsDir, BACKEND_TEST_OUTPUT_LIMIT_RECOVERY_REPORT);
|
|
375
|
+
const attempt = input.attempt ?? 1;
|
|
376
|
+
const maxAttempts = input.maxAttempts ?? 3;
|
|
377
|
+
const classification = classifyBackendTestWriterCompletenessFailure(input.progress);
|
|
378
|
+
const body = [
|
|
379
|
+
"# Backend-test Output Limit Recovery",
|
|
380
|
+
"",
|
|
381
|
+
`- Role: ${input.progress.role}`,
|
|
382
|
+
`- Status: ${input.progress.status}`,
|
|
383
|
+
`- Attempt: ${attempt}/${maxAttempts}`,
|
|
384
|
+
`- Reason: ${classification.reason}`,
|
|
385
|
+
`- Recoverable: ${classification.recoverable}`,
|
|
386
|
+
"",
|
|
387
|
+
"## Target paths",
|
|
388
|
+
"",
|
|
389
|
+
...(input.progress.targetPaths.length
|
|
390
|
+
? input.progress.targetPaths.map((item) => `- ${item}`)
|
|
391
|
+
: ["- (none)"]),
|
|
392
|
+
"",
|
|
393
|
+
"## Issues",
|
|
394
|
+
"",
|
|
395
|
+
...(input.progress.issues.length
|
|
396
|
+
? input.progress.issues.map((issue) => `- [${issue.code}] ${issue.path ?? "(workspace)"}: ${issue.detail}`)
|
|
397
|
+
: ["- None"]),
|
|
398
|
+
"",
|
|
399
|
+
`<!-- sha256:${createHash("sha256").update(JSON.stringify(parsed)).digest("hex")} -->`,
|
|
400
|
+
"",
|
|
401
|
+
].join("\n");
|
|
402
|
+
await writeFile(reportPath, body, "utf8");
|
|
403
|
+
return { factsPath, reportPath };
|
|
404
|
+
}
|
|
405
|
+
export function isBackendTestCompletenessRetryCandidate(task) {
|
|
406
|
+
if (task.executor !== "pi" || task.role !== "implementer")
|
|
407
|
+
return false;
|
|
408
|
+
if (task.toolProfile !== "write" || task.writePolicy !== "exclusive") {
|
|
409
|
+
return false;
|
|
410
|
+
}
|
|
411
|
+
if ((task.writeSet?.length ?? 0) === 0)
|
|
412
|
+
return false;
|
|
413
|
+
if (task.writerOutcomePolicy?.type !== "implementation-outcome-v1") {
|
|
414
|
+
return false;
|
|
415
|
+
}
|
|
416
|
+
return (task.id === "generate-backend-md-cases-pi" ||
|
|
417
|
+
task.id === "generate-backend-pytest-pi");
|
|
418
|
+
}
|
|
@@ -6,6 +6,9 @@ const LIST_ITEM = /^\s*(?:[-+*]|\d+[.)])\s+(.+)$/;
|
|
|
6
6
|
const INDENTED_CODE = /^(?: {4,}|\t+)(\S.*)$/;
|
|
7
7
|
const EXPLICIT_COMMAND = /^(?:(?:shell|terminal)(?:\s+command)?|command|run command|execute command)\s*:\s*(.+)$/i;
|
|
8
8
|
const BLOCKED_REASON_PREFIX = /^(?:(?:blocked|forbidden|reject(?:ed)?|disallow(?:ed)?|prohibited)(?:\s+reason)?\s*(?::|\bbecause\b)|(?:do not|must not|never)\s+(?:run|execute|use)\b)/i;
|
|
9
|
+
/** Standalone blockedReason / environment tokens (not executable commands). */
|
|
10
|
+
const BLOCKED_REASON_TOKEN = /^(?:playwright-cli-unavailable|frontend-base-url-unreachable|curl-unavailable|browser-command-capability-unavailable|browser-command-evidence-missing|token-budget-exhausted|current-user-data-unavailable|data-ownership-unverifiable|safe-test-data-setup-unavailable|invalid-evidence-shape)(?:\s|$|[.,;:)`'"\]])/i;
|
|
11
|
+
const BLOCKED_PROSE_HINT = /(?:blockedReason|blocked\s*reason|environmentProbe|evidenceDir|\bunavailable\b|不可用|写\s*blocked|写入\s*blocked|伪命令|自然语言|元数据|fail-closed|探测失败)/i;
|
|
9
12
|
const IMPERATIVE_COMMAND = /^(?:(?:run|execute|use)(?:\s+(?:the\s+)?(?:(?:shell|terminal)\s+)?command)?|in\s+(?:the\s+)?(?:shell|terminal|console)\s*,?\s*(?:run|execute|use))\s*:?\s+(.+)$/i;
|
|
10
13
|
const INLINE_CODE_STEP = /^`([^`\r\n]+)`[.!?]?$/;
|
|
11
14
|
const AUTOMATION_EXECUTABLE = /^(?:playwright-cli\b|playwright\b|@playwright\/test\b|npx\b|npm\b|pnpm\b|yarn\b|bunx?\b|node(?:js)?\b|python(?:3)?\b|bash\b|sh\b|zsh\b|fish\b|powershell\b|pwsh\b|cmd(?:\.exe)?\b|cypress\b|selenium\b|webdriverio\b|chromedriver\b|google-chrome\b|chrome\b|firefox\b|curl\b|wget\b)/i;
|
|
@@ -31,7 +34,53 @@ function isExecutableFence(language) {
|
|
|
31
34
|
return COMMAND_FENCE_LANGUAGE.test(language.trim());
|
|
32
35
|
}
|
|
33
36
|
function isBlockedReason(value) {
|
|
34
|
-
|
|
37
|
+
const trimmed = value.trim();
|
|
38
|
+
if (!trimmed)
|
|
39
|
+
return false;
|
|
40
|
+
if (BLOCKED_REASON_PREFIX.test(trimmed))
|
|
41
|
+
return true;
|
|
42
|
+
// Bare reason enum / metadata line (BugPilot blocked sections).
|
|
43
|
+
if (BLOCKED_REASON_TOKEN.test(trimmed.replace(/^`+|`+$/g, "")))
|
|
44
|
+
return true;
|
|
45
|
+
if (/^blockedReason\s*[:=]/i.test(trimmed))
|
|
46
|
+
return true;
|
|
47
|
+
return false;
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Non-executable prose that mentions playwright-cli / reason tokens for blocked
|
|
51
|
+
* paths (e.g. "playwright-cli 不可用时… blockedReason: …"). Must not enter the
|
|
52
|
+
* command allowlist gate.
|
|
53
|
+
*/
|
|
54
|
+
function isNonExecutableBlockedProse(value) {
|
|
55
|
+
const trimmed = value.trim();
|
|
56
|
+
if (!trimmed)
|
|
57
|
+
return false;
|
|
58
|
+
if (isBlockedReason(trimmed))
|
|
59
|
+
return true;
|
|
60
|
+
const normalized = normalizeCommand(trimmed);
|
|
61
|
+
const bare = normalized.replace(/^`+|`+$/g, "").trim();
|
|
62
|
+
if (BLOCKED_REASON_TOKEN.test(bare))
|
|
63
|
+
return true;
|
|
64
|
+
// "playwright-cli 不可用…" / "playwright-cli unavailable…" documentation.
|
|
65
|
+
if (/^playwright-cli\b/i.test(normalized) &&
|
|
66
|
+
BLOCKED_PROSE_HINT.test(normalized) &&
|
|
67
|
+
!/^playwright-cli\s+[a-z][a-z0-9-]*\b/i.test(normalized)) {
|
|
68
|
+
return true;
|
|
69
|
+
}
|
|
70
|
+
const pw = normalized.match(/^playwright-cli\s+(\S+)([\s\S]*)$/i);
|
|
71
|
+
if (pw) {
|
|
72
|
+
const first = pw[1].replace(/^[`'"]+|[`'":,,。→]+$/g, "");
|
|
73
|
+
if (!isPlaywrightCliCommand(first.toLowerCase())) {
|
|
74
|
+
// Non-verb after playwright-cli in prose (e.g. 不可用 / unavailable).
|
|
75
|
+
return /[\u4e00-\u9fff]/.test(first) || BLOCKED_PROSE_HINT.test(normalized) || !/^[a-z][a-z0-9-]*$/i.test(first);
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
// Reason token embedded without looking like a real CLI invocation.
|
|
79
|
+
if (BLOCKED_PROSE_HINT.test(trimmed) &&
|
|
80
|
+
!/^playwright-cli\s+[a-z][a-z0-9-]*(\s+--|\s+https?:|\s*$)/i.test(normalized)) {
|
|
81
|
+
return true;
|
|
82
|
+
}
|
|
83
|
+
return false;
|
|
35
84
|
}
|
|
36
85
|
function stripLeadingCommandWrappers(command) {
|
|
37
86
|
let remaining = command.trim();
|
|
@@ -64,17 +113,30 @@ function isCommandLikeExecutable(command) {
|
|
|
64
113
|
}
|
|
65
114
|
function executableListStep(value) {
|
|
66
115
|
const trimmed = value.trim();
|
|
116
|
+
if (isNonExecutableBlockedProse(trimmed))
|
|
117
|
+
return null;
|
|
67
118
|
const explicit = trimmed.match(EXPLICIT_COMMAND);
|
|
68
|
-
if (explicit)
|
|
69
|
-
|
|
119
|
+
if (explicit) {
|
|
120
|
+
if (isBlockedReason(explicit[1]) || isNonExecutableBlockedProse(explicit[1]))
|
|
121
|
+
return null;
|
|
122
|
+
return normalizeCommand(explicit[1]);
|
|
123
|
+
}
|
|
70
124
|
const imperative = trimmed.match(IMPERATIVE_COMMAND);
|
|
71
|
-
if (imperative)
|
|
125
|
+
if (imperative) {
|
|
126
|
+
if (isNonExecutableBlockedProse(imperative[1]))
|
|
127
|
+
return null;
|
|
72
128
|
return normalizeCommand(imperative[1]);
|
|
129
|
+
}
|
|
73
130
|
const inlineCode = trimmed.match(INLINE_CODE_STEP);
|
|
74
|
-
if (inlineCode)
|
|
131
|
+
if (inlineCode) {
|
|
132
|
+
if (isNonExecutableBlockedProse(inlineCode[1]))
|
|
133
|
+
return null;
|
|
75
134
|
return normalizeCommand(inlineCode[1]);
|
|
135
|
+
}
|
|
76
136
|
const prompted = /^[$>]\s*\S/.test(trimmed);
|
|
77
137
|
const normalized = normalizeCommand(trimmed);
|
|
138
|
+
if (isNonExecutableBlockedProse(normalized))
|
|
139
|
+
return null;
|
|
78
140
|
if (prompted || isCommandLikeExecutable(normalized))
|
|
79
141
|
return normalized;
|
|
80
142
|
return null;
|
|
@@ -127,12 +189,15 @@ function extractExecutableInstructions(markdown) {
|
|
|
127
189
|
if (fenceLanguage !== null) {
|
|
128
190
|
if (!isExecutableFence(fenceLanguage) || /^(?:#|\/\/)/.test(trimmed))
|
|
129
191
|
continue;
|
|
130
|
-
|
|
192
|
+
// Fenced blocks are strict: only skip pure reason-token / blockedReason lines.
|
|
193
|
+
if (isBlockedReason(trimmed) || BLOCKED_REASON_TOKEN.test(normalizeCommand(trimmed).replace(/^`+|`+$/g, "")))
|
|
194
|
+
continue;
|
|
195
|
+
instructions.push({ command: normalizeCommand(trimmed), lineNumber, fromFence: true });
|
|
131
196
|
continue;
|
|
132
197
|
}
|
|
133
198
|
const listed = line.match(LIST_ITEM);
|
|
134
199
|
if (listed) {
|
|
135
|
-
if (isBlockedReason(listed[1]))
|
|
200
|
+
if (isBlockedReason(listed[1]) || isNonExecutableBlockedProse(listed[1]))
|
|
136
201
|
continue;
|
|
137
202
|
const command = executableListStep(listed[1]);
|
|
138
203
|
if (command !== null)
|
|
@@ -141,17 +206,20 @@ function extractExecutableInstructions(markdown) {
|
|
|
141
206
|
}
|
|
142
207
|
const indented = line.match(INDENTED_CODE);
|
|
143
208
|
if (indented) {
|
|
144
|
-
if (isBlockedReason(indented[1]))
|
|
209
|
+
if (isBlockedReason(indented[1]) || isNonExecutableBlockedProse(indented[1]))
|
|
145
210
|
continue;
|
|
146
211
|
const explicitIndented = indented[1].match(EXPLICIT_COMMAND);
|
|
147
|
-
if (explicitIndented && isBlockedReason(explicitIndented[1]))
|
|
212
|
+
if (explicitIndented && (isBlockedReason(explicitIndented[1]) || isNonExecutableBlockedProse(explicitIndented[1])))
|
|
213
|
+
continue;
|
|
214
|
+
const fromStep = executableListStep(indented[1]);
|
|
215
|
+
if (fromStep === null)
|
|
148
216
|
continue;
|
|
149
|
-
instructions.push({ command:
|
|
217
|
+
instructions.push({ command: fromStep, lineNumber });
|
|
150
218
|
continue;
|
|
151
219
|
}
|
|
152
220
|
const explicit = trimmed.match(EXPLICIT_COMMAND);
|
|
153
221
|
if (explicit) {
|
|
154
|
-
if (isBlockedReason(explicit[1]))
|
|
222
|
+
if (isBlockedReason(explicit[1]) || isNonExecutableBlockedProse(explicit[1]))
|
|
155
223
|
continue;
|
|
156
224
|
instructions.push({ command: normalizeCommand(explicit[1]), lineNumber });
|
|
157
225
|
continue;
|
|
@@ -163,9 +231,16 @@ function extractExecutableInstructions(markdown) {
|
|
|
163
231
|
return instructions;
|
|
164
232
|
}
|
|
165
233
|
function commandGateIssue(input) {
|
|
234
|
+
// Outside fences, blocked/unavailable prose must never become a gate failure.
|
|
235
|
+
if (!input.instruction.fromFence && isNonExecutableBlockedProse(input.instruction.command)) {
|
|
236
|
+
return null;
|
|
237
|
+
}
|
|
238
|
+
if (input.instruction.fromFence && isBlockedReason(input.instruction.command)) {
|
|
239
|
+
return null;
|
|
240
|
+
}
|
|
166
241
|
const commandMatch = input.instruction.command.match(/^playwright-cli\s+([^\s`]+)/i);
|
|
167
242
|
if (commandMatch) {
|
|
168
|
-
const command = commandMatch[1].toLowerCase();
|
|
243
|
+
const command = commandMatch[1].toLowerCase().replace(/^[`'"]+|[`'":,,。→]+$/g, "");
|
|
169
244
|
if (isPlaywrightCliCommand(command) && !hasShellControl(input.instruction.command))
|
|
170
245
|
return null;
|
|
171
246
|
if (isPlaywrightCliCommand(command)) {
|
|
@@ -178,6 +253,10 @@ function commandGateIssue(input) {
|
|
|
178
253
|
detail: "shell control or additional executable fragments are not allowed after playwright-cli commands",
|
|
179
254
|
};
|
|
180
255
|
}
|
|
256
|
+
// Fenced bad verbs stay rejected; unfenced non-verbs already filtered as prose.
|
|
257
|
+
if (!input.instruction.fromFence && isNonExecutableBlockedProse(input.instruction.command)) {
|
|
258
|
+
return null;
|
|
259
|
+
}
|
|
181
260
|
return {
|
|
182
261
|
ruleId: "playwright-cli-command-not-allowed",
|
|
183
262
|
caseId: input.caseId,
|
|
@@ -210,7 +289,7 @@ export async function validateFrontendCaseChecklist(input) {
|
|
|
210
289
|
const issues = [];
|
|
211
290
|
const caseIdRe = /^FE-[A-Za-z0-9][A-Za-z0-9-]*$/;
|
|
212
291
|
const acIdRe = /^AC(?:-[A-Z0-9]+)+$/i;
|
|
213
|
-
const openRe = /playwright-cli\s+open\s+--browser=chrome\s+--
|
|
292
|
+
const openRe = /playwright-cli\s+open\s+--browser=chrome\s+--headless\s+https?:\/\/\S+/i;
|
|
214
293
|
const productionHostRe = /(^|[.-])(prod|production)([.-]|$)/i;
|
|
215
294
|
for (const raw of manifest.cases) {
|
|
216
295
|
const item = raw;
|
|
@@ -231,8 +310,8 @@ export async function validateFrontendCaseChecklist(input) {
|
|
|
231
310
|
issues.push({ ruleId: "case-path-mismatch", caseId: id, casePath, detail: `${casePath} must equal ${expectedPath}` });
|
|
232
311
|
const body = await readFile(absolute, "utf8");
|
|
233
312
|
if (!openRe.test(body))
|
|
234
|
-
issues.push({ ruleId: "open-prefix", caseId: id, casePath, detail: "missing playwright-cli open --browser=chrome --
|
|
235
|
-
const match = body.match(/playwright-cli\s+open\s+--browser=chrome\s+--
|
|
313
|
+
issues.push({ ruleId: "open-prefix", caseId: id, casePath, detail: "missing playwright-cli open --browser=chrome --headless <absolute-url>" });
|
|
314
|
+
const match = body.match(/playwright-cli\s+open\s+--browser=chrome\s+--headless\s+(https?:\/\/\S+)/i);
|
|
236
315
|
if (match) {
|
|
237
316
|
try {
|
|
238
317
|
const url = new URL(match[1].replace(/[)\]},.\"'`]+$/, ""));
|