@tea-agent/loop-agent 0.27.1-beta.2 → 0.28.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +40 -1
- package/dist/application/task-lifecycle/observe.js +5 -0
- package/dist/application/task-lifecycle/plan-transitions.js +7 -2
- package/dist/cli/program.js +1 -1
- package/dist/commands/client-recovery.js +439 -20
- package/dist/commands/init.js +42 -6
- package/dist/executors/dag-pi-executor.js +165 -56
- package/dist/executors/pi-playwright-cli-tool.js +955 -0
- package/dist/executors/pi-sdk-executor.js +56 -0
- package/dist/executors/playwright-cli-launcher.js +63 -0
- package/dist/executors/shell-executor.js +128 -0
- package/dist/shared/playwright-cli-command-policy.js +41 -0
- package/dist/worker/observability/read-model.js +66 -8
- package/dist/worker/observe/static/dag-model.js +85 -13
- package/dist/workflows/dag/dynamic-runtime/loop-until.js +4 -0
- package/dist/workflows/dag/dynamic-runtime/map.js +13 -13
- package/dist/workflows/dag/frontend-implementation-contract.js +6 -124
- package/dist/workflows/dag/frontend-prewrite-gate.js +5 -35
- package/dist/workflows/dag/frontend-test-case-checklist.js +201 -8
- package/dist/workflows/dag/frontend-test-result-contract.js +52 -3
- package/dist/workflows/dag/init-hybrid.js +154 -95
- package/dist/workflows/dag/lifecycle.js +33 -2
- package/dist/workflows/dag/node-execution.js +11 -5
- package/dist/workflows/dag/output-protocol.js +25 -106
- package/dist/workflows/dag/report.js +9 -2
- package/dist/workflows/dag/rerun-run.js +62 -3
- package/dist/workflows/dag/run-store.js +6 -1
- package/dist/workflows/dag/runner.js +15 -3
- package/dist/workflows/dag/types.js +27 -0
- package/dist/workflows/dag/validate.js +121 -1
- package/docs/architecture/runtime-boundaries.md +13 -11
- package/docs/init-surface.manifest.json +6 -2
- package/docs/templates/README.md +9 -1
- package/docs/templates/frontend-implementation-contract.schema.json +2 -2
- package/docs/templates/frontend-test-dag.generate-cases.prompt.md +14 -7
- package/docs/templates/frontend-test-dag.json +55 -15
- package/docs/templates/frontend-test-dag.retrieve-context.prompt.md +7 -9
- package/docs/templates/frontend-test-dag.retrospect.prompt.md +1 -1
- package/docs/templates/frontend-test-dag.review-cases.prompt.md +1 -1
- package/docs/templates/frontend-test-dag.review-execution.prompt.md +1 -1
- package/harness.json +4 -4
- package/package.json +1 -1
- package/skills/loop-agent/SKILL.md +1 -1
- package/skills/loop-agent/references/command-reference.md +18 -6
- package/skills/playwright-cli/SKILL.md +69 -402
- package/skills/playwright-cli/references/tracing.md +3 -137
- package/skills/playwright-cli/references/video-recording.md +3 -141
- package/skills/playwright-cli-case-generator/SKILL.md +53 -46
|
@@ -3,6 +3,7 @@ import path from "node:path";
|
|
|
3
3
|
import { writeDagRunJsonArtifact } from "../../../infrastructure/harness/artifact-store.js";
|
|
4
4
|
import { executeDagNode, } from "../node-execution.js";
|
|
5
5
|
import { writeRunSpec } from "../run-store.js";
|
|
6
|
+
import { assertValidMaterializedDagTask } from "../validate.js";
|
|
6
7
|
import { freshNodeRecord, parseJsonFromText, renderDynamicPatternList, renderDynamicTemplate, resolveItemsFromSelector, sha256Json, } from "./shared.js";
|
|
7
8
|
export function buildExpandedChildTask(input) {
|
|
8
9
|
const { parent, expansion, item, index, nodeId } = input;
|
|
@@ -20,6 +21,7 @@ export function buildExpandedChildTask(input) {
|
|
|
20
21
|
role: child.role,
|
|
21
22
|
skills: child.skills,
|
|
22
23
|
toolProfile: child.toolProfile,
|
|
24
|
+
commandPolicy: child.commandPolicy,
|
|
23
25
|
writePolicy: child.writePolicy,
|
|
24
26
|
allowedPaths: renderDynamicPatternList(child.allowedPaths, item, index, expansion.itemName) ?? [],
|
|
25
27
|
forbiddenPaths: renderDynamicPatternList(child.forbiddenPaths, item, index, expansion.itemName) ?? [],
|
|
@@ -98,14 +100,11 @@ async function materializeBlockedCaseEvidence(input) {
|
|
|
98
100
|
const caseId = resolveCaseIdFromItem(input.item);
|
|
99
101
|
try {
|
|
100
102
|
const existing = JSON.parse(await readFile(resultPath, "utf-8"));
|
|
101
|
-
if (existing.caseId === caseId &&
|
|
102
|
-
(existing.status === "passed" ||
|
|
103
|
-
existing.status === "failed" ||
|
|
104
|
-
existing.status === "blocked"))
|
|
103
|
+
if (existing.caseId === caseId && existing.status === "blocked")
|
|
105
104
|
return;
|
|
106
105
|
}
|
|
107
106
|
catch {
|
|
108
|
-
// Missing or malformed evidence is replaced
|
|
107
|
+
// Missing or malformed evidence is replaced by controller-owned terminal evidence.
|
|
109
108
|
}
|
|
110
109
|
await writeFile(path.join(workspace, "execution.md"), `# ${caseId}\n\nStatus: blocked\n\nReason: ${input.reason}\n`, "utf-8");
|
|
111
110
|
await writeFile(resultPath, `${JSON.stringify({
|
|
@@ -130,14 +129,11 @@ async function materializeFailedCaseEvidence(input) {
|
|
|
130
129
|
const caseId = resolveCaseIdFromItem(input.item);
|
|
131
130
|
try {
|
|
132
131
|
const existing = JSON.parse(await readFile(resultPath, "utf-8"));
|
|
133
|
-
if (existing.caseId === caseId &&
|
|
134
|
-
(existing.status === "passed" ||
|
|
135
|
-
existing.status === "failed" ||
|
|
136
|
-
existing.status === "blocked"))
|
|
132
|
+
if (existing.caseId === caseId && existing.status === "failed")
|
|
137
133
|
return;
|
|
138
134
|
}
|
|
139
135
|
catch {
|
|
140
|
-
// Missing or malformed evidence is replaced
|
|
136
|
+
// Missing or malformed evidence is replaced by controller-owned terminal evidence.
|
|
141
137
|
}
|
|
142
138
|
await writeFile(path.join(workspace, "execution.md"), `# ${caseId}\n\nStatus: failed\n\nReason: ${input.reason}\n`, "utf-8");
|
|
143
139
|
await writeFile(resultPath, `${JSON.stringify({
|
|
@@ -170,6 +166,9 @@ export async function executeDynamicMapExpansion(input) {
|
|
|
170
166
|
index,
|
|
171
167
|
nodeId,
|
|
172
168
|
}));
|
|
169
|
+
for (const child of children) {
|
|
170
|
+
assertValidMaterializedDagTask(child);
|
|
171
|
+
}
|
|
173
172
|
const writeSetConflicts = collectDynamicChildWriteSetConflicts(children);
|
|
174
173
|
if (writeSetConflicts.length > 0) {
|
|
175
174
|
throw new Error(`dynamic map_agent ${input.task.id} expanded overlapping writeSets: ${writeSetConflicts.join("; ")}`);
|
|
@@ -356,9 +355,9 @@ export async function executeDynamicMapExpansion(input) {
|
|
|
356
355
|
throw new Error(`failed to materialize case outcome evidence for ${nodeId}: ${error instanceof Error ? error.message : String(error)}`);
|
|
357
356
|
}
|
|
358
357
|
caseOutcomeNotes.push({ nodeId, reason, status: outcomeStatus });
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
358
|
+
// `tolerateChildFailures` changes only the map barrier's business
|
|
359
|
+
// aggregation. The child terminal state and failure category remain
|
|
360
|
+
// executor-owned facts for downstream authority checks.
|
|
362
361
|
}
|
|
363
362
|
}
|
|
364
363
|
const budgetBlockedIds = new Set(blockedChildren.map((entry) => entry.nodeId));
|
|
@@ -380,6 +379,7 @@ export async function executeDynamicMapExpansion(input) {
|
|
|
380
379
|
item: items[index],
|
|
381
380
|
workspaceRef: workspaceRefs[index],
|
|
382
381
|
status: input.state.nodes[nodeId]?.status,
|
|
382
|
+
failureCategory: input.state.nodes[nodeId]?.failureCategory,
|
|
383
383
|
stdout: input.state.nodes[nodeId]?.stdout,
|
|
384
384
|
output: parseJsonFromText(input.state.nodes[nodeId]?.stdout),
|
|
385
385
|
assistantText: input.state.nodes[nodeId]?.assistantText,
|
|
@@ -155,7 +155,7 @@ export const frontendImplementationContractSchema = z
|
|
|
155
155
|
"not-needed",
|
|
156
156
|
]),
|
|
157
157
|
productionDefaultOff: z.literal(true),
|
|
158
|
-
activation: z.
|
|
158
|
+
activation: z.string().min(1),
|
|
159
159
|
endpoints: z.array(z
|
|
160
160
|
.object({
|
|
161
161
|
method: z.enum([
|
|
@@ -168,10 +168,7 @@ export const frontendImplementationContractSchema = z
|
|
|
168
168
|
"OPTIONS",
|
|
169
169
|
]),
|
|
170
170
|
path: z.string().startsWith("/"),
|
|
171
|
-
|
|
172
|
-
// intentionally not needed. Treat it like an omitted optional
|
|
173
|
-
// field; active Mock strategies still fail the refinement below.
|
|
174
|
-
fixture: z.preprocess((value) => (value === "" || value === null ? undefined : value), safePath.optional()),
|
|
171
|
+
fixture: safePath.optional(),
|
|
175
172
|
consumer: safePath.optional(),
|
|
176
173
|
})
|
|
177
174
|
.strict()),
|
|
@@ -190,7 +187,7 @@ export const frontendImplementationContractSchema = z
|
|
|
190
187
|
type: z.enum(["static", "unit", "component", "integration", "mock"]),
|
|
191
188
|
commandLabel: z.string().min(1),
|
|
192
189
|
file: safePath,
|
|
193
|
-
symbol: z.
|
|
190
|
+
symbol: z.string().min(1).optional(),
|
|
194
191
|
requirementIds: z.array(id),
|
|
195
192
|
uiStates: z.array(z.string().min(1)),
|
|
196
193
|
})
|
|
@@ -346,127 +343,12 @@ function secretIssues(value, at = "$", issues = []) {
|
|
|
346
343
|
}
|
|
347
344
|
export function extractFrontendImplementationJson(text) {
|
|
348
345
|
const trimmed = text.trim();
|
|
349
|
-
const parse = (source) => {
|
|
350
|
-
try {
|
|
351
|
-
return JSON.parse(source);
|
|
352
|
-
}
|
|
353
|
-
catch (error) {
|
|
354
|
-
// Models sometimes put ordinary ASCII quotes inside a JSON string
|
|
355
|
-
// (for example: `reason: "支持..."`). Repair only quotes that are
|
|
356
|
-
// clearly not structural: a closing quote is followed by JSON
|
|
357
|
-
// punctuation, while an embedded quote is followed by content.
|
|
358
|
-
let repaired = "";
|
|
359
|
-
let inString = false;
|
|
360
|
-
let escaped = false;
|
|
361
|
-
for (let index = 0; index < source.length; index += 1) {
|
|
362
|
-
const character = source[index];
|
|
363
|
-
if (character !== '"') {
|
|
364
|
-
repaired += character;
|
|
365
|
-
if (inString && character === "\\" && !escaped)
|
|
366
|
-
escaped = true;
|
|
367
|
-
else
|
|
368
|
-
escaped = false;
|
|
369
|
-
continue;
|
|
370
|
-
}
|
|
371
|
-
if (escaped) {
|
|
372
|
-
repaired += character;
|
|
373
|
-
escaped = false;
|
|
374
|
-
continue;
|
|
375
|
-
}
|
|
376
|
-
if (!inString) {
|
|
377
|
-
inString = true;
|
|
378
|
-
repaired += character;
|
|
379
|
-
continue;
|
|
380
|
-
}
|
|
381
|
-
const next = source.slice(index + 1).trimStart()[0];
|
|
382
|
-
if ([",", "}", "]", ":"].includes(next ?? "") || source.slice(index + 1).trim() === "") {
|
|
383
|
-
inString = false;
|
|
384
|
-
repaired += character;
|
|
385
|
-
}
|
|
386
|
-
else {
|
|
387
|
-
repaired += "\\\"";
|
|
388
|
-
}
|
|
389
|
-
}
|
|
390
|
-
try {
|
|
391
|
-
return JSON.parse(repaired);
|
|
392
|
-
}
|
|
393
|
-
catch {
|
|
394
|
-
throw error;
|
|
395
|
-
}
|
|
396
|
-
}
|
|
397
|
-
};
|
|
398
|
-
const isContract = (candidate) => {
|
|
399
|
-
const record = asRecord(candidate);
|
|
400
|
-
return (record?.schemaVersion === 1 &&
|
|
401
|
-
asRecord(record.targets) !== null &&
|
|
402
|
-
Array.isArray(record.requirements) &&
|
|
403
|
-
Array.isArray(record.verificationTargets));
|
|
404
|
-
};
|
|
405
346
|
if (trimmed.startsWith("{") && trimmed.endsWith("}"))
|
|
406
|
-
return parse(trimmed);
|
|
407
|
-
const balancedObjects = [];
|
|
408
|
-
for (let start = 0; start < trimmed.length; start += 1) {
|
|
409
|
-
if (trimmed[start] !== "{")
|
|
410
|
-
continue;
|
|
411
|
-
let depth = 0;
|
|
412
|
-
let inString = false;
|
|
413
|
-
let escaped = false;
|
|
414
|
-
for (let index = start; index < trimmed.length; index += 1) {
|
|
415
|
-
const character = trimmed[index];
|
|
416
|
-
if (inString) {
|
|
417
|
-
if (escaped)
|
|
418
|
-
escaped = false;
|
|
419
|
-
else if (character === "\\")
|
|
420
|
-
escaped = true;
|
|
421
|
-
else if (character === '"')
|
|
422
|
-
inString = false;
|
|
423
|
-
continue;
|
|
424
|
-
}
|
|
425
|
-
if (character === '"')
|
|
426
|
-
inString = true;
|
|
427
|
-
else if (character === "{")
|
|
428
|
-
depth += 1;
|
|
429
|
-
else if (character === "}" && --depth === 0) {
|
|
430
|
-
try {
|
|
431
|
-
const candidate = parse(trimmed.slice(start, index + 1));
|
|
432
|
-
if (candidate && typeof candidate === "object" && !Array.isArray(candidate))
|
|
433
|
-
balancedObjects.push(candidate);
|
|
434
|
-
}
|
|
435
|
-
catch {
|
|
436
|
-
// Continue scanning for a later complete JSON object.
|
|
437
|
-
}
|
|
438
|
-
break;
|
|
439
|
-
}
|
|
440
|
-
}
|
|
441
|
-
}
|
|
442
|
-
const balancedContracts = balancedObjects.filter(isContract);
|
|
443
|
-
if (balancedContracts.length === 1)
|
|
444
|
-
return balancedContracts[0];
|
|
445
|
-
if (balancedObjects.length === 1)
|
|
446
|
-
return balancedObjects[0];
|
|
347
|
+
return JSON.parse(trimmed);
|
|
447
348
|
const blocks = [...trimmed.matchAll(/```json\s*\n([\s\S]*?)\n```/gi)];
|
|
448
|
-
if (blocks.length
|
|
349
|
+
if (blocks.length !== 1)
|
|
449
350
|
throw new Error("output must contain exactly one fenced json object");
|
|
450
|
-
|
|
451
|
-
for (const block of blocks) {
|
|
452
|
-
try {
|
|
453
|
-
const candidate = parse(block[1]);
|
|
454
|
-
if (candidate && typeof candidate === "object" && !Array.isArray(candidate))
|
|
455
|
-
candidates.push(candidate);
|
|
456
|
-
}
|
|
457
|
-
catch {
|
|
458
|
-
// Ignore incomplete model scratch blocks. A later complete contract
|
|
459
|
-
// block may still be deterministically recoverable.
|
|
460
|
-
}
|
|
461
|
-
}
|
|
462
|
-
const contractCandidates = candidates.filter(isContract);
|
|
463
|
-
if (contractCandidates.length === 1)
|
|
464
|
-
return contractCandidates[0];
|
|
465
|
-
if (candidates.length === 1)
|
|
466
|
-
return candidates[0];
|
|
467
|
-
if (candidates.length === 0)
|
|
468
|
-
throw new Error("output must contain exactly one valid fenced json object (found 0)");
|
|
469
|
-
throw new Error(`output must contain exactly one valid frontend contract json object (found ${contractCandidates.length || candidates.length})`);
|
|
351
|
+
return JSON.parse(blocks[0][1]);
|
|
470
352
|
}
|
|
471
353
|
/**
|
|
472
354
|
* Build the authoritative frontend-implementation-contract sourceBinding from
|
|
@@ -1,8 +1,7 @@
|
|
|
1
1
|
import { readFile, stat } from "node:fs/promises";
|
|
2
|
-
import { createHash } from "node:crypto";
|
|
3
2
|
import path from "node:path";
|
|
4
3
|
import { isOpenspecSpecFilePath } from "../../shared/openspec-spec.js";
|
|
5
|
-
import { frontendImplementationContractSchema,
|
|
4
|
+
import { frontendImplementationContractSchema, materializeFrontendImplementationContract, assertFrontendSourceBindingFresh, } from "./frontend-implementation-contract.js";
|
|
6
5
|
import { captureFrontendWorktreeBaseline } from "./frontend-worktree-diff.js";
|
|
7
6
|
import { pathMatchesPattern } from "../../shared/git-progress.js";
|
|
8
7
|
async function selectNode(runDir, primary, fallbacks) {
|
|
@@ -47,12 +46,7 @@ function firstNonEmptyVerdictLine(text) {
|
|
|
47
46
|
// The prompt requires VERDICT to be the first non-empty line, but model
|
|
48
47
|
// output can still prepend a summary. Keep the protocol strict in the
|
|
49
48
|
// prompt while making the deterministic gate resilient to that drift.
|
|
50
|
-
|
|
51
|
-
if (/^VERDICT:\s*pass(?:\s*[.!?。!?]|\s+|$)/i.test(verdictLine))
|
|
52
|
-
return "VERDICT: pass";
|
|
53
|
-
if (/^VERDICT:\s*request-revision(?:\s*[.!?。!?]|\s+|$)/i.test(verdictLine))
|
|
54
|
-
return "VERDICT: request-revision";
|
|
55
|
-
return verdictLine;
|
|
49
|
+
return normalizedLines.find((line) => /^VERDICT:/.test(line)) ?? first;
|
|
56
50
|
}
|
|
57
51
|
function eventArgs(event) {
|
|
58
52
|
return event.args ?? event.toolInput ?? event.input ?? {};
|
|
@@ -128,23 +122,6 @@ async function checkOpenspecReadEvidence(input) {
|
|
|
128
122
|
}
|
|
129
123
|
return [...matched];
|
|
130
124
|
}
|
|
131
|
-
async function existingOpenspecCandidates(candidates, repoRoot) {
|
|
132
|
-
const existing = [];
|
|
133
|
-
for (const candidate of candidates) {
|
|
134
|
-
if (!isOpenspecSpecFilePath(candidate))
|
|
135
|
-
continue;
|
|
136
|
-
try {
|
|
137
|
-
const info = await stat(path.resolve(repoRoot, candidate));
|
|
138
|
-
if (info.isFile())
|
|
139
|
-
existing.push(candidate);
|
|
140
|
-
}
|
|
141
|
-
catch {
|
|
142
|
-
// Model-generated typos and stale capability paths are not readable
|
|
143
|
-
// evidence candidates and must not create a false prewrite block.
|
|
144
|
-
}
|
|
145
|
-
}
|
|
146
|
-
return existing;
|
|
147
|
-
}
|
|
148
125
|
export async function runFrontendPrewriteGate(input) {
|
|
149
126
|
const workspaceRoot = input.workspaceRoot ?? input.repoRoot;
|
|
150
127
|
if (workspaceRoot) {
|
|
@@ -178,20 +155,13 @@ export async function runFrontendPrewriteGate(input) {
|
|
|
178
155
|
if (missingIds.length > 0) {
|
|
179
156
|
throw new Error(`frontend prewrite gate missing requirement ids: ${missingIds.join(", ")}`);
|
|
180
157
|
}
|
|
181
|
-
const
|
|
182
|
-
const artifact = await stat(artifactPath)
|
|
183
|
-
.then(async () => ({
|
|
184
|
-
path: artifactPath,
|
|
185
|
-
schemaId: FRONTEND_IMPLEMENTATION_CONTRACT_SCHEMA_ID,
|
|
186
|
-
sha256: createHash("sha256").update(await readFile(artifactPath)).digest("hex"),
|
|
187
|
-
}))
|
|
188
|
-
.catch(() => materializeFrontendImplementationContract({
|
|
158
|
+
const artifact = await materializeFrontendImplementationContract({
|
|
189
159
|
runDir: input.runDir,
|
|
190
160
|
fromNodeId: planNodeId,
|
|
191
161
|
artifactName: input.config.artifactName,
|
|
192
162
|
outputDir: input.config.outputDir,
|
|
193
163
|
sourceBinding: input.sourceBinding,
|
|
194
|
-
})
|
|
164
|
+
});
|
|
195
165
|
const raw = JSON.parse(await readFile(artifact.path, "utf8"));
|
|
196
166
|
const contract = frontendImplementationContractSchema.parse(raw);
|
|
197
167
|
if (!input.config.allowedMockStrategies.includes(contract.mockApi.strategy)) {
|
|
@@ -205,7 +175,7 @@ export async function runFrontendPrewriteGate(input) {
|
|
|
205
175
|
throw new Error(`frontend prewrite gate contract target is outside implementation writeSet: ${uncoveredTargets.join(", ")}`);
|
|
206
176
|
}
|
|
207
177
|
}
|
|
208
|
-
const candidatePaths =
|
|
178
|
+
const candidatePaths = input.config.openspecCandidatePaths ?? [];
|
|
209
179
|
const openspecReadPaths = await checkOpenspecReadEvidence({
|
|
210
180
|
runDir: input.runDir,
|
|
211
181
|
candidatePaths,
|
|
@@ -1,5 +1,19 @@
|
|
|
1
1
|
import { readFile, stat } from "node:fs/promises";
|
|
2
2
|
import path from "node:path";
|
|
3
|
+
import { isPlaywrightCliCommand } from "../../shared/playwright-cli-command-policy.js";
|
|
4
|
+
const COMMAND_FENCE_LANGUAGE = /^(?:|bash|sh|shell|zsh|fish|console|terminal|command|cmd|powershell|pwsh|javascript|js|typescript|ts|python|py)$/i;
|
|
5
|
+
const LIST_ITEM = /^\s*(?:[-+*]|\d+[.)])\s+(.+)$/;
|
|
6
|
+
const INDENTED_CODE = /^(?: {4,}|\t+)(\S.*)$/;
|
|
7
|
+
const EXPLICIT_COMMAND = /^(?:(?:shell|terminal)(?:\s+command)?|command|run command|execute command)\s*:\s*(.+)$/i;
|
|
8
|
+
const BLOCKED_REASON_PREFIX = /^(?:(?:blocked|forbidden|reject(?:ed)?|disallow(?:ed)?|prohibited)(?:\s+reason)?\s*(?::|\bbecause\b)|(?:do not|must not|never)\s+(?:run|execute|use)\b)/i;
|
|
9
|
+
const IMPERATIVE_COMMAND = /^(?:(?:run|execute|use)(?:\s+(?:the\s+)?(?:(?:shell|terminal)\s+)?command)?|in\s+(?:the\s+)?(?:shell|terminal|console)\s*,?\s*(?:run|execute|use))\s*:?\s+(.+)$/i;
|
|
10
|
+
const INLINE_CODE_STEP = /^`([^`\r\n]+)`[.!?]?$/;
|
|
11
|
+
const AUTOMATION_EXECUTABLE = /^(?:playwright-cli\b|playwright\b|@playwright\/test\b|npx\b|npm\b|pnpm\b|yarn\b|bunx?\b|node(?:js)?\b|python(?:3)?\b|bash\b|sh\b|zsh\b|fish\b|powershell\b|pwsh\b|cmd(?:\.exe)?\b|cypress\b|selenium\b|webdriverio\b|chromedriver\b|google-chrome\b|chrome\b|firefox\b|curl\b|wget\b)/i;
|
|
12
|
+
const SHELL_WRAPPED_EXECUTABLE = /^(?:sudo\s+|env(?:\s+[A-Za-z_][A-Za-z0-9_]*=[^\s]+)*\s+|(?:\.{1,2}[\\/]|~[\\/]|[A-Za-z]:[\\/]|\/)[^\s]+)/i;
|
|
13
|
+
const ENV_ASSIGNMENT = /^[A-Za-z_][A-Za-z0-9_]*=(?:"[^"]*"|'[^']*'|[^\s]*)/;
|
|
14
|
+
const ENV_WRAPPER = /^env(?:\s+(?:-[A-Za-z]+|--[A-Za-z][A-Za-z-]*))*\s+/i;
|
|
15
|
+
const SHELL_BUILTIN_WRAPPER = /^(?:command|exec|sudo)\s+/i;
|
|
16
|
+
const PLAYWRIGHT_API = /^(?:from\s+playwright(?:\.[A-Za-z_][A-Za-z0-9_]*)*\s+import\b|import\s+playwright(?:\.[A-Za-z_][A-Za-z0-9_]*)*(?:\s+as\s+[A-Za-z_][A-Za-z0-9_]*)?\b|import\s+.+\s+from\s+["'](?:playwright|@playwright\/test)["']|(?:const|let|var)\s+.+\brequire\s*\(\s*["'](?:playwright|@playwright\/test)["']\s*\)|require\s*\(\s*["'](?:playwright|@playwright\/test)["']\s*\)|(?:sync_playwright|async_playwright|chromium|firefox|webkit)\.(?:launch|connect)\b)/i;
|
|
3
17
|
async function exists(file) {
|
|
4
18
|
try {
|
|
5
19
|
return (await stat(file)).isFile();
|
|
@@ -8,6 +22,180 @@ async function exists(file) {
|
|
|
8
22
|
return false;
|
|
9
23
|
}
|
|
10
24
|
}
|
|
25
|
+
function normalizeCommand(value) {
|
|
26
|
+
const trimmed = value.trim().replace(/^[$>]\s*/, "");
|
|
27
|
+
const wrapped = trimmed.match(/^`([^`]+)`$/);
|
|
28
|
+
return (wrapped?.[1] ?? trimmed).trim();
|
|
29
|
+
}
|
|
30
|
+
function isExecutableFence(language) {
|
|
31
|
+
return COMMAND_FENCE_LANGUAGE.test(language.trim());
|
|
32
|
+
}
|
|
33
|
+
function isBlockedReason(value) {
|
|
34
|
+
return BLOCKED_REASON_PREFIX.test(value.trim());
|
|
35
|
+
}
|
|
36
|
+
function stripLeadingCommandWrappers(command) {
|
|
37
|
+
let remaining = command.trim();
|
|
38
|
+
let consumed = true;
|
|
39
|
+
while (remaining && consumed) {
|
|
40
|
+
consumed = false;
|
|
41
|
+
const assignment = remaining.match(ENV_ASSIGNMENT);
|
|
42
|
+
if (assignment) {
|
|
43
|
+
remaining = remaining.slice(assignment[0].length).trimStart();
|
|
44
|
+
consumed = true;
|
|
45
|
+
continue;
|
|
46
|
+
}
|
|
47
|
+
const env = remaining.match(ENV_WRAPPER);
|
|
48
|
+
if (env) {
|
|
49
|
+
remaining = remaining.slice(env[0].length).trimStart();
|
|
50
|
+
consumed = true;
|
|
51
|
+
continue;
|
|
52
|
+
}
|
|
53
|
+
const wrapper = remaining.match(SHELL_BUILTIN_WRAPPER);
|
|
54
|
+
if (wrapper) {
|
|
55
|
+
remaining = remaining.slice(wrapper[0].length).trimStart();
|
|
56
|
+
consumed = true;
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
return remaining;
|
|
60
|
+
}
|
|
61
|
+
function isCommandLikeExecutable(command) {
|
|
62
|
+
const executable = stripLeadingCommandWrappers(command);
|
|
63
|
+
return AUTOMATION_EXECUTABLE.test(executable) || SHELL_WRAPPED_EXECUTABLE.test(executable) || PLAYWRIGHT_API.test(executable);
|
|
64
|
+
}
|
|
65
|
+
function executableListStep(value) {
|
|
66
|
+
const trimmed = value.trim();
|
|
67
|
+
const explicit = trimmed.match(EXPLICIT_COMMAND);
|
|
68
|
+
if (explicit)
|
|
69
|
+
return isBlockedReason(explicit[1]) ? null : normalizeCommand(explicit[1]);
|
|
70
|
+
const imperative = trimmed.match(IMPERATIVE_COMMAND);
|
|
71
|
+
if (imperative)
|
|
72
|
+
return normalizeCommand(imperative[1]);
|
|
73
|
+
const inlineCode = trimmed.match(INLINE_CODE_STEP);
|
|
74
|
+
if (inlineCode)
|
|
75
|
+
return normalizeCommand(inlineCode[1]);
|
|
76
|
+
const prompted = /^[$>]\s*\S/.test(trimmed);
|
|
77
|
+
const normalized = normalizeCommand(trimmed);
|
|
78
|
+
if (prompted || isCommandLikeExecutable(normalized))
|
|
79
|
+
return normalized;
|
|
80
|
+
return null;
|
|
81
|
+
}
|
|
82
|
+
function hasShellControl(command) {
|
|
83
|
+
let quote = null;
|
|
84
|
+
for (let index = 0; index < command.length; index += 1) {
|
|
85
|
+
const character = command[index];
|
|
86
|
+
if (character === "\\" && quote !== "'") {
|
|
87
|
+
index += 1;
|
|
88
|
+
continue;
|
|
89
|
+
}
|
|
90
|
+
if (quote !== null) {
|
|
91
|
+
if (character === quote)
|
|
92
|
+
quote = null;
|
|
93
|
+
else if (quote === '"' && (character === "`" || (character === "$" && command[index + 1] === "(")))
|
|
94
|
+
return true;
|
|
95
|
+
continue;
|
|
96
|
+
}
|
|
97
|
+
if (character === "'" || character === '"') {
|
|
98
|
+
quote = character;
|
|
99
|
+
continue;
|
|
100
|
+
}
|
|
101
|
+
if (character === "#" && (index === 0 || /\s/.test(command[index - 1])))
|
|
102
|
+
break;
|
|
103
|
+
if (character === "`" || character === ";" || character === "|" || character === "&" || character === "<" || character === ">")
|
|
104
|
+
return true;
|
|
105
|
+
if (character === "$" && command[index + 1] === "(")
|
|
106
|
+
return true;
|
|
107
|
+
}
|
|
108
|
+
return false;
|
|
109
|
+
}
|
|
110
|
+
/**
|
|
111
|
+
* Return only Markdown lines that are presented as commands. Prose can mention
|
|
112
|
+
* forbidden commands as a blocked reason without becoming an executable step.
|
|
113
|
+
*/
|
|
114
|
+
function extractExecutableInstructions(markdown) {
|
|
115
|
+
const instructions = [];
|
|
116
|
+
let fenceLanguage = null;
|
|
117
|
+
for (const [index, line] of markdown.split(/\r?\n/).entries()) {
|
|
118
|
+
const lineNumber = index + 1;
|
|
119
|
+
const fence = line.match(/^\s*```([^`]*)$/);
|
|
120
|
+
if (fence) {
|
|
121
|
+
fenceLanguage = fenceLanguage === null ? fence[1].trim() : null;
|
|
122
|
+
continue;
|
|
123
|
+
}
|
|
124
|
+
const trimmed = line.trim();
|
|
125
|
+
if (!trimmed)
|
|
126
|
+
continue;
|
|
127
|
+
if (fenceLanguage !== null) {
|
|
128
|
+
if (!isExecutableFence(fenceLanguage) || /^(?:#|\/\/)/.test(trimmed))
|
|
129
|
+
continue;
|
|
130
|
+
instructions.push({ command: normalizeCommand(trimmed), lineNumber });
|
|
131
|
+
continue;
|
|
132
|
+
}
|
|
133
|
+
const listed = line.match(LIST_ITEM);
|
|
134
|
+
if (listed) {
|
|
135
|
+
if (isBlockedReason(listed[1]))
|
|
136
|
+
continue;
|
|
137
|
+
const command = executableListStep(listed[1]);
|
|
138
|
+
if (command !== null)
|
|
139
|
+
instructions.push({ command, lineNumber });
|
|
140
|
+
continue;
|
|
141
|
+
}
|
|
142
|
+
const indented = line.match(INDENTED_CODE);
|
|
143
|
+
if (indented) {
|
|
144
|
+
if (isBlockedReason(indented[1]))
|
|
145
|
+
continue;
|
|
146
|
+
const explicitIndented = indented[1].match(EXPLICIT_COMMAND);
|
|
147
|
+
if (explicitIndented && isBlockedReason(explicitIndented[1]))
|
|
148
|
+
continue;
|
|
149
|
+
instructions.push({ command: executableListStep(indented[1]) ?? normalizeCommand(indented[1]), lineNumber });
|
|
150
|
+
continue;
|
|
151
|
+
}
|
|
152
|
+
const explicit = trimmed.match(EXPLICIT_COMMAND);
|
|
153
|
+
if (explicit) {
|
|
154
|
+
if (isBlockedReason(explicit[1]))
|
|
155
|
+
continue;
|
|
156
|
+
instructions.push({ command: normalizeCommand(explicit[1]), lineNumber });
|
|
157
|
+
continue;
|
|
158
|
+
}
|
|
159
|
+
const command = executableListStep(trimmed);
|
|
160
|
+
if (command !== null)
|
|
161
|
+
instructions.push({ command, lineNumber });
|
|
162
|
+
}
|
|
163
|
+
return instructions;
|
|
164
|
+
}
|
|
165
|
+
function commandGateIssue(input) {
|
|
166
|
+
const commandMatch = input.instruction.command.match(/^playwright-cli\s+([^\s`]+)/i);
|
|
167
|
+
if (commandMatch) {
|
|
168
|
+
const command = commandMatch[1].toLowerCase();
|
|
169
|
+
if (isPlaywrightCliCommand(command) && !hasShellControl(input.instruction.command))
|
|
170
|
+
return null;
|
|
171
|
+
if (isPlaywrightCliCommand(command)) {
|
|
172
|
+
return {
|
|
173
|
+
ruleId: "alternative-executable-command",
|
|
174
|
+
caseId: input.caseId,
|
|
175
|
+
casePath: input.casePath,
|
|
176
|
+
lineNumber: input.instruction.lineNumber,
|
|
177
|
+
command: input.instruction.command,
|
|
178
|
+
detail: "shell control or additional executable fragments are not allowed after playwright-cli commands",
|
|
179
|
+
};
|
|
180
|
+
}
|
|
181
|
+
return {
|
|
182
|
+
ruleId: "playwright-cli-command-not-allowed",
|
|
183
|
+
caseId: input.caseId,
|
|
184
|
+
casePath: input.casePath,
|
|
185
|
+
lineNumber: input.instruction.lineNumber,
|
|
186
|
+
command: input.instruction.command,
|
|
187
|
+
detail: `playwright-cli command is not allowlisted: ${command}`,
|
|
188
|
+
};
|
|
189
|
+
}
|
|
190
|
+
return {
|
|
191
|
+
ruleId: "alternative-executable-command",
|
|
192
|
+
caseId: input.caseId,
|
|
193
|
+
casePath: input.casePath,
|
|
194
|
+
lineNumber: input.instruction.lineNumber,
|
|
195
|
+
command: input.instruction.command,
|
|
196
|
+
detail: "only allowlisted playwright-cli commands may appear in executable case instructions",
|
|
197
|
+
};
|
|
198
|
+
}
|
|
11
199
|
export async function validateFrontendCaseChecklist(input) {
|
|
12
200
|
const root = path.join(input.workspaceRoot, "testcase/frontend/cases");
|
|
13
201
|
const draft = path.join(root, "manifest.draft.json");
|
|
@@ -36,34 +224,39 @@ export async function validateFrontendCaseChecklist(input) {
|
|
|
36
224
|
const absolute = path.resolve(input.workspaceRoot, casePath);
|
|
37
225
|
const relative = path.relative(input.workspaceRoot, absolute);
|
|
38
226
|
if (!casePath || relative.startsWith("..") || path.isAbsolute(relative) || !(await exists(absolute))) {
|
|
39
|
-
issues.push({ ruleId: "case-file-missing", caseId: id, detail: casePath || "missing casePath" });
|
|
227
|
+
issues.push({ ruleId: "case-file-missing", caseId: id, casePath: casePath || undefined, detail: casePath || "missing casePath" });
|
|
40
228
|
continue;
|
|
41
229
|
}
|
|
42
230
|
if (expectedPath && casePath.replaceAll("\\", "/") !== expectedPath)
|
|
43
|
-
issues.push({ ruleId: "case-path-mismatch", caseId: id, detail: `${casePath} must equal ${expectedPath}` });
|
|
231
|
+
issues.push({ ruleId: "case-path-mismatch", caseId: id, casePath, detail: `${casePath} must equal ${expectedPath}` });
|
|
44
232
|
const body = await readFile(absolute, "utf8");
|
|
45
233
|
if (!openRe.test(body))
|
|
46
|
-
issues.push({ ruleId: "open-prefix", caseId: id, detail: "missing playwright-cli open --browser=chrome --headed <absolute-url>" });
|
|
234
|
+
issues.push({ ruleId: "open-prefix", caseId: id, casePath, detail: "missing playwright-cli open --browser=chrome --headed <absolute-url>" });
|
|
47
235
|
const match = body.match(/playwright-cli\s+open\s+--browser=chrome\s+--headed\s+(https?:\/\/\S+)/i);
|
|
48
236
|
if (match) {
|
|
49
237
|
try {
|
|
50
238
|
const url = new URL(match[1].replace(/[)\]},.\"'`]+$/, ""));
|
|
51
239
|
if (productionHostRe.test(url.hostname))
|
|
52
|
-
issues.push({ ruleId: "production-url", caseId: id, detail: match[1] });
|
|
240
|
+
issues.push({ ruleId: "production-url", caseId: id, casePath, detail: match[1] });
|
|
53
241
|
}
|
|
54
242
|
catch {
|
|
55
|
-
issues.push({ ruleId: "production-url", caseId: id, detail: match[1] });
|
|
243
|
+
issues.push({ ruleId: "production-url", caseId: id, casePath, detail: match[1] });
|
|
56
244
|
}
|
|
57
245
|
}
|
|
246
|
+
for (const instruction of extractExecutableInstructions(body)) {
|
|
247
|
+
const issue = commandGateIssue({ caseId: id, casePath, instruction });
|
|
248
|
+
if (issue)
|
|
249
|
+
issues.push(issue);
|
|
250
|
+
}
|
|
58
251
|
if (!Array.isArray(item?.acIds) || item.acIds.length === 0) {
|
|
59
|
-
issues.push({ ruleId: "ac-mapping", caseId: id, detail: "acIds required" });
|
|
252
|
+
issues.push({ ruleId: "ac-mapping", caseId: id, casePath, detail: "acIds required" });
|
|
60
253
|
}
|
|
61
254
|
else {
|
|
62
255
|
for (const ac of item.acIds) {
|
|
63
256
|
if (typeof ac !== "string" || !acIdRe.test(ac))
|
|
64
|
-
issues.push({ ruleId: "ac-id-shape", caseId: id, detail: String(ac) });
|
|
257
|
+
issues.push({ ruleId: "ac-id-shape", caseId: id, casePath, detail: String(ac) });
|
|
65
258
|
else if (declaredAc.size > 0 && !declaredAc.has(ac))
|
|
66
|
-
issues.push({ ruleId: "unknown-ac", caseId: id, detail: `${ac} not in sourceBinding` });
|
|
259
|
+
issues.push({ ruleId: "unknown-ac", caseId: id, casePath, detail: `${ac} not in sourceBinding` });
|
|
67
260
|
}
|
|
68
261
|
}
|
|
69
262
|
}
|
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
import { createHash } from "node:crypto";
|
|
2
|
-
import { lstat, readFile, realpath, stat } from "node:fs/promises";
|
|
2
|
+
import { lstat, readdir, readFile, realpath, stat } from "node:fs/promises";
|
|
3
3
|
import path from "node:path";
|
|
4
4
|
import { z } from "zod";
|
|
5
5
|
import { writeDagRunJsonArtifact } from "../../infrastructure/harness/artifact-store.js";
|
|
6
|
+
import { readPlaywrightCliReceipts, summarizePlaywrightCliReceipts, } from "../../executors/pi-playwright-cli-tool.js";
|
|
6
7
|
import { validateFrontendCaseContent } from "./frontend-test-case-quality.js";
|
|
7
8
|
export const FRONTEND_TEST_RESULT_SCHEMA_ID = "frontend-test-result-v1";
|
|
8
9
|
const safeRelativePathSchema = z.string().min(1).refine((value) => !path.posix.isAbsolute(value) &&
|
|
@@ -180,6 +181,38 @@ export async function validateFrontendCaseEvidence(input) {
|
|
|
180
181
|
}
|
|
181
182
|
return { cases: manifest.cases.length, issues, hardFail };
|
|
182
183
|
}
|
|
184
|
+
export function evaluatePassedCaseBrowserReceipts(summary, caseId) {
|
|
185
|
+
if (!summary || summary.caseIds.size !== 1 || !summary.caseIds.has(caseId)) {
|
|
186
|
+
return { ok: false, reason: "browser-command-evidence-missing" };
|
|
187
|
+
}
|
|
188
|
+
if (!summary.hasOrderedPassedReceiptChain) {
|
|
189
|
+
return { ok: false, reason: "browser-command-evidence-missing" };
|
|
190
|
+
}
|
|
191
|
+
return { ok: true };
|
|
192
|
+
}
|
|
193
|
+
async function loadCaseBrowserReceiptSummary(input) {
|
|
194
|
+
try {
|
|
195
|
+
const entries = await readdir(input.runDir, { withFileTypes: true });
|
|
196
|
+
// A passed chain must belong to exactly one browser child stream. Never
|
|
197
|
+
// stitch receipts across DAG nodes or accept ambiguous duplicate authority.
|
|
198
|
+
const authorizingStreams = [];
|
|
199
|
+
for (const entry of entries) {
|
|
200
|
+
if (!entry.isDirectory())
|
|
201
|
+
continue;
|
|
202
|
+
const nodeReceipts = (await readPlaywrightCliReceipts(input.runDir, entry.name))
|
|
203
|
+
.filter((item) => item.caseId === input.caseId);
|
|
204
|
+
const summary = summarizePlaywrightCliReceipts(nodeReceipts);
|
|
205
|
+
if (summary.hasOrderedPassedReceiptChain)
|
|
206
|
+
authorizingStreams.push(summary);
|
|
207
|
+
}
|
|
208
|
+
return authorizingStreams.length === 1
|
|
209
|
+
? authorizingStreams[0]
|
|
210
|
+
: summarizePlaywrightCliReceipts([]);
|
|
211
|
+
}
|
|
212
|
+
catch {
|
|
213
|
+
return summarizePlaywrightCliReceipts([]);
|
|
214
|
+
}
|
|
215
|
+
}
|
|
183
216
|
function sha256(content) {
|
|
184
217
|
return createHash("sha256").update(content).digest("hex");
|
|
185
218
|
}
|
|
@@ -296,7 +329,7 @@ export async function materializeFrontendTestResult(input) {
|
|
|
296
329
|
advisoryFindings.push({ ruleId: "missing-or-invalid-case-result", caseId: item.caseId, detail: "case-result.json is missing or invalid" });
|
|
297
330
|
}
|
|
298
331
|
const parsedStatus = caseStatusSchema.safeParse(resultRaw.status);
|
|
299
|
-
|
|
332
|
+
let status = parsedStatus.success ? parsedStatus.data : "blocked";
|
|
300
333
|
if (resultRaw.caseId !== undefined && resultRaw.caseId !== item.caseId)
|
|
301
334
|
advisoryFindings.push({ ruleId: "case-result-identity", caseId: item.caseId, detail: "case-result caseId does not match manifest" });
|
|
302
335
|
if (status === "blocked" && resultRaw.blockedReason !== undefined && (typeof resultRaw.blockedReason !== "string" || !resultRaw.blockedReason.trim()))
|
|
@@ -318,8 +351,24 @@ export async function materializeFrontendTestResult(input) {
|
|
|
318
351
|
}
|
|
319
352
|
if (status === "passed" && !evidence.some((entry) => /\.(png|jpg|jpeg|webp|zip|har|webm|mp4|ya?ml|md)$/i.test(entry.path)))
|
|
320
353
|
advisoryFindings.push({ ruleId: "passed-without-browser-evidence", caseId: item.caseId, detail: "passed case has no browser evidence" });
|
|
354
|
+
let blockedReason = status === "blocked" && typeof resultRaw.blockedReason === "string" && resultRaw.blockedReason.trim() ? resultRaw.blockedReason.trim() : undefined;
|
|
355
|
+
if (status === "passed") {
|
|
356
|
+
const receiptSummary = await loadCaseBrowserReceiptSummary({
|
|
357
|
+
runDir: input.runDir,
|
|
358
|
+
caseId: item.caseId,
|
|
359
|
+
});
|
|
360
|
+
const receiptGate = evaluatePassedCaseBrowserReceipts(receiptSummary, item.caseId);
|
|
361
|
+
if (!receiptGate.ok) {
|
|
362
|
+
status = "blocked";
|
|
363
|
+
blockedReason = receiptGate.reason ?? "browser-command-evidence-missing";
|
|
364
|
+
advisoryFindings.push({
|
|
365
|
+
ruleId: "browser-command-evidence-missing",
|
|
366
|
+
caseId: item.caseId,
|
|
367
|
+
detail: "passed case lacks controller-owned open + interaction/assertion + cleanup receipts",
|
|
368
|
+
});
|
|
369
|
+
}
|
|
370
|
+
}
|
|
321
371
|
const caseContent = await readFrontendCaseContent(input.workspaceRoot, item.casePath);
|
|
322
|
-
const blockedReason = status === "blocked" && typeof resultRaw.blockedReason === "string" && resultRaw.blockedReason.trim() ? resultRaw.blockedReason.trim() : undefined;
|
|
323
372
|
const explicitAnalysis = typeof resultRaw.errorAnalysis === "string" && resultRaw.errorAnalysis.trim()
|
|
324
373
|
? resultRaw.errorAnalysis.trim()
|
|
325
374
|
: typeof resultRaw.errorSummary === "string" && resultRaw.errorSummary.trim()
|