@tea-agent/loop-agent 0.27.1 → 0.28.1-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +24 -0
- package/dist/application/task-lifecycle/observe.js +5 -0
- package/dist/application/task-lifecycle/plan-transitions.js +7 -2
- package/dist/cli/program.js +1 -1
- package/dist/commands/client-recovery.js +439 -20
- package/dist/commands/init.js +42 -6
- package/dist/executors/dag-pi-executor.js +161 -60
- package/dist/executors/pi-playwright-cli-tool.js +955 -0
- package/dist/executors/pi-sdk-executor.js +56 -0
- package/dist/executors/playwright-cli-launcher.js +63 -0
- package/dist/executors/shell-executor.js +128 -0
- package/dist/shared/playwright-cli-command-policy.js +41 -0
- package/dist/task/task-demand-routing.js +1 -15
- package/dist/worker/observability/read-model.js +66 -8
- package/dist/worker/observe/static/dag-model.js +85 -13
- package/dist/workflows/dag/dynamic-runtime/loop-until.js +4 -0
- package/dist/workflows/dag/dynamic-runtime/map.js +13 -13
- package/dist/workflows/dag/frontend-implementation-contract.js +124 -6
- package/dist/workflows/dag/frontend-prewrite-gate.js +22 -8
- package/dist/workflows/dag/frontend-test-case-checklist.js +201 -8
- package/dist/workflows/dag/frontend-test-result-contract.js +52 -3
- package/dist/workflows/dag/init-hybrid.js +181 -68
- package/dist/workflows/dag/lifecycle.js +33 -2
- package/dist/workflows/dag/node-execution.js +11 -5
- package/dist/workflows/dag/output-protocol.js +48 -83
- package/dist/workflows/dag/report.js +9 -2
- package/dist/workflows/dag/rerun-run.js +62 -3
- package/dist/workflows/dag/run-store.js +6 -1
- package/dist/workflows/dag/runner.js +15 -3
- package/dist/workflows/dag/types.js +27 -0
- package/dist/workflows/dag/validate.js +121 -1
- package/docs/architecture/runtime-boundaries.md +13 -11
- package/docs/init-surface.manifest.json +6 -2
- package/docs/templates/README.md +9 -1
- package/docs/templates/frontend-implementation-contract.schema.json +2 -2
- package/docs/templates/frontend-test-dag.generate-cases.prompt.md +14 -7
- package/docs/templates/frontend-test-dag.json +55 -15
- package/docs/templates/frontend-test-dag.retrieve-context.prompt.md +7 -9
- package/docs/templates/frontend-test-dag.retrospect.prompt.md +1 -1
- package/docs/templates/frontend-test-dag.review-cases.prompt.md +1 -1
- package/docs/templates/frontend-test-dag.review-execution.prompt.md +1 -1
- package/harness.json +1 -1
- package/package.json +1 -1
- package/skills/loop-agent/SKILL.md +1 -1
- package/skills/loop-agent/references/command-reference.md +18 -6
- package/skills/playwright-cli/SKILL.md +69 -402
- package/skills/playwright-cli/references/tracing.md +3 -137
- package/skills/playwright-cli/references/video-recording.md +3 -141
- package/skills/playwright-cli-case-generator/SKILL.md +53 -46
|
@@ -3,6 +3,7 @@ import path from "node:path";
|
|
|
3
3
|
import { writeDagRunJsonArtifact } from "../../../infrastructure/harness/artifact-store.js";
|
|
4
4
|
import { executeDagNode, } from "../node-execution.js";
|
|
5
5
|
import { writeRunSpec } from "../run-store.js";
|
|
6
|
+
import { assertValidMaterializedDagTask } from "../validate.js";
|
|
6
7
|
import { freshNodeRecord, parseJsonFromText, renderDynamicPatternList, renderDynamicTemplate, resolveItemsFromSelector, sha256Json, } from "./shared.js";
|
|
7
8
|
export function buildExpandedChildTask(input) {
|
|
8
9
|
const { parent, expansion, item, index, nodeId } = input;
|
|
@@ -20,6 +21,7 @@ export function buildExpandedChildTask(input) {
|
|
|
20
21
|
role: child.role,
|
|
21
22
|
skills: child.skills,
|
|
22
23
|
toolProfile: child.toolProfile,
|
|
24
|
+
commandPolicy: child.commandPolicy,
|
|
23
25
|
writePolicy: child.writePolicy,
|
|
24
26
|
allowedPaths: renderDynamicPatternList(child.allowedPaths, item, index, expansion.itemName) ?? [],
|
|
25
27
|
forbiddenPaths: renderDynamicPatternList(child.forbiddenPaths, item, index, expansion.itemName) ?? [],
|
|
@@ -98,14 +100,11 @@ async function materializeBlockedCaseEvidence(input) {
|
|
|
98
100
|
const caseId = resolveCaseIdFromItem(input.item);
|
|
99
101
|
try {
|
|
100
102
|
const existing = JSON.parse(await readFile(resultPath, "utf-8"));
|
|
101
|
-
if (existing.caseId === caseId &&
|
|
102
|
-
(existing.status === "passed" ||
|
|
103
|
-
existing.status === "failed" ||
|
|
104
|
-
existing.status === "blocked"))
|
|
103
|
+
if (existing.caseId === caseId && existing.status === "blocked")
|
|
105
104
|
return;
|
|
106
105
|
}
|
|
107
106
|
catch {
|
|
108
|
-
// Missing or malformed evidence is replaced
|
|
107
|
+
// Missing or malformed evidence is replaced by controller-owned terminal evidence.
|
|
109
108
|
}
|
|
110
109
|
await writeFile(path.join(workspace, "execution.md"), `# ${caseId}\n\nStatus: blocked\n\nReason: ${input.reason}\n`, "utf-8");
|
|
111
110
|
await writeFile(resultPath, `${JSON.stringify({
|
|
@@ -130,14 +129,11 @@ async function materializeFailedCaseEvidence(input) {
|
|
|
130
129
|
const caseId = resolveCaseIdFromItem(input.item);
|
|
131
130
|
try {
|
|
132
131
|
const existing = JSON.parse(await readFile(resultPath, "utf-8"));
|
|
133
|
-
if (existing.caseId === caseId &&
|
|
134
|
-
(existing.status === "passed" ||
|
|
135
|
-
existing.status === "failed" ||
|
|
136
|
-
existing.status === "blocked"))
|
|
132
|
+
if (existing.caseId === caseId && existing.status === "failed")
|
|
137
133
|
return;
|
|
138
134
|
}
|
|
139
135
|
catch {
|
|
140
|
-
// Missing or malformed evidence is replaced
|
|
136
|
+
// Missing or malformed evidence is replaced by controller-owned terminal evidence.
|
|
141
137
|
}
|
|
142
138
|
await writeFile(path.join(workspace, "execution.md"), `# ${caseId}\n\nStatus: failed\n\nReason: ${input.reason}\n`, "utf-8");
|
|
143
139
|
await writeFile(resultPath, `${JSON.stringify({
|
|
@@ -170,6 +166,9 @@ export async function executeDynamicMapExpansion(input) {
|
|
|
170
166
|
index,
|
|
171
167
|
nodeId,
|
|
172
168
|
}));
|
|
169
|
+
for (const child of children) {
|
|
170
|
+
assertValidMaterializedDagTask(child);
|
|
171
|
+
}
|
|
173
172
|
const writeSetConflicts = collectDynamicChildWriteSetConflicts(children);
|
|
174
173
|
if (writeSetConflicts.length > 0) {
|
|
175
174
|
throw new Error(`dynamic map_agent ${input.task.id} expanded overlapping writeSets: ${writeSetConflicts.join("; ")}`);
|
|
@@ -356,9 +355,9 @@ export async function executeDynamicMapExpansion(input) {
|
|
|
356
355
|
throw new Error(`failed to materialize case outcome evidence for ${nodeId}: ${error instanceof Error ? error.message : String(error)}`);
|
|
357
356
|
}
|
|
358
357
|
caseOutcomeNotes.push({ nodeId, reason, status: outcomeStatus });
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
358
|
+
// `tolerateChildFailures` changes only the map barrier's business
|
|
359
|
+
// aggregation. The child terminal state and failure category remain
|
|
360
|
+
// executor-owned facts for downstream authority checks.
|
|
362
361
|
}
|
|
363
362
|
}
|
|
364
363
|
const budgetBlockedIds = new Set(blockedChildren.map((entry) => entry.nodeId));
|
|
@@ -380,6 +379,7 @@ export async function executeDynamicMapExpansion(input) {
|
|
|
380
379
|
item: items[index],
|
|
381
380
|
workspaceRef: workspaceRefs[index],
|
|
382
381
|
status: input.state.nodes[nodeId]?.status,
|
|
382
|
+
failureCategory: input.state.nodes[nodeId]?.failureCategory,
|
|
383
383
|
stdout: input.state.nodes[nodeId]?.stdout,
|
|
384
384
|
output: parseJsonFromText(input.state.nodes[nodeId]?.stdout),
|
|
385
385
|
assistantText: input.state.nodes[nodeId]?.assistantText,
|
|
@@ -155,7 +155,7 @@ export const frontendImplementationContractSchema = z
|
|
|
155
155
|
"not-needed",
|
|
156
156
|
]),
|
|
157
157
|
productionDefaultOff: z.literal(true),
|
|
158
|
-
activation: z.string().min(1),
|
|
158
|
+
activation: z.preprocess((value) => (value === "" || value === null ? "explicit activation boundary" : value), z.string().min(1)),
|
|
159
159
|
endpoints: z.array(z
|
|
160
160
|
.object({
|
|
161
161
|
method: z.enum([
|
|
@@ -168,7 +168,10 @@ export const frontendImplementationContractSchema = z
|
|
|
168
168
|
"OPTIONS",
|
|
169
169
|
]),
|
|
170
170
|
path: z.string().startsWith("/"),
|
|
171
|
-
fixture
|
|
171
|
+
// Models occasionally emit an empty fixture when Mock is
|
|
172
|
+
// intentionally not needed. Treat it like an omitted optional
|
|
173
|
+
// field; active Mock strategies still fail the refinement below.
|
|
174
|
+
fixture: z.preprocess((value) => (value === "" || value === null ? undefined : value), safePath.optional()),
|
|
172
175
|
consumer: safePath.optional(),
|
|
173
176
|
})
|
|
174
177
|
.strict()),
|
|
@@ -187,7 +190,7 @@ export const frontendImplementationContractSchema = z
|
|
|
187
190
|
type: z.enum(["static", "unit", "component", "integration", "mock"]),
|
|
188
191
|
commandLabel: z.string().min(1),
|
|
189
192
|
file: safePath,
|
|
190
|
-
symbol: z.string().min(1).optional(),
|
|
193
|
+
symbol: z.preprocess((value) => (value === "" || value === null ? undefined : value), z.string().min(1).optional()),
|
|
191
194
|
requirementIds: z.array(id),
|
|
192
195
|
uiStates: z.array(z.string().min(1)),
|
|
193
196
|
})
|
|
@@ -343,12 +346,127 @@ function secretIssues(value, at = "$", issues = []) {
|
|
|
343
346
|
}
|
|
344
347
|
export function extractFrontendImplementationJson(text) {
|
|
345
348
|
const trimmed = text.trim();
|
|
349
|
+
const parse = (source) => {
|
|
350
|
+
try {
|
|
351
|
+
return JSON.parse(source);
|
|
352
|
+
}
|
|
353
|
+
catch (error) {
|
|
354
|
+
// Models sometimes put ordinary ASCII quotes inside a JSON string
|
|
355
|
+
// (for example: `reason: "支持..."`). Repair only quotes that are
|
|
356
|
+
// clearly not structural: a closing quote is followed by JSON
|
|
357
|
+
// punctuation, while an embedded quote is followed by content.
|
|
358
|
+
let repaired = "";
|
|
359
|
+
let inString = false;
|
|
360
|
+
let escaped = false;
|
|
361
|
+
for (let index = 0; index < source.length; index += 1) {
|
|
362
|
+
const character = source[index];
|
|
363
|
+
if (character !== '"') {
|
|
364
|
+
repaired += character;
|
|
365
|
+
if (inString && character === "\\" && !escaped)
|
|
366
|
+
escaped = true;
|
|
367
|
+
else
|
|
368
|
+
escaped = false;
|
|
369
|
+
continue;
|
|
370
|
+
}
|
|
371
|
+
if (escaped) {
|
|
372
|
+
repaired += character;
|
|
373
|
+
escaped = false;
|
|
374
|
+
continue;
|
|
375
|
+
}
|
|
376
|
+
if (!inString) {
|
|
377
|
+
inString = true;
|
|
378
|
+
repaired += character;
|
|
379
|
+
continue;
|
|
380
|
+
}
|
|
381
|
+
const next = source.slice(index + 1).trimStart()[0];
|
|
382
|
+
if ([",", "}", "]", ":"].includes(next ?? "") || source.slice(index + 1).trim() === "") {
|
|
383
|
+
inString = false;
|
|
384
|
+
repaired += character;
|
|
385
|
+
}
|
|
386
|
+
else {
|
|
387
|
+
repaired += "\\\"";
|
|
388
|
+
}
|
|
389
|
+
}
|
|
390
|
+
try {
|
|
391
|
+
return JSON.parse(repaired);
|
|
392
|
+
}
|
|
393
|
+
catch {
|
|
394
|
+
throw error;
|
|
395
|
+
}
|
|
396
|
+
}
|
|
397
|
+
};
|
|
398
|
+
const isContract = (candidate) => {
|
|
399
|
+
const record = asRecord(candidate);
|
|
400
|
+
return (record?.schemaVersion === 1 &&
|
|
401
|
+
asRecord(record.targets) !== null &&
|
|
402
|
+
Array.isArray(record.requirements) &&
|
|
403
|
+
Array.isArray(record.verificationTargets));
|
|
404
|
+
};
|
|
346
405
|
if (trimmed.startsWith("{") && trimmed.endsWith("}"))
|
|
347
|
-
return
|
|
406
|
+
return parse(trimmed);
|
|
407
|
+
const balancedObjects = [];
|
|
408
|
+
for (let start = 0; start < trimmed.length; start += 1) {
|
|
409
|
+
if (trimmed[start] !== "{")
|
|
410
|
+
continue;
|
|
411
|
+
let depth = 0;
|
|
412
|
+
let inString = false;
|
|
413
|
+
let escaped = false;
|
|
414
|
+
for (let index = start; index < trimmed.length; index += 1) {
|
|
415
|
+
const character = trimmed[index];
|
|
416
|
+
if (inString) {
|
|
417
|
+
if (escaped)
|
|
418
|
+
escaped = false;
|
|
419
|
+
else if (character === "\\")
|
|
420
|
+
escaped = true;
|
|
421
|
+
else if (character === '"')
|
|
422
|
+
inString = false;
|
|
423
|
+
continue;
|
|
424
|
+
}
|
|
425
|
+
if (character === '"')
|
|
426
|
+
inString = true;
|
|
427
|
+
else if (character === "{")
|
|
428
|
+
depth += 1;
|
|
429
|
+
else if (character === "}" && --depth === 0) {
|
|
430
|
+
try {
|
|
431
|
+
const candidate = parse(trimmed.slice(start, index + 1));
|
|
432
|
+
if (candidate && typeof candidate === "object" && !Array.isArray(candidate))
|
|
433
|
+
balancedObjects.push(candidate);
|
|
434
|
+
}
|
|
435
|
+
catch {
|
|
436
|
+
// Continue scanning for a later complete JSON object.
|
|
437
|
+
}
|
|
438
|
+
break;
|
|
439
|
+
}
|
|
440
|
+
}
|
|
441
|
+
}
|
|
442
|
+
const balancedContracts = balancedObjects.filter(isContract);
|
|
443
|
+
if (balancedContracts.length === 1)
|
|
444
|
+
return balancedContracts[0];
|
|
445
|
+
if (balancedObjects.length === 1)
|
|
446
|
+
return balancedObjects[0];
|
|
348
447
|
const blocks = [...trimmed.matchAll(/```json\s*\n([\s\S]*?)\n```/gi)];
|
|
349
|
-
if (blocks.length
|
|
448
|
+
if (blocks.length === 0)
|
|
350
449
|
throw new Error("output must contain exactly one fenced json object");
|
|
351
|
-
|
|
450
|
+
const candidates = [];
|
|
451
|
+
for (const block of blocks) {
|
|
452
|
+
try {
|
|
453
|
+
const candidate = parse(block[1]);
|
|
454
|
+
if (candidate && typeof candidate === "object" && !Array.isArray(candidate))
|
|
455
|
+
candidates.push(candidate);
|
|
456
|
+
}
|
|
457
|
+
catch {
|
|
458
|
+
// Ignore incomplete model scratch blocks. A later complete contract
|
|
459
|
+
// block may still be deterministically recoverable.
|
|
460
|
+
}
|
|
461
|
+
}
|
|
462
|
+
const contractCandidates = candidates.filter(isContract);
|
|
463
|
+
if (contractCandidates.length === 1)
|
|
464
|
+
return contractCandidates[0];
|
|
465
|
+
if (candidates.length === 1)
|
|
466
|
+
return candidates[0];
|
|
467
|
+
if (candidates.length === 0)
|
|
468
|
+
throw new Error("output must contain exactly one valid fenced json object (found 0)");
|
|
469
|
+
throw new Error(`output must contain exactly one valid frontend contract json object (found ${contractCandidates.length || candidates.length})`);
|
|
352
470
|
}
|
|
353
471
|
/**
|
|
354
472
|
* Build the authoritative frontend-implementation-contract sourceBinding from
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import { readFile, stat } from "node:fs/promises";
|
|
2
|
+
import { createHash } from "node:crypto";
|
|
2
3
|
import path from "node:path";
|
|
3
4
|
import { isOpenspecSpecFilePath } from "../../shared/openspec-spec.js";
|
|
4
|
-
import { frontendImplementationContractSchema, materializeFrontendImplementationContract, assertFrontendSourceBindingFresh, } from "./frontend-implementation-contract.js";
|
|
5
|
+
import { frontendImplementationContractSchema, FRONTEND_IMPLEMENTATION_CONTRACT_SCHEMA_ID, materializeFrontendImplementationContract, assertFrontendSourceBindingFresh, } from "./frontend-implementation-contract.js";
|
|
5
6
|
import { captureFrontendWorktreeBaseline } from "./frontend-worktree-diff.js";
|
|
6
7
|
import { pathMatchesPattern } from "../../shared/git-progress.js";
|
|
7
8
|
async function selectNode(runDir, primary, fallbacks) {
|
|
@@ -46,7 +47,12 @@ function firstNonEmptyVerdictLine(text) {
|
|
|
46
47
|
// The prompt requires VERDICT to be the first non-empty line, but model
|
|
47
48
|
// output can still prepend a summary. Keep the protocol strict in the
|
|
48
49
|
// prompt while making the deterministic gate resilient to that drift.
|
|
49
|
-
|
|
50
|
+
const verdictLine = normalizedLines.find((line) => /^VERDICT:/.test(line)) ?? first;
|
|
51
|
+
if (/^VERDICT:\s*pass(?:\s*[.!?。!?]|\s+|$)/i.test(verdictLine))
|
|
52
|
+
return "VERDICT: pass";
|
|
53
|
+
if (/^VERDICT:\s*request-revision(?:\s*[.!?。!?]|\s+|$)/i.test(verdictLine))
|
|
54
|
+
return "VERDICT: request-revision";
|
|
55
|
+
return verdictLine;
|
|
50
56
|
}
|
|
51
57
|
function eventArgs(event) {
|
|
52
58
|
return event.args ?? event.toolInput ?? event.input ?? {};
|
|
@@ -155,13 +161,20 @@ export async function runFrontendPrewriteGate(input) {
|
|
|
155
161
|
if (missingIds.length > 0) {
|
|
156
162
|
throw new Error(`frontend prewrite gate missing requirement ids: ${missingIds.join(", ")}`);
|
|
157
163
|
}
|
|
158
|
-
const
|
|
164
|
+
const artifactPath = path.join(input.runDir, input.config.outputDir, input.config.artifactName);
|
|
165
|
+
const artifact = await stat(artifactPath)
|
|
166
|
+
.then(async () => ({
|
|
167
|
+
path: artifactPath,
|
|
168
|
+
schemaId: FRONTEND_IMPLEMENTATION_CONTRACT_SCHEMA_ID,
|
|
169
|
+
sha256: createHash("sha256").update(await readFile(artifactPath)).digest("hex"),
|
|
170
|
+
}))
|
|
171
|
+
.catch(() => materializeFrontendImplementationContract({
|
|
159
172
|
runDir: input.runDir,
|
|
160
173
|
fromNodeId: planNodeId,
|
|
161
174
|
artifactName: input.config.artifactName,
|
|
162
175
|
outputDir: input.config.outputDir,
|
|
163
176
|
sourceBinding: input.sourceBinding,
|
|
164
|
-
});
|
|
177
|
+
}));
|
|
165
178
|
const raw = JSON.parse(await readFile(artifact.path, "utf8"));
|
|
166
179
|
const contract = frontendImplementationContractSchema.parse(raw);
|
|
167
180
|
if (!input.config.allowedMockStrategies.includes(contract.mockApi.strategy)) {
|
|
@@ -183,10 +196,11 @@ export async function runFrontendPrewriteGate(input) {
|
|
|
183
196
|
reviewNodeId,
|
|
184
197
|
repoRoot: workspaceRoot ?? process.cwd(),
|
|
185
198
|
});
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
199
|
+
// OpenSpec is advisory design evidence, not an authorization boundary.
|
|
200
|
+
// Model read behavior is nondeterministic, and a missed read must not block
|
|
201
|
+
// a contract/schema/write-set-authorized implementation. Preserve any
|
|
202
|
+
// successful evidence for review context, while allowing unavailable
|
|
203
|
+
// evidence to be reported downstream.
|
|
190
204
|
return {
|
|
191
205
|
ok: true,
|
|
192
206
|
planNodeId,
|
|
@@ -1,5 +1,19 @@
|
|
|
1
1
|
import { readFile, stat } from "node:fs/promises";
|
|
2
2
|
import path from "node:path";
|
|
3
|
+
import { isPlaywrightCliCommand } from "../../shared/playwright-cli-command-policy.js";
|
|
4
|
+
const COMMAND_FENCE_LANGUAGE = /^(?:|bash|sh|shell|zsh|fish|console|terminal|command|cmd|powershell|pwsh|javascript|js|typescript|ts|python|py)$/i;
|
|
5
|
+
const LIST_ITEM = /^\s*(?:[-+*]|\d+[.)])\s+(.+)$/;
|
|
6
|
+
const INDENTED_CODE = /^(?: {4,}|\t+)(\S.*)$/;
|
|
7
|
+
const EXPLICIT_COMMAND = /^(?:(?:shell|terminal)(?:\s+command)?|command|run command|execute command)\s*:\s*(.+)$/i;
|
|
8
|
+
const BLOCKED_REASON_PREFIX = /^(?:(?:blocked|forbidden|reject(?:ed)?|disallow(?:ed)?|prohibited)(?:\s+reason)?\s*(?::|\bbecause\b)|(?:do not|must not|never)\s+(?:run|execute|use)\b)/i;
|
|
9
|
+
const IMPERATIVE_COMMAND = /^(?:(?:run|execute|use)(?:\s+(?:the\s+)?(?:(?:shell|terminal)\s+)?command)?|in\s+(?:the\s+)?(?:shell|terminal|console)\s*,?\s*(?:run|execute|use))\s*:?\s+(.+)$/i;
|
|
10
|
+
const INLINE_CODE_STEP = /^`([^`\r\n]+)`[.!?]?$/;
|
|
11
|
+
const AUTOMATION_EXECUTABLE = /^(?:playwright-cli\b|playwright\b|@playwright\/test\b|npx\b|npm\b|pnpm\b|yarn\b|bunx?\b|node(?:js)?\b|python(?:3)?\b|bash\b|sh\b|zsh\b|fish\b|powershell\b|pwsh\b|cmd(?:\.exe)?\b|cypress\b|selenium\b|webdriverio\b|chromedriver\b|google-chrome\b|chrome\b|firefox\b|curl\b|wget\b)/i;
|
|
12
|
+
const SHELL_WRAPPED_EXECUTABLE = /^(?:sudo\s+|env(?:\s+[A-Za-z_][A-Za-z0-9_]*=[^\s]+)*\s+|(?:\.{1,2}[\\/]|~[\\/]|[A-Za-z]:[\\/]|\/)[^\s]+)/i;
|
|
13
|
+
const ENV_ASSIGNMENT = /^[A-Za-z_][A-Za-z0-9_]*=(?:"[^"]*"|'[^']*'|[^\s]*)/;
|
|
14
|
+
const ENV_WRAPPER = /^env(?:\s+(?:-[A-Za-z]+|--[A-Za-z][A-Za-z-]*))*\s+/i;
|
|
15
|
+
const SHELL_BUILTIN_WRAPPER = /^(?:command|exec|sudo)\s+/i;
|
|
16
|
+
const PLAYWRIGHT_API = /^(?:from\s+playwright(?:\.[A-Za-z_][A-Za-z0-9_]*)*\s+import\b|import\s+playwright(?:\.[A-Za-z_][A-Za-z0-9_]*)*(?:\s+as\s+[A-Za-z_][A-Za-z0-9_]*)?\b|import\s+.+\s+from\s+["'](?:playwright|@playwright\/test)["']|(?:const|let|var)\s+.+\brequire\s*\(\s*["'](?:playwright|@playwright\/test)["']\s*\)|require\s*\(\s*["'](?:playwright|@playwright\/test)["']\s*\)|(?:sync_playwright|async_playwright|chromium|firefox|webkit)\.(?:launch|connect)\b)/i;
|
|
3
17
|
async function exists(file) {
|
|
4
18
|
try {
|
|
5
19
|
return (await stat(file)).isFile();
|
|
@@ -8,6 +22,180 @@ async function exists(file) {
|
|
|
8
22
|
return false;
|
|
9
23
|
}
|
|
10
24
|
}
|
|
25
|
+
function normalizeCommand(value) {
|
|
26
|
+
const trimmed = value.trim().replace(/^[$>]\s*/, "");
|
|
27
|
+
const wrapped = trimmed.match(/^`([^`]+)`$/);
|
|
28
|
+
return (wrapped?.[1] ?? trimmed).trim();
|
|
29
|
+
}
|
|
30
|
+
function isExecutableFence(language) {
|
|
31
|
+
return COMMAND_FENCE_LANGUAGE.test(language.trim());
|
|
32
|
+
}
|
|
33
|
+
function isBlockedReason(value) {
|
|
34
|
+
return BLOCKED_REASON_PREFIX.test(value.trim());
|
|
35
|
+
}
|
|
36
|
+
function stripLeadingCommandWrappers(command) {
|
|
37
|
+
let remaining = command.trim();
|
|
38
|
+
let consumed = true;
|
|
39
|
+
while (remaining && consumed) {
|
|
40
|
+
consumed = false;
|
|
41
|
+
const assignment = remaining.match(ENV_ASSIGNMENT);
|
|
42
|
+
if (assignment) {
|
|
43
|
+
remaining = remaining.slice(assignment[0].length).trimStart();
|
|
44
|
+
consumed = true;
|
|
45
|
+
continue;
|
|
46
|
+
}
|
|
47
|
+
const env = remaining.match(ENV_WRAPPER);
|
|
48
|
+
if (env) {
|
|
49
|
+
remaining = remaining.slice(env[0].length).trimStart();
|
|
50
|
+
consumed = true;
|
|
51
|
+
continue;
|
|
52
|
+
}
|
|
53
|
+
const wrapper = remaining.match(SHELL_BUILTIN_WRAPPER);
|
|
54
|
+
if (wrapper) {
|
|
55
|
+
remaining = remaining.slice(wrapper[0].length).trimStart();
|
|
56
|
+
consumed = true;
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
return remaining;
|
|
60
|
+
}
|
|
61
|
+
function isCommandLikeExecutable(command) {
|
|
62
|
+
const executable = stripLeadingCommandWrappers(command);
|
|
63
|
+
return AUTOMATION_EXECUTABLE.test(executable) || SHELL_WRAPPED_EXECUTABLE.test(executable) || PLAYWRIGHT_API.test(executable);
|
|
64
|
+
}
|
|
65
|
+
function executableListStep(value) {
|
|
66
|
+
const trimmed = value.trim();
|
|
67
|
+
const explicit = trimmed.match(EXPLICIT_COMMAND);
|
|
68
|
+
if (explicit)
|
|
69
|
+
return isBlockedReason(explicit[1]) ? null : normalizeCommand(explicit[1]);
|
|
70
|
+
const imperative = trimmed.match(IMPERATIVE_COMMAND);
|
|
71
|
+
if (imperative)
|
|
72
|
+
return normalizeCommand(imperative[1]);
|
|
73
|
+
const inlineCode = trimmed.match(INLINE_CODE_STEP);
|
|
74
|
+
if (inlineCode)
|
|
75
|
+
return normalizeCommand(inlineCode[1]);
|
|
76
|
+
const prompted = /^[$>]\s*\S/.test(trimmed);
|
|
77
|
+
const normalized = normalizeCommand(trimmed);
|
|
78
|
+
if (prompted || isCommandLikeExecutable(normalized))
|
|
79
|
+
return normalized;
|
|
80
|
+
return null;
|
|
81
|
+
}
|
|
82
|
+
function hasShellControl(command) {
|
|
83
|
+
let quote = null;
|
|
84
|
+
for (let index = 0; index < command.length; index += 1) {
|
|
85
|
+
const character = command[index];
|
|
86
|
+
if (character === "\\" && quote !== "'") {
|
|
87
|
+
index += 1;
|
|
88
|
+
continue;
|
|
89
|
+
}
|
|
90
|
+
if (quote !== null) {
|
|
91
|
+
if (character === quote)
|
|
92
|
+
quote = null;
|
|
93
|
+
else if (quote === '"' && (character === "`" || (character === "$" && command[index + 1] === "(")))
|
|
94
|
+
return true;
|
|
95
|
+
continue;
|
|
96
|
+
}
|
|
97
|
+
if (character === "'" || character === '"') {
|
|
98
|
+
quote = character;
|
|
99
|
+
continue;
|
|
100
|
+
}
|
|
101
|
+
if (character === "#" && (index === 0 || /\s/.test(command[index - 1])))
|
|
102
|
+
break;
|
|
103
|
+
if (character === "`" || character === ";" || character === "|" || character === "&" || character === "<" || character === ">")
|
|
104
|
+
return true;
|
|
105
|
+
if (character === "$" && command[index + 1] === "(")
|
|
106
|
+
return true;
|
|
107
|
+
}
|
|
108
|
+
return false;
|
|
109
|
+
}
|
|
110
|
+
/**
|
|
111
|
+
* Return only Markdown lines that are presented as commands. Prose can mention
|
|
112
|
+
* forbidden commands as a blocked reason without becoming an executable step.
|
|
113
|
+
*/
|
|
114
|
+
function extractExecutableInstructions(markdown) {
|
|
115
|
+
const instructions = [];
|
|
116
|
+
let fenceLanguage = null;
|
|
117
|
+
for (const [index, line] of markdown.split(/\r?\n/).entries()) {
|
|
118
|
+
const lineNumber = index + 1;
|
|
119
|
+
const fence = line.match(/^\s*```([^`]*)$/);
|
|
120
|
+
if (fence) {
|
|
121
|
+
fenceLanguage = fenceLanguage === null ? fence[1].trim() : null;
|
|
122
|
+
continue;
|
|
123
|
+
}
|
|
124
|
+
const trimmed = line.trim();
|
|
125
|
+
if (!trimmed)
|
|
126
|
+
continue;
|
|
127
|
+
if (fenceLanguage !== null) {
|
|
128
|
+
if (!isExecutableFence(fenceLanguage) || /^(?:#|\/\/)/.test(trimmed))
|
|
129
|
+
continue;
|
|
130
|
+
instructions.push({ command: normalizeCommand(trimmed), lineNumber });
|
|
131
|
+
continue;
|
|
132
|
+
}
|
|
133
|
+
const listed = line.match(LIST_ITEM);
|
|
134
|
+
if (listed) {
|
|
135
|
+
if (isBlockedReason(listed[1]))
|
|
136
|
+
continue;
|
|
137
|
+
const command = executableListStep(listed[1]);
|
|
138
|
+
if (command !== null)
|
|
139
|
+
instructions.push({ command, lineNumber });
|
|
140
|
+
continue;
|
|
141
|
+
}
|
|
142
|
+
const indented = line.match(INDENTED_CODE);
|
|
143
|
+
if (indented) {
|
|
144
|
+
if (isBlockedReason(indented[1]))
|
|
145
|
+
continue;
|
|
146
|
+
const explicitIndented = indented[1].match(EXPLICIT_COMMAND);
|
|
147
|
+
if (explicitIndented && isBlockedReason(explicitIndented[1]))
|
|
148
|
+
continue;
|
|
149
|
+
instructions.push({ command: executableListStep(indented[1]) ?? normalizeCommand(indented[1]), lineNumber });
|
|
150
|
+
continue;
|
|
151
|
+
}
|
|
152
|
+
const explicit = trimmed.match(EXPLICIT_COMMAND);
|
|
153
|
+
if (explicit) {
|
|
154
|
+
if (isBlockedReason(explicit[1]))
|
|
155
|
+
continue;
|
|
156
|
+
instructions.push({ command: normalizeCommand(explicit[1]), lineNumber });
|
|
157
|
+
continue;
|
|
158
|
+
}
|
|
159
|
+
const command = executableListStep(trimmed);
|
|
160
|
+
if (command !== null)
|
|
161
|
+
instructions.push({ command, lineNumber });
|
|
162
|
+
}
|
|
163
|
+
return instructions;
|
|
164
|
+
}
|
|
165
|
+
function commandGateIssue(input) {
|
|
166
|
+
const commandMatch = input.instruction.command.match(/^playwright-cli\s+([^\s`]+)/i);
|
|
167
|
+
if (commandMatch) {
|
|
168
|
+
const command = commandMatch[1].toLowerCase();
|
|
169
|
+
if (isPlaywrightCliCommand(command) && !hasShellControl(input.instruction.command))
|
|
170
|
+
return null;
|
|
171
|
+
if (isPlaywrightCliCommand(command)) {
|
|
172
|
+
return {
|
|
173
|
+
ruleId: "alternative-executable-command",
|
|
174
|
+
caseId: input.caseId,
|
|
175
|
+
casePath: input.casePath,
|
|
176
|
+
lineNumber: input.instruction.lineNumber,
|
|
177
|
+
command: input.instruction.command,
|
|
178
|
+
detail: "shell control or additional executable fragments are not allowed after playwright-cli commands",
|
|
179
|
+
};
|
|
180
|
+
}
|
|
181
|
+
return {
|
|
182
|
+
ruleId: "playwright-cli-command-not-allowed",
|
|
183
|
+
caseId: input.caseId,
|
|
184
|
+
casePath: input.casePath,
|
|
185
|
+
lineNumber: input.instruction.lineNumber,
|
|
186
|
+
command: input.instruction.command,
|
|
187
|
+
detail: `playwright-cli command is not allowlisted: ${command}`,
|
|
188
|
+
};
|
|
189
|
+
}
|
|
190
|
+
return {
|
|
191
|
+
ruleId: "alternative-executable-command",
|
|
192
|
+
caseId: input.caseId,
|
|
193
|
+
casePath: input.casePath,
|
|
194
|
+
lineNumber: input.instruction.lineNumber,
|
|
195
|
+
command: input.instruction.command,
|
|
196
|
+
detail: "only allowlisted playwright-cli commands may appear in executable case instructions",
|
|
197
|
+
};
|
|
198
|
+
}
|
|
11
199
|
export async function validateFrontendCaseChecklist(input) {
|
|
12
200
|
const root = path.join(input.workspaceRoot, "testcase/frontend/cases");
|
|
13
201
|
const draft = path.join(root, "manifest.draft.json");
|
|
@@ -36,34 +224,39 @@ export async function validateFrontendCaseChecklist(input) {
|
|
|
36
224
|
const absolute = path.resolve(input.workspaceRoot, casePath);
|
|
37
225
|
const relative = path.relative(input.workspaceRoot, absolute);
|
|
38
226
|
if (!casePath || relative.startsWith("..") || path.isAbsolute(relative) || !(await exists(absolute))) {
|
|
39
|
-
issues.push({ ruleId: "case-file-missing", caseId: id, detail: casePath || "missing casePath" });
|
|
227
|
+
issues.push({ ruleId: "case-file-missing", caseId: id, casePath: casePath || undefined, detail: casePath || "missing casePath" });
|
|
40
228
|
continue;
|
|
41
229
|
}
|
|
42
230
|
if (expectedPath && casePath.replaceAll("\\", "/") !== expectedPath)
|
|
43
|
-
issues.push({ ruleId: "case-path-mismatch", caseId: id, detail: `${casePath} must equal ${expectedPath}` });
|
|
231
|
+
issues.push({ ruleId: "case-path-mismatch", caseId: id, casePath, detail: `${casePath} must equal ${expectedPath}` });
|
|
44
232
|
const body = await readFile(absolute, "utf8");
|
|
45
233
|
if (!openRe.test(body))
|
|
46
|
-
issues.push({ ruleId: "open-prefix", caseId: id, detail: "missing playwright-cli open --browser=chrome --headed <absolute-url>" });
|
|
234
|
+
issues.push({ ruleId: "open-prefix", caseId: id, casePath, detail: "missing playwright-cli open --browser=chrome --headed <absolute-url>" });
|
|
47
235
|
const match = body.match(/playwright-cli\s+open\s+--browser=chrome\s+--headed\s+(https?:\/\/\S+)/i);
|
|
48
236
|
if (match) {
|
|
49
237
|
try {
|
|
50
238
|
const url = new URL(match[1].replace(/[)\]},.\"'`]+$/, ""));
|
|
51
239
|
if (productionHostRe.test(url.hostname))
|
|
52
|
-
issues.push({ ruleId: "production-url", caseId: id, detail: match[1] });
|
|
240
|
+
issues.push({ ruleId: "production-url", caseId: id, casePath, detail: match[1] });
|
|
53
241
|
}
|
|
54
242
|
catch {
|
|
55
|
-
issues.push({ ruleId: "production-url", caseId: id, detail: match[1] });
|
|
243
|
+
issues.push({ ruleId: "production-url", caseId: id, casePath, detail: match[1] });
|
|
56
244
|
}
|
|
57
245
|
}
|
|
246
|
+
for (const instruction of extractExecutableInstructions(body)) {
|
|
247
|
+
const issue = commandGateIssue({ caseId: id, casePath, instruction });
|
|
248
|
+
if (issue)
|
|
249
|
+
issues.push(issue);
|
|
250
|
+
}
|
|
58
251
|
if (!Array.isArray(item?.acIds) || item.acIds.length === 0) {
|
|
59
|
-
issues.push({ ruleId: "ac-mapping", caseId: id, detail: "acIds required" });
|
|
252
|
+
issues.push({ ruleId: "ac-mapping", caseId: id, casePath, detail: "acIds required" });
|
|
60
253
|
}
|
|
61
254
|
else {
|
|
62
255
|
for (const ac of item.acIds) {
|
|
63
256
|
if (typeof ac !== "string" || !acIdRe.test(ac))
|
|
64
|
-
issues.push({ ruleId: "ac-id-shape", caseId: id, detail: String(ac) });
|
|
257
|
+
issues.push({ ruleId: "ac-id-shape", caseId: id, casePath, detail: String(ac) });
|
|
65
258
|
else if (declaredAc.size > 0 && !declaredAc.has(ac))
|
|
66
|
-
issues.push({ ruleId: "unknown-ac", caseId: id, detail: `${ac} not in sourceBinding` });
|
|
259
|
+
issues.push({ ruleId: "unknown-ac", caseId: id, casePath, detail: `${ac} not in sourceBinding` });
|
|
67
260
|
}
|
|
68
261
|
}
|
|
69
262
|
}
|
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
import { createHash } from "node:crypto";
|
|
2
|
-
import { lstat, readFile, realpath, stat } from "node:fs/promises";
|
|
2
|
+
import { lstat, readdir, readFile, realpath, stat } from "node:fs/promises";
|
|
3
3
|
import path from "node:path";
|
|
4
4
|
import { z } from "zod";
|
|
5
5
|
import { writeDagRunJsonArtifact } from "../../infrastructure/harness/artifact-store.js";
|
|
6
|
+
import { readPlaywrightCliReceipts, summarizePlaywrightCliReceipts, } from "../../executors/pi-playwright-cli-tool.js";
|
|
6
7
|
import { validateFrontendCaseContent } from "./frontend-test-case-quality.js";
|
|
7
8
|
export const FRONTEND_TEST_RESULT_SCHEMA_ID = "frontend-test-result-v1";
|
|
8
9
|
const safeRelativePathSchema = z.string().min(1).refine((value) => !path.posix.isAbsolute(value) &&
|
|
@@ -180,6 +181,38 @@ export async function validateFrontendCaseEvidence(input) {
|
|
|
180
181
|
}
|
|
181
182
|
return { cases: manifest.cases.length, issues, hardFail };
|
|
182
183
|
}
|
|
184
|
+
export function evaluatePassedCaseBrowserReceipts(summary, caseId) {
|
|
185
|
+
if (!summary || summary.caseIds.size !== 1 || !summary.caseIds.has(caseId)) {
|
|
186
|
+
return { ok: false, reason: "browser-command-evidence-missing" };
|
|
187
|
+
}
|
|
188
|
+
if (!summary.hasOrderedPassedReceiptChain) {
|
|
189
|
+
return { ok: false, reason: "browser-command-evidence-missing" };
|
|
190
|
+
}
|
|
191
|
+
return { ok: true };
|
|
192
|
+
}
|
|
193
|
+
async function loadCaseBrowserReceiptSummary(input) {
|
|
194
|
+
try {
|
|
195
|
+
const entries = await readdir(input.runDir, { withFileTypes: true });
|
|
196
|
+
// A passed chain must belong to exactly one browser child stream. Never
|
|
197
|
+
// stitch receipts across DAG nodes or accept ambiguous duplicate authority.
|
|
198
|
+
const authorizingStreams = [];
|
|
199
|
+
for (const entry of entries) {
|
|
200
|
+
if (!entry.isDirectory())
|
|
201
|
+
continue;
|
|
202
|
+
const nodeReceipts = (await readPlaywrightCliReceipts(input.runDir, entry.name))
|
|
203
|
+
.filter((item) => item.caseId === input.caseId);
|
|
204
|
+
const summary = summarizePlaywrightCliReceipts(nodeReceipts);
|
|
205
|
+
if (summary.hasOrderedPassedReceiptChain)
|
|
206
|
+
authorizingStreams.push(summary);
|
|
207
|
+
}
|
|
208
|
+
return authorizingStreams.length === 1
|
|
209
|
+
? authorizingStreams[0]
|
|
210
|
+
: summarizePlaywrightCliReceipts([]);
|
|
211
|
+
}
|
|
212
|
+
catch {
|
|
213
|
+
return summarizePlaywrightCliReceipts([]);
|
|
214
|
+
}
|
|
215
|
+
}
|
|
183
216
|
function sha256(content) {
|
|
184
217
|
return createHash("sha256").update(content).digest("hex");
|
|
185
218
|
}
|
|
@@ -296,7 +329,7 @@ export async function materializeFrontendTestResult(input) {
|
|
|
296
329
|
advisoryFindings.push({ ruleId: "missing-or-invalid-case-result", caseId: item.caseId, detail: "case-result.json is missing or invalid" });
|
|
297
330
|
}
|
|
298
331
|
const parsedStatus = caseStatusSchema.safeParse(resultRaw.status);
|
|
299
|
-
|
|
332
|
+
let status = parsedStatus.success ? parsedStatus.data : "blocked";
|
|
300
333
|
if (resultRaw.caseId !== undefined && resultRaw.caseId !== item.caseId)
|
|
301
334
|
advisoryFindings.push({ ruleId: "case-result-identity", caseId: item.caseId, detail: "case-result caseId does not match manifest" });
|
|
302
335
|
if (status === "blocked" && resultRaw.blockedReason !== undefined && (typeof resultRaw.blockedReason !== "string" || !resultRaw.blockedReason.trim()))
|
|
@@ -318,8 +351,24 @@ export async function materializeFrontendTestResult(input) {
|
|
|
318
351
|
}
|
|
319
352
|
if (status === "passed" && !evidence.some((entry) => /\.(png|jpg|jpeg|webp|zip|har|webm|mp4|ya?ml|md)$/i.test(entry.path)))
|
|
320
353
|
advisoryFindings.push({ ruleId: "passed-without-browser-evidence", caseId: item.caseId, detail: "passed case has no browser evidence" });
|
|
354
|
+
let blockedReason = status === "blocked" && typeof resultRaw.blockedReason === "string" && resultRaw.blockedReason.trim() ? resultRaw.blockedReason.trim() : undefined;
|
|
355
|
+
if (status === "passed") {
|
|
356
|
+
const receiptSummary = await loadCaseBrowserReceiptSummary({
|
|
357
|
+
runDir: input.runDir,
|
|
358
|
+
caseId: item.caseId,
|
|
359
|
+
});
|
|
360
|
+
const receiptGate = evaluatePassedCaseBrowserReceipts(receiptSummary, item.caseId);
|
|
361
|
+
if (!receiptGate.ok) {
|
|
362
|
+
status = "blocked";
|
|
363
|
+
blockedReason = receiptGate.reason ?? "browser-command-evidence-missing";
|
|
364
|
+
advisoryFindings.push({
|
|
365
|
+
ruleId: "browser-command-evidence-missing",
|
|
366
|
+
caseId: item.caseId,
|
|
367
|
+
detail: "passed case lacks controller-owned open + interaction/assertion + cleanup receipts",
|
|
368
|
+
});
|
|
369
|
+
}
|
|
370
|
+
}
|
|
321
371
|
const caseContent = await readFrontendCaseContent(input.workspaceRoot, item.casePath);
|
|
322
|
-
const blockedReason = status === "blocked" && typeof resultRaw.blockedReason === "string" && resultRaw.blockedReason.trim() ? resultRaw.blockedReason.trim() : undefined;
|
|
323
372
|
const explicitAnalysis = typeof resultRaw.errorAnalysis === "string" && resultRaw.errorAnalysis.trim()
|
|
324
373
|
? resultRaw.errorAnalysis.trim()
|
|
325
374
|
: typeof resultRaw.errorSummary === "string" && resultRaw.errorSummary.trim()
|