@ivorycanvas/qamap 0.4.2 → 0.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +19 -0
- package/README.md +10 -1
- package/dist/e2e.d.ts +25 -0
- package/dist/e2e.js +245 -6
- package/dist/e2e.js.map +1 -1
- package/dist/index.d.ts +3 -1
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/qa.d.ts +2 -1
- package/dist/qa.js +68 -10
- package/dist/qa.js.map +1 -1
- package/dist/scenario-routing.d.ts +11 -0
- package/dist/scenario-routing.js +72 -0
- package/dist/scenario-routing.js.map +1 -0
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/docs/agent-format.md +10 -8
- package/docs/architecture.md +22 -1
- package/docs/assets/qamap-cover.png +0 -0
- package/docs/assets/qamap-cover.svg +56 -0
- package/docs/benchmarking.md +8 -1
- package/docs/e2e-output-examples.md +32 -19
- package/docs/release-validation.md +17 -0
- package/docs/roadmap.md +2 -0
- package/package.json +1 -1
- package/schema/qamap-agent.schema.json +51 -1
package/CHANGELOG.md
CHANGED
|
@@ -2,6 +2,25 @@
|
|
|
2
2
|
|
|
3
3
|
## Unreleased
|
|
4
4
|
|
|
5
|
+
## 0.4.3 - 2026-07-14
|
|
6
|
+
|
|
7
|
+
### Added
|
|
8
|
+
|
|
9
|
+
- Added evidence-ranked QA scenario routing. Each intent-backed scenario is now classified as `required`, `recommended`, or `review-only` from its priority, confidence, review requirement, and exact diff sources instead of treating every inferred path as equally actionable.
|
|
10
|
+
- Added scenario-level automation receipts to human, JSON, and agent output. QAMap reports whether each selected scenario was `compiled`, `partial`, `not-compiled`, or `review-only`, including mapped step and assertion counts plus a concrete blocker when automation is incomplete.
|
|
11
|
+
- Added benchmark contracts for selected-to-compiled coverage so public fixtures fail CI when required QA is proposed without a traceable automation handoff.
|
|
12
|
+
|
|
13
|
+
### Changed
|
|
14
|
+
|
|
15
|
+
- E2E readiness now includes required scenario compilation gaps. A syntactically valid draft cannot be promoted as runnable evidence when a required QA scenario was omitted or only partially mapped.
|
|
16
|
+
- Playwright failure-path generation now requires an unchanged mockable endpoint boundary, an action selector related to the changed flow, and a repository-observed failure outcome before it emits executable setup, action, and assertion steps.
|
|
17
|
+
- Contributor guidance now requires domain-neutral product rules, positive evidence combinations, negative controls, and unrelated fixture coverage before a specialized inference can enter the shared engine.
|
|
18
|
+
|
|
19
|
+
### Fixed
|
|
20
|
+
|
|
21
|
+
- Unrelated controls can no longer be reused merely because they are the first stable selector near a changed endpoint.
|
|
22
|
+
- Maestro automation receipts now reflect commands and assertions that were actually generated instead of assuming coverage from the selected runner alone.
|
|
23
|
+
|
|
5
24
|
## 0.4.2 - 2026-07-13
|
|
6
25
|
|
|
7
26
|
### Added
|
package/README.md
CHANGED
|
@@ -4,6 +4,8 @@
|
|
|
4
4
|
[](https://www.npmjs.com/package/@ivorycanvas/qamap)
|
|
5
5
|
[](LICENSE)
|
|
6
6
|
|
|
7
|
+

|
|
8
|
+
|
|
7
9
|
**A local, zero-LLM PR QA designer. QAMap turns commit intent and code diffs into evidence-backed behavior lifecycles, missing QA, and deterministic automation drafts. No cloud. No source upload. No token.**
|
|
8
10
|
|
|
9
11
|
QAMap reads commit subjects and bodies, git changes, project structure, selectors, existing tests, and optional repo QA memory, then answers the question every reviewer asks: *"What behavior did this PR intend to change, how does that behavior move through the product, and what should we verify before merge?"* Playwright, Maestro, and manual checklists are output adapters after that judgment, not the product recommendation itself.
|
|
@@ -26,7 +28,7 @@ Run one read-only command on a branch. A manifest and test runner are not requir
|
|
|
26
28
|
pnpm dlx @ivorycanvas/qamap qa . --base origin/main --head HEAD
|
|
27
29
|
```
|
|
28
30
|
|
|
29
|
-
The report starts with the inferred behavior and keeps each proposed scenario connected to the commit or exact base/head diff line that caused it. Removed guards and validation remain visible as base-side evidence; each source is labeled `direct`, `supporting`, or `contextual` (trimmed from the committed lifecycle benchmark):
|
|
31
|
+
The report starts with the inferred behavior and keeps each proposed scenario connected to the commit or exact base/head diff line that caused it. Removed guards and validation remain visible as base-side evidence; each source is labeled `direct`, `supporting`, or `contextual`. QAMap then shows two separate receipts: why the scenario was routed as `required`, `recommended`, or `review-only`, and whether the E2E adapter compiled it fully, partially, or not at all (trimmed from the committed lifecycle benchmark):
|
|
30
32
|
|
|
31
33
|
```txt
|
|
32
34
|
Change intent: Submit notification preferences and show the saved state [high]
|
|
@@ -34,15 +36,21 @@ Behavior lifecycle: trigger -> state-change -> side-effect -> observable-outcome
|
|
|
34
36
|
|
|
35
37
|
QA scenarios:
|
|
36
38
|
- [critical] changed preference lifecycle (confidence: high)
|
|
39
|
+
- Routing: required — 3 supporting diff hunks
|
|
40
|
+
- E2E mapping: partial — steps 0/3, assertions 1/1
|
|
37
41
|
- Source: src/pages/preferences.tsx:17, symbol onClick [supporting, head]
|
|
38
42
|
- Source: src/pages/preferences.tsx:8, symbol setSaved [supporting, head]
|
|
39
43
|
- Assert: the saved state becomes observable
|
|
40
44
|
- [recommended] failure, timeout, and retry handling (confidence: medium; review required)
|
|
45
|
+
- Routing: recommended — 1 supporting diff hunk
|
|
46
|
+
- E2E mapping: not compiled — no deterministic failure compiler matched all required evidence
|
|
41
47
|
- Source: src/pages/preferences.tsx:7, symbol fetch [supporting, head]
|
|
42
48
|
- Assert: retries do not duplicate requests or side effects
|
|
43
49
|
|
|
44
50
|
Optional automation:
|
|
45
51
|
- Adapter candidate: Playwright
|
|
52
|
+
- Scenario routing: 2 required, 2 recommended, 0 review-only
|
|
53
|
+
- E2E mapping: 0 compiled, 1 partial, 3 not compiled
|
|
46
54
|
- qamap e2e draft . --base origin/main --head HEAD
|
|
47
55
|
```
|
|
48
56
|
|
|
@@ -76,6 +84,7 @@ One minified JSON object (`schema: qamap.qa`) with change intents, lifecycle sta
|
|
|
76
84
|
- **Judgment first, generation second.** Deciding *what deserves testing* for a given change is the missing layer. QAMap makes that judgment statically, deterministically, and locally for free.
|
|
77
85
|
- **The repo remembers.** Team QA knowledge lives in `.qamap/manifest.yaml`, reviewed once and reused on every PR — instead of re-prompting an agent each session.
|
|
78
86
|
- **Honest output.** Drafts state what blocks them from being trusted; changed endpoints are observed, never mocked away; generated specs never assert what cannot pass. Configuration, docs, generated artifacts, and changed tests stay in verification mode instead of fabricating a product-journey E2E.
|
|
87
|
+
- **Selected is not compiled.** Every evidence-routed scenario keeps a static automation receipt. Required behavior that remains partial or not compiled lowers readiness instead of being hidden behind a smoke test. `compiled` still does not claim the application was executed.
|
|
79
88
|
|
|
80
89
|
Positioning against recorders, LLM test generation, and impact-analysis tools: [where QAMap fits](docs/adoption.md#where-qamap-fits).
|
|
81
90
|
|
package/dist/e2e.d.ts
CHANGED
|
@@ -8,6 +8,7 @@ import type { LocalHistoryReference } from "./history.js";
|
|
|
8
8
|
import type { CoverageEvidence, TestSuiteSummary } from "./test-evidence.js";
|
|
9
9
|
import type { BehaviorGraph } from "./behavior.js";
|
|
10
10
|
import type { BehaviorLifecycleStage, ChangeIntentAnalysis, ChangeIntentConfidence, ChangeIntentEvidence, IntentQaScenario } from "./change-intent.js";
|
|
11
|
+
import type { QaScenarioDecision } from "./scenario-routing.js";
|
|
11
12
|
export type E2eProjectType = "expo-react-native" | "react-native" | "web" | "api-service" | "design-tokens" | "data-catalog" | "cli" | "unknown";
|
|
12
13
|
export type E2eRunnerName = "maestro" | "playwright" | "manual";
|
|
13
14
|
export type E2eFlowKind = "ui" | "api" | "state" | "content" | "config" | "test-evidence" | "documentation" | "generated-artifact" | "artifact" | "catalog" | "command" | "domain" | "changed-file";
|
|
@@ -247,6 +248,23 @@ export interface E2eDraftFile {
|
|
|
247
248
|
intentConfidence?: ChangeIntentConfidence;
|
|
248
249
|
lifecycle?: BehaviorLifecycleStage[];
|
|
249
250
|
qaScenarios?: IntentQaScenario[];
|
|
251
|
+
scenarioAutomation?: E2eScenarioAutomationReceipt[];
|
|
252
|
+
}
|
|
253
|
+
export type E2eScenarioAutomationStatus = "compiled" | "partial" | "not-compiled" | "review-only";
|
|
254
|
+
export interface E2eScenarioAutomationReceipt {
|
|
255
|
+
scenarioId: string;
|
|
256
|
+
title: string;
|
|
257
|
+
kind: IntentQaScenario["kind"];
|
|
258
|
+
priority: IntentQaScenario["priority"];
|
|
259
|
+
decision: QaScenarioDecision;
|
|
260
|
+
status: E2eScenarioAutomationStatus;
|
|
261
|
+
requiredSourceCount: number;
|
|
262
|
+
referenceSourceCount: number;
|
|
263
|
+
mappedSteps: number;
|
|
264
|
+
totalSteps: number;
|
|
265
|
+
mappedAssertions: number;
|
|
266
|
+
totalAssertions: number;
|
|
267
|
+
blockers: string[];
|
|
250
268
|
}
|
|
251
269
|
export type E2eDraftActionKind = "assertion" | "fixture" | "manifest" | "runner" | "selector" | "setup" | "validation";
|
|
252
270
|
export type E2eDraftActionPriority = "required" | "recommended";
|
|
@@ -299,6 +317,13 @@ export interface E2eDraftReadinessSummary {
|
|
|
299
317
|
totalTodos: number;
|
|
300
318
|
filesWithExecutionBlockers: number;
|
|
301
319
|
totalExecutionBlockers: number;
|
|
320
|
+
requiredScenarios: number;
|
|
321
|
+
recommendedScenarios: number;
|
|
322
|
+
reviewOnlyScenarios: number;
|
|
323
|
+
compiledScenarios: number;
|
|
324
|
+
partialScenarios: number;
|
|
325
|
+
notCompiledScenarios: number;
|
|
326
|
+
requiredScenarioGaps: number;
|
|
302
327
|
topBlockers: string[];
|
|
303
328
|
}
|
|
304
329
|
export declare function formatDraftReadinessStage(summary: Pick<E2eDraftReadinessSummary, "level" | "score">): string;
|
package/dist/e2e.js
CHANGED
|
@@ -14,6 +14,7 @@ import { loadCoreFlowManifest, matchCoreFlows } from "./flows.js";
|
|
|
14
14
|
import { loadVerificationManifest, matchVerificationManifest } from "./manifest.js";
|
|
15
15
|
import { collectTestSuiteInventory, evaluateFlowCoverageEvidence, summarizeTestSuiteInventory, } from "./test-evidence.js";
|
|
16
16
|
import { addedDiffTextFromEvidence, collectAddedDiffEvidence, collectAddedDiffText, generateTestPlan, } from "./test-plan.js";
|
|
17
|
+
import { routeQaScenario } from "./scenario-routing.js";
|
|
17
18
|
import { TOOL_NAME, VERSION } from "./version.js";
|
|
18
19
|
// Human reports render readiness as a stage on a fixed journey instead of a
|
|
19
20
|
// verdict, so a first run reads as "you are at the start", not "you failed".
|
|
@@ -222,8 +223,9 @@ export async function generateE2eDraft(rootInput, options = {}) {
|
|
|
222
223
|
const content = shouldSkip ? ((await readTextIfExists(filePath)) ?? "") : draftContentForFlow(plan, flow, runner, addedDiffText);
|
|
223
224
|
const todoCount = countTodos(content);
|
|
224
225
|
const selfCheck = evaluateDraftSelfCheck(plan, flow, runner, content, todoCount);
|
|
225
|
-
const
|
|
226
|
-
const
|
|
226
|
+
const scenarioAutomation = buildScenarioAutomationReceipts(flow, runner, content, selfCheck);
|
|
227
|
+
const actionItems = buildDraftActionItems(plan, flow, runner, validationSummary, promotionGuidance, selfCheck, scenarioAutomation);
|
|
228
|
+
const executionBlockers = draftExecutionBlockers(plan, flow, runner, selfCheck, scenarioAutomation);
|
|
227
229
|
const runnableStatus = draftRunnableStatus(plan, flow, runner, executionBlockers, selfCheck);
|
|
228
230
|
const fileDetails = draftFileDetails(flow);
|
|
229
231
|
if (dryRun) {
|
|
@@ -243,6 +245,7 @@ export async function generateE2eDraft(rootInput, options = {}) {
|
|
|
243
245
|
runnableStatus,
|
|
244
246
|
executionBlockers,
|
|
245
247
|
selfCheck,
|
|
248
|
+
scenarioAutomation,
|
|
246
249
|
todoCount,
|
|
247
250
|
entrypointCount: flow.entrypoints.length,
|
|
248
251
|
primaryEntrypoint: primaryEntrypointLabel(flow),
|
|
@@ -274,6 +277,7 @@ export async function generateE2eDraft(rootInput, options = {}) {
|
|
|
274
277
|
runnableStatus,
|
|
275
278
|
executionBlockers,
|
|
276
279
|
selfCheck,
|
|
280
|
+
scenarioAutomation,
|
|
277
281
|
todoCount,
|
|
278
282
|
entrypointCount: flow.entrypoints.length,
|
|
279
283
|
primaryEntrypoint: primaryEntrypointLabel(flow),
|
|
@@ -304,6 +308,7 @@ export async function generateE2eDraft(rootInput, options = {}) {
|
|
|
304
308
|
runnableStatus,
|
|
305
309
|
executionBlockers,
|
|
306
310
|
selfCheck,
|
|
311
|
+
scenarioAutomation,
|
|
307
312
|
todoCount,
|
|
308
313
|
entrypointCount: flow.entrypoints.length,
|
|
309
314
|
primaryEntrypoint: primaryEntrypointLabel(flow),
|
|
@@ -1019,7 +1024,7 @@ function buildDraftPromotionGuidance(flow) {
|
|
|
1019
1024
|
action: "Create a domain or flow manifest after the first real user journey is identified.",
|
|
1020
1025
|
};
|
|
1021
1026
|
}
|
|
1022
|
-
function buildDraftActionItems(plan, flow, runner, validationSummary, promotionGuidance, selfCheck) {
|
|
1027
|
+
function buildDraftActionItems(plan, flow, runner, validationSummary, promotionGuidance, selfCheck, scenarioAutomation = []) {
|
|
1023
1028
|
const items = [];
|
|
1024
1029
|
const verificationOnly = isVerificationOnlyFlow(flow);
|
|
1025
1030
|
const runnerGap = runnerSetupGap(plan, runner);
|
|
@@ -1061,6 +1066,14 @@ function buildDraftActionItems(plan, flow, runner, validationSummary, promotionG
|
|
|
1061
1066
|
if (!verificationOnly && flow.steps.length > 0 && draftNeedsAssertionWork(selfCheck)) {
|
|
1062
1067
|
items.push(draftActionItem("assertion", "required", "Replace starter smoke assertions with domain assertions", `Preserve the success signal "${flow.languageBrief.successSignal}" while replacing weak fallback interactions and generic expects.`));
|
|
1063
1068
|
}
|
|
1069
|
+
const requiredScenarioGaps = scenarioAutomation.filter((receipt) => receipt.decision === "required" && receipt.status !== "compiled");
|
|
1070
|
+
if (!verificationOnly && requiredScenarioGaps.length > 0) {
|
|
1071
|
+
const details = requiredScenarioGaps.slice(0, 3).map((receipt) => {
|
|
1072
|
+
const blocker = receipt.blockers[0] ?? "the selected scenario has no executable action and assertion mapping";
|
|
1073
|
+
return `"${receipt.title}" is ${receipt.status}: ${blocker}`;
|
|
1074
|
+
});
|
|
1075
|
+
items.push(draftActionItem("assertion", "required", "Compile required QA scenarios into executable coverage", details.join(" ")));
|
|
1076
|
+
}
|
|
1064
1077
|
if (!verificationOnly && selfCheck?.status === "fail") {
|
|
1065
1078
|
items.push(draftActionItem("validation", "required", "Resolve generated draft self-check", selfCheck.blockers[0] ?? selfCheck.summary));
|
|
1066
1079
|
}
|
|
@@ -1223,13 +1236,22 @@ function summarizeDraftReadiness(files, actionSummary) {
|
|
|
1223
1236
|
const totalTodos = files.reduce((sum, file) => sum + (file.todoCount ?? 0), 0);
|
|
1224
1237
|
const filesWithExecutionBlockers = files.filter((file) => (file.executionBlockers?.length ?? 0) > 0).length;
|
|
1225
1238
|
const totalExecutionBlockers = files.reduce((sum, file) => sum + (file.executionBlockers?.length ?? 0), 0);
|
|
1239
|
+
const scenarioAutomation = files.flatMap((file) => file.scenarioAutomation ?? []);
|
|
1240
|
+
const requiredScenarios = scenarioAutomation.filter((receipt) => receipt.decision === "required").length;
|
|
1241
|
+
const recommendedScenarios = scenarioAutomation.filter((receipt) => receipt.decision === "recommended").length;
|
|
1242
|
+
const reviewOnlyScenarios = scenarioAutomation.filter((receipt) => receipt.decision === "review-only").length;
|
|
1243
|
+
const compiledScenarios = scenarioAutomation.filter((receipt) => receipt.status === "compiled").length;
|
|
1244
|
+
const partialScenarios = scenarioAutomation.filter((receipt) => receipt.status === "partial").length;
|
|
1245
|
+
const notCompiledScenarios = scenarioAutomation.filter((receipt) => receipt.status === "not-compiled").length;
|
|
1246
|
+
const requiredScenarioGaps = scenarioAutomation.filter((receipt) => receipt.decision === "required" && receipt.status !== "compiled").length;
|
|
1226
1247
|
const topBlockers = topDraftReadinessBlockers(files);
|
|
1227
1248
|
const statusScore = (runnableCandidates * 100 + nearRunnable * 75 + reviewOnly * 35) / totalFiles;
|
|
1228
1249
|
const selfCheckPenalty = (selfCheckWarning * 10 + selfCheckFail * 25) / totalFiles;
|
|
1229
1250
|
const requiredActionPenalty = Math.min(25, actionSummary.required * 3);
|
|
1230
1251
|
const todoPenalty = Math.min(15, totalTodos);
|
|
1231
1252
|
const blockerPenalty = Math.min(20, filesWithExecutionBlockers * 5);
|
|
1232
|
-
const
|
|
1253
|
+
const scenarioPenalty = Math.min(20, requiredScenarioGaps * 4);
|
|
1254
|
+
const score = clampReadinessScore(Math.round(statusScore - selfCheckPenalty - requiredActionPenalty - todoPenalty - blockerPenalty - scenarioPenalty));
|
|
1233
1255
|
const level = draftReadinessLevel(score, actionSummary.required, selfCheckFail, reviewOnly);
|
|
1234
1256
|
return {
|
|
1235
1257
|
score,
|
|
@@ -1245,6 +1267,13 @@ function summarizeDraftReadiness(files, actionSummary) {
|
|
|
1245
1267
|
totalTodos,
|
|
1246
1268
|
filesWithExecutionBlockers,
|
|
1247
1269
|
totalExecutionBlockers,
|
|
1270
|
+
requiredScenarios,
|
|
1271
|
+
recommendedScenarios,
|
|
1272
|
+
reviewOnlyScenarios,
|
|
1273
|
+
compiledScenarios,
|
|
1274
|
+
partialScenarios,
|
|
1275
|
+
notCompiledScenarios,
|
|
1276
|
+
requiredScenarioGaps,
|
|
1248
1277
|
topBlockers,
|
|
1249
1278
|
};
|
|
1250
1279
|
}
|
|
@@ -1922,9 +1951,13 @@ function appendChangeIntentMarkdown(lines, analysis) {
|
|
|
1922
1951
|
lines.push(`${index + 1}. **${stage.kind}**: ${escapeMarkdownInline(stage.label)} [${stage.confidence}]`);
|
|
1923
1952
|
});
|
|
1924
1953
|
lines.push("");
|
|
1925
|
-
lines.push("
|
|
1954
|
+
lines.push("Routed QA scenarios:");
|
|
1926
1955
|
for (const scenario of intent.scenarios.slice(0, 4)) {
|
|
1956
|
+
const routing = routeQaScenario(scenario);
|
|
1927
1957
|
lines.push(`- **${scenario.priority} / ${scenario.kind}**: ${escapeMarkdownInline(scenario.title)}`);
|
|
1958
|
+
lines.push(` - Routing: ${routing.decision} - ${escapeMarkdownInline(routing.reason)}`);
|
|
1959
|
+
lines.push(` - Evidence: ${routing.requiredEvidence.length} required diff source${routing.requiredEvidence.length === 1 ? "" : "s"}, ` +
|
|
1960
|
+
`${routing.referenceEvidence.length} reference source${routing.referenceEvidence.length === 1 ? "" : "s"}`);
|
|
1928
1961
|
for (const step of scenario.steps.slice(0, 3)) {
|
|
1929
1962
|
lines.push(` - Step: ${escapeMarkdownInline(step)}`);
|
|
1930
1963
|
}
|
|
@@ -2000,6 +2033,11 @@ export function formatMarkdownE2eDraft(result) {
|
|
|
2000
2033
|
lines.push(`- Self-checks: ${result.readinessSummary.selfCheckPass} pass, ${result.readinessSummary.selfCheckWarning} warning, ${result.readinessSummary.selfCheckFail} fail`);
|
|
2001
2034
|
lines.push(`- TODO markers: ${result.readinessSummary.totalTodos} across ${result.readinessSummary.filesWithTodos} file${result.readinessSummary.filesWithTodos === 1 ? "" : "s"}`);
|
|
2002
2035
|
lines.push(`- Execution blockers: ${result.readinessSummary.totalExecutionBlockers} across ${result.readinessSummary.filesWithExecutionBlockers} file${result.readinessSummary.filesWithExecutionBlockers === 1 ? "" : "s"}`);
|
|
2036
|
+
lines.push(`- Scenario routing: ${result.readinessSummary.requiredScenarios} required, ` +
|
|
2037
|
+
`${result.readinessSummary.recommendedScenarios} recommended, ${result.readinessSummary.reviewOnlyScenarios} review-only`);
|
|
2038
|
+
lines.push(`- Scenario automation: ${result.readinessSummary.compiledScenarios} compiled, ` +
|
|
2039
|
+
`${result.readinessSummary.partialScenarios} partial, ${result.readinessSummary.notCompiledScenarios} not compiled; ` +
|
|
2040
|
+
`${result.readinessSummary.requiredScenarioGaps} required gap${result.readinessSummary.requiredScenarioGaps === 1 ? "" : "s"}`);
|
|
2003
2041
|
if (result.readinessSummary.topBlockers.length > 0) {
|
|
2004
2042
|
lines.push("- Top blockers:");
|
|
2005
2043
|
for (const blocker of result.readinessSummary.topBlockers.slice(0, 3)) {
|
|
@@ -2042,6 +2080,22 @@ export function formatMarkdownE2eDraft(result) {
|
|
|
2042
2080
|
}
|
|
2043
2081
|
lines.push("");
|
|
2044
2082
|
}
|
|
2083
|
+
const filesWithScenarioAutomation = result.files.filter((file) => (file.scenarioAutomation?.length ?? 0) > 0);
|
|
2084
|
+
if (filesWithScenarioAutomation.length > 0) {
|
|
2085
|
+
lines.push("## Scenario Automation Receipts");
|
|
2086
|
+
lines.push("");
|
|
2087
|
+
for (const file of filesWithScenarioAutomation) {
|
|
2088
|
+
lines.push(`- \`${escapeMarkdownInline(file.flowTitle)}\` (${escapeMarkdownInline(file.path)})`);
|
|
2089
|
+
for (const receipt of file.scenarioAutomation ?? []) {
|
|
2090
|
+
lines.push(` - [${receipt.decision}] ${escapeMarkdownInline(receipt.title)}: ${receipt.status} ` +
|
|
2091
|
+
`(steps ${receipt.mappedSteps}/${receipt.totalSteps}, assertions ${receipt.mappedAssertions}/${receipt.totalAssertions})`);
|
|
2092
|
+
for (const blocker of receipt.blockers.slice(0, 2)) {
|
|
2093
|
+
lines.push(` - Blocker: ${escapeMarkdownInline(blocker)}`);
|
|
2094
|
+
}
|
|
2095
|
+
}
|
|
2096
|
+
}
|
|
2097
|
+
lines.push("");
|
|
2098
|
+
}
|
|
2045
2099
|
const filesWithActionItems = result.files.filter((file) => (file.actionItems?.length ?? 0) > 0);
|
|
2046
2100
|
if (filesWithActionItems.length > 0) {
|
|
2047
2101
|
lines.push("## Draft Action Items");
|
|
@@ -5421,6 +5475,104 @@ function draftFlowSource(flow) {
|
|
|
5421
5475
|
function domainScenarioForFlow(flow) {
|
|
5422
5476
|
return flow.domainScenario;
|
|
5423
5477
|
}
|
|
5478
|
+
function buildScenarioAutomationReceipts(flow, runner, content, selfCheck) {
|
|
5479
|
+
const scenarios = flow.qaScenarios ?? [];
|
|
5480
|
+
if (scenarios.length === 0) {
|
|
5481
|
+
return [];
|
|
5482
|
+
}
|
|
5483
|
+
const routedPlaywrightScenarios = runner === "playwright" ? playwrightRoutedScenarioDrafts(flow) : [];
|
|
5484
|
+
const primaryContent = content.split("// Routed QA scenario:", 1)[0] ?? content;
|
|
5485
|
+
return scenarios.map((scenario) => {
|
|
5486
|
+
const selection = routeQaScenario(scenario);
|
|
5487
|
+
const base = {
|
|
5488
|
+
scenarioId: scenario.id,
|
|
5489
|
+
title: scenario.title,
|
|
5490
|
+
kind: scenario.kind,
|
|
5491
|
+
priority: scenario.priority,
|
|
5492
|
+
decision: selection.decision,
|
|
5493
|
+
requiredSourceCount: selection.requiredEvidence.length,
|
|
5494
|
+
referenceSourceCount: selection.referenceEvidence.length,
|
|
5495
|
+
totalSteps: scenario.steps.length,
|
|
5496
|
+
totalAssertions: scenario.assertions.length,
|
|
5497
|
+
};
|
|
5498
|
+
if (selection.decision === "review-only" || runner === "manual") {
|
|
5499
|
+
return {
|
|
5500
|
+
...base,
|
|
5501
|
+
status: "review-only",
|
|
5502
|
+
mappedSteps: 0,
|
|
5503
|
+
mappedAssertions: 0,
|
|
5504
|
+
blockers: [
|
|
5505
|
+
selection.decision === "review-only"
|
|
5506
|
+
? selection.reason
|
|
5507
|
+
: "The repository has no executable browser or device adapter for this scenario, so it remains review evidence.",
|
|
5508
|
+
],
|
|
5509
|
+
};
|
|
5510
|
+
}
|
|
5511
|
+
if (scenario.kind !== "primary") {
|
|
5512
|
+
const routed = routedPlaywrightScenarios.find((candidate) => candidate.scenarioId === scenario.id);
|
|
5513
|
+
if (routed) {
|
|
5514
|
+
const fullyMapped = routed.mappedSteps >= scenario.steps.length &&
|
|
5515
|
+
routed.mappedAssertions >= scenario.assertions.length;
|
|
5516
|
+
const blockers = [
|
|
5517
|
+
routed.mappedSteps < scenario.steps.length
|
|
5518
|
+
? `${scenario.steps.length - routed.mappedSteps} selected action step${scenario.steps.length - routed.mappedSteps === 1 ? "" : "s"} remain outside executable coverage.`
|
|
5519
|
+
: undefined,
|
|
5520
|
+
routed.mappedAssertions < scenario.assertions.length
|
|
5521
|
+
? `${scenario.assertions.length - routed.mappedAssertions} selected assertion${scenario.assertions.length - routed.mappedAssertions === 1 ? "" : "s"} remain outside executable coverage.`
|
|
5522
|
+
: undefined,
|
|
5523
|
+
].filter((value) => Boolean(value));
|
|
5524
|
+
return {
|
|
5525
|
+
...base,
|
|
5526
|
+
status: fullyMapped ? "compiled" : "partial",
|
|
5527
|
+
mappedSteps: routed.mappedSteps,
|
|
5528
|
+
mappedAssertions: routed.mappedAssertions,
|
|
5529
|
+
blockers,
|
|
5530
|
+
};
|
|
5531
|
+
}
|
|
5532
|
+
return {
|
|
5533
|
+
...base,
|
|
5534
|
+
status: "not-compiled",
|
|
5535
|
+
mappedSteps: 0,
|
|
5536
|
+
mappedAssertions: 0,
|
|
5537
|
+
blockers: [
|
|
5538
|
+
`No deterministic ${scenario.kind} compiler matched a repository entrypoint, action locator, fixture boundary, and observable outcome.`,
|
|
5539
|
+
],
|
|
5540
|
+
};
|
|
5541
|
+
}
|
|
5542
|
+
return primaryScenarioAutomationReceipt(base, runner, primaryContent, scenario, selfCheck);
|
|
5543
|
+
});
|
|
5544
|
+
}
|
|
5545
|
+
function primaryScenarioAutomationReceipt(base, runner, content, scenario, selfCheck) {
|
|
5546
|
+
const mappedSteps = scenario.steps.filter((step) => content.includes(`Step intent: ${step}`)).length;
|
|
5547
|
+
const assertionStepMarkers = scenario.assertions.filter((assertion) => content.includes(`Step intent: ${assertion}`)).length;
|
|
5548
|
+
const executableAssertions = runner === "maestro"
|
|
5549
|
+
? content.match(/^- assertVisible:\s+(?!["']\.\*["']$).+$/gm)?.length ?? 0
|
|
5550
|
+
: content.match(/await expect\((?!page\.locator\(["']body["']\))/g)?.length ?? 0;
|
|
5551
|
+
const mappedAssertions = Math.min(assertionStepMarkers, executableAssertions);
|
|
5552
|
+
const fallbackCount = content.match(/QAMap could not infer a stable (?:locator|Maestro selector)/g)?.length ?? 0;
|
|
5553
|
+
const blockers = [];
|
|
5554
|
+
if (selfCheck?.status === "fail") {
|
|
5555
|
+
blockers.push(selfCheck.blockers[0] ?? selfCheck.summary);
|
|
5556
|
+
}
|
|
5557
|
+
if (mappedSteps < scenario.steps.length) {
|
|
5558
|
+
blockers.push(`${scenario.steps.length - mappedSteps} selected action step${scenario.steps.length - mappedSteps === 1 ? "" : "s"} did not map to generated commands.`);
|
|
5559
|
+
}
|
|
5560
|
+
if (mappedAssertions < scenario.assertions.length) {
|
|
5561
|
+
blockers.push(`${scenario.assertions.length - mappedAssertions} selected assertion${scenario.assertions.length - mappedAssertions === 1 ? "" : "s"} did not map to an observable generated assertion.`);
|
|
5562
|
+
}
|
|
5563
|
+
if (fallbackCount > 0) {
|
|
5564
|
+
blockers.push(`${fallbackCount} generated step${fallbackCount === 1 ? "" : "s"} still use fallback smoke behavior.`);
|
|
5565
|
+
}
|
|
5566
|
+
const fullyMapped = selfCheck?.status !== "fail" && blockers.length === 0;
|
|
5567
|
+
const partiallyMapped = mappedSteps > 0 || mappedAssertions > 0;
|
|
5568
|
+
return {
|
|
5569
|
+
...base,
|
|
5570
|
+
status: fullyMapped ? "compiled" : partiallyMapped ? "partial" : "not-compiled",
|
|
5571
|
+
mappedSteps,
|
|
5572
|
+
mappedAssertions,
|
|
5573
|
+
blockers: uniqueStrings(blockers),
|
|
5574
|
+
};
|
|
5575
|
+
}
|
|
5424
5576
|
function draftStability(plan, flow) {
|
|
5425
5577
|
const needsSelector = !isVerificationOnlyFlow(flow) && flow.missingTestability.length > 0;
|
|
5426
5578
|
const needsSetup = plan.missingTestability.some((gap) => /No \.maestro|No Playwright config/i.test(gap));
|
|
@@ -5435,7 +5587,7 @@ function draftStability(plan, flow) {
|
|
|
5435
5587
|
}
|
|
5436
5588
|
return "ready";
|
|
5437
5589
|
}
|
|
5438
|
-
function draftExecutionBlockers(plan, flow, runner, selfCheck) {
|
|
5590
|
+
function draftExecutionBlockers(plan, flow, runner, selfCheck, scenarioAutomation = []) {
|
|
5439
5591
|
const verificationOnly = isVerificationOnlyFlow(flow);
|
|
5440
5592
|
const blockers = verificationOnly ? [] : [...plan.executionProfile.blockers];
|
|
5441
5593
|
if (!verificationOnly && runner !== "manual" && flow.entrypoints.length === 0) {
|
|
@@ -5452,6 +5604,12 @@ function draftExecutionBlockers(plan, flow, runner, selfCheck) {
|
|
|
5452
5604
|
if (selfCheck?.status === "fail") {
|
|
5453
5605
|
blockers.push(...selfCheck.blockers);
|
|
5454
5606
|
}
|
|
5607
|
+
if (!verificationOnly && runner !== "manual") {
|
|
5608
|
+
for (const receipt of scenarioAutomation.filter((item) => item.decision === "required" && item.status !== "compiled")) {
|
|
5609
|
+
blockers.push(`Required QA scenario "${receipt.title}" is ${receipt.status}: ` +
|
|
5610
|
+
(receipt.blockers[0] ?? "no executable action and assertion mapping was produced."));
|
|
5611
|
+
}
|
|
5612
|
+
}
|
|
5455
5613
|
return uniqueStrings(blockers).slice(0, 8);
|
|
5456
5614
|
}
|
|
5457
5615
|
function draftRunnableStatus(plan, flow, runner, executionBlockers, selfCheck) {
|
|
@@ -5738,6 +5896,9 @@ function buildMaestroDraft(plan, flow) {
|
|
|
5738
5896
|
appendManifestMatchComments(lines, flow, "#");
|
|
5739
5897
|
for (const step of draftExecutableSteps(flow, "maestro")) {
|
|
5740
5898
|
const command = maestroCommandForStep(step, selectorQueue);
|
|
5899
|
+
if (maestroCommandProvidesCoverage(command)) {
|
|
5900
|
+
lines.push(`# Step intent: ${step}`);
|
|
5901
|
+
}
|
|
5741
5902
|
lines.push(...formatMaestroCommand(command));
|
|
5742
5903
|
}
|
|
5743
5904
|
appendDomainScenarioComments(lines, flow, "#");
|
|
@@ -5766,6 +5927,12 @@ function buildMaestroDraft(plan, flow) {
|
|
|
5766
5927
|
lines.push("");
|
|
5767
5928
|
return lines.join("\n");
|
|
5768
5929
|
}
|
|
5930
|
+
function maestroCommandProvidesCoverage(command) {
|
|
5931
|
+
if (command.kind === "comment") {
|
|
5932
|
+
return false;
|
|
5933
|
+
}
|
|
5934
|
+
return command.kind !== "assertVisible" || command.value !== quoteYaml(".*");
|
|
5935
|
+
}
|
|
5769
5936
|
function maestroCommandForStep(step, selectors) {
|
|
5770
5937
|
if (isGestureStep(step)) {
|
|
5771
5938
|
return {
|
|
@@ -5862,6 +6029,11 @@ function buildPlaywrightDraft(plan, flow, addedDiffText = {}) {
|
|
|
5862
6029
|
appendDomainScenarioComments(lines, flow, " //");
|
|
5863
6030
|
appendPlaywrightCoverageComments(lines, flow);
|
|
5864
6031
|
lines.push("});");
|
|
6032
|
+
for (const routedScenario of playwrightRoutedScenarioDrafts(flow)) {
|
|
6033
|
+
lines.push("");
|
|
6034
|
+
lines.push(`// Routed QA scenario: ${routedScenario.scenarioId}`);
|
|
6035
|
+
lines.push(...routedScenario.lines);
|
|
6036
|
+
}
|
|
5865
6037
|
if (flow.missingTestability.length > 0) {
|
|
5866
6038
|
lines.push("");
|
|
5867
6039
|
lines.push("// Testability gaps to address before this spec is stable:");
|
|
@@ -5886,6 +6058,73 @@ function buildPlaywrightDraft(plan, flow, addedDiffText = {}) {
|
|
|
5886
6058
|
lines.push("");
|
|
5887
6059
|
return lines.join("\n");
|
|
5888
6060
|
}
|
|
6061
|
+
function playwrightRoutedScenarioDrafts(flow) {
|
|
6062
|
+
const routeEntrypoint = primaryRouteEntrypoint(flow);
|
|
6063
|
+
if (!routeEntrypoint) {
|
|
6064
|
+
return [];
|
|
6065
|
+
}
|
|
6066
|
+
const routeDraft = buildPlaywrightRouteDraft(routeEntrypoint.value, flow.entrypoints);
|
|
6067
|
+
if (routeDraft.params.length > 0) {
|
|
6068
|
+
return [];
|
|
6069
|
+
}
|
|
6070
|
+
const endpoint = playwrightFailureMockEndpoint(flow);
|
|
6071
|
+
if (!endpoint) {
|
|
6072
|
+
return [];
|
|
6073
|
+
}
|
|
6074
|
+
const drafts = [];
|
|
6075
|
+
for (const scenario of flow.qaScenarios ?? []) {
|
|
6076
|
+
const selection = routeQaScenario(scenario);
|
|
6077
|
+
if (scenario.kind !== "failure" || selection.decision === "review-only") {
|
|
6078
|
+
continue;
|
|
6079
|
+
}
|
|
6080
|
+
const scenarioText = [scenario.title, ...scenario.steps, ...scenario.edgeCases].join(" ");
|
|
6081
|
+
if (!isFailurePathStep(scenarioText)) {
|
|
6082
|
+
continue;
|
|
6083
|
+
}
|
|
6084
|
+
const selectorContext = `${flow.title} ${scenarioText}`;
|
|
6085
|
+
const actionSelector = routedScenarioSelector(flow.selectors, selectorContext, (selector) => !isInputSelector(selector) && selectorCanDriveInteraction(selector));
|
|
6086
|
+
const outcomeSelector = routedScenarioSelector(flow.selectors, selectorContext, (selector) => selectorCanSupportAssertion(selector) && isFailureOutcomeText(selector.value));
|
|
6087
|
+
if (!actionSelector || !outcomeSelector) {
|
|
6088
|
+
continue;
|
|
6089
|
+
}
|
|
6090
|
+
const testName = `${flow.title}: ${scenario.title}`.replaceAll('"', "'");
|
|
6091
|
+
const routePattern = playwrightMockRoutePattern(endpoint);
|
|
6092
|
+
const status = failureResponseStatus(scenarioText);
|
|
6093
|
+
drafts.push({
|
|
6094
|
+
scenarioId: scenario.id,
|
|
6095
|
+
mappedSteps: Math.min(1, scenario.steps.length),
|
|
6096
|
+
mappedAssertions: Math.min(1, scenario.assertions.length),
|
|
6097
|
+
lines: [
|
|
6098
|
+
`test("${testName}", async ({ page }) => {`,
|
|
6099
|
+
` await page.route("${quoteJs(routePattern)}", async (route) => {`,
|
|
6100
|
+
" await route.fulfill({",
|
|
6101
|
+
` status: ${status},`,
|
|
6102
|
+
' contentType: "application/json",',
|
|
6103
|
+
' body: JSON.stringify({ error: "QAMap simulated failure" }),',
|
|
6104
|
+
" });",
|
|
6105
|
+
" });",
|
|
6106
|
+
` await page.goto(${routeDraft.expression});`,
|
|
6107
|
+
` await ${playwrightLocator(actionSelector)}.click();`,
|
|
6108
|
+
` await expect(${playwrightLocator(outcomeSelector)}).toBeVisible();`,
|
|
6109
|
+
"});",
|
|
6110
|
+
],
|
|
6111
|
+
});
|
|
6112
|
+
}
|
|
6113
|
+
return drafts;
|
|
6114
|
+
}
|
|
6115
|
+
function routedScenarioSelector(selectors, context, predicate) {
|
|
6116
|
+
const keywords = keywordsForStep(context);
|
|
6117
|
+
return selectors
|
|
6118
|
+
.filter(predicate)
|
|
6119
|
+
.map((selector) => ({
|
|
6120
|
+
selector,
|
|
6121
|
+
score: keywords.filter((keyword) => selector.value.toLowerCase().includes(keyword)).length,
|
|
6122
|
+
}))
|
|
6123
|
+
.filter((candidate) => candidate.score > 0)
|
|
6124
|
+
.sort((left, right) => right.score - left.score ||
|
|
6125
|
+
Number(Boolean(right.selector.addedInDiff)) - Number(Boolean(left.selector.addedInDiff)) ||
|
|
6126
|
+
left.selector.value.localeCompare(right.selector.value))[0]?.selector;
|
|
6127
|
+
}
|
|
5889
6128
|
function draftExecutableSteps(flow, runner) {
|
|
5890
6129
|
const executableSteps = flow.steps.filter((step) => !shouldSkipDraftStep(step, flow, runner));
|
|
5891
6130
|
return executableSteps.length > 0 ? executableSteps : flow.steps;
|