@ivorycanvas/qamap 0.4.1 → 0.4.2

package/CHANGELOG.md

@@ -2,6 +2,23 @@
 
 ## Unreleased
 
+## 0.4.2 - 2026-07-13
+
+### Added
+
+- Added benchmark contracts for draft readiness, runnable candidates, self-check passes, review-only files, TODO markers, and execution blockers. Public fixtures can now fail CI when QAMap finds the right flow but produces an unusable automation draft.
+- Added deterministic Playwright failure-path compilation when repository evidence exposes a mockable endpoint, a stable action control, and an observable failure message. Generated drafts replace the success route with a 4xx/5xx response, repeat the user action, and assert the failure UI instead of falling back to a body-visible smoke check.
+
+### Changed
+
+- Draft execution readiness now measures whether a generated file can be tried locally from runner, entrypoint, selector, fixture, and self-check evidence. Missing pre-existing validation coverage remains required PR guidance, but no longer falsely claims that the new file cannot execute.
+- A partial fixture match can qualify as a runnable candidate when QAMap generated executable mock scaffolding and every other execution check passes. It still remains a review item before the draft becomes trusted regression evidence.
+- Playwright self-checks now report body-only smoke assertions as domain-assertion warnings. Such files remain tryable, but cannot be called runnable candidates until they assert an observable product outcome.
+
+### Fixed
+
+- Draft reports no longer tell users to replace TODO locators when the generated files contain no TODO markers.
+
 ## 0.4.1 - 2026-07-13
 
 ### Added

package/dist/e2e.js

@@ -223,8 +223,8 @@ export async function generateE2eDraft(rootInput, options = {}) {
         const todoCount = countTodos(content);
         const selfCheck = evaluateDraftSelfCheck(plan, flow, runner, content, todoCount);
         const actionItems = buildDraftActionItems(plan, flow, runner, validationSummary, promotionGuidance, selfCheck);
-        const executionBlockers = draftExecutionBlockers(plan, flow, runner, validationSummary, selfCheck);
-        const runnableStatus = draftRunnableStatus(plan, flow, runner, validationSummary, executionBlockers, selfCheck);
+        const executionBlockers = draftExecutionBlockers(plan, flow, runner, selfCheck);
+        const runnableStatus = draftRunnableStatus(plan, flow, runner, executionBlockers, selfCheck);
         const fileDetails = draftFileDetails(flow);
         if (dryRun) {
             files.push({
@@ -331,7 +331,7 @@ export async function generateE2eDraft(rootInput, options = {}) {
         files,
         actionSummary,
         readinessSummary: summarizeDraftReadiness(files, actionSummary),
-        nextSteps: buildDraftNextSteps(plan, runner),
+        nextSteps: buildDraftNextSteps(plan, runner, files),
     };
 }
 function buildCoverageTargets(kind, files, runner) {
@@ -1108,7 +1108,8 @@ function draftNeedsAssertionWork(selfCheck) {
         return true;
     }
     return selfCheck.checks.some((check) => (check.name === "Unresolved placeholders" && check.status !== "pass") ||
-        (check.name === "TODO comments" && check.status !== "pass"));
+        (check.name === "TODO comments" && check.status !== "pass") ||
+        (check.name === "Domain assertions" && check.status !== "pass"));
 }
 function runnerSetupGap(plan, runner) {
     if (runner === "playwright") {
@@ -5434,7 +5435,7 @@ function draftStability(plan, flow) {
     }
     return "ready";
 }
-function draftExecutionBlockers(plan, flow, runner, validationSummary, selfCheck) {
+function draftExecutionBlockers(plan, flow, runner, selfCheck) {
     const verificationOnly = isVerificationOnlyFlow(flow);
     const blockers = verificationOnly ? [] : [...plan.executionProfile.blockers];
     if (!verificationOnly && runner !== "manual" && flow.entrypoints.length === 0) {
@@ -5446,15 +5447,14 @@ function draftExecutionBlockers(plan, flow, runner, validationSummary, selfCheck
     if (!verificationOnly && flow.fixtureReadiness.status === "missing") {
         blockers.push(flow.fixtureReadiness.nextActions[0] ?? flow.fixtureReadiness.reason);
     }
-    if (validationSummary.blockingGapCount > 0) {
-        blockers.push(`${validationSummary.blockingGapCount} blocking validation gap${validationSummary.blockingGapCount === 1 ? "" : "s"} ${validationSummary.blockingGapCount === 1 ? "remains" : "remain"}.`);
-    }
+    // Missing validation evidence describes the repository before this draft exists.
+    // Keep it as PR guidance, but do not treat it as proof that the generated file cannot execute.
     if (selfCheck?.status === "fail") {
         blockers.push(...selfCheck.blockers);
     }
     return uniqueStrings(blockers).slice(0, 8);
 }
-function draftRunnableStatus(plan, flow, runner, validationSummary, executionBlockers, selfCheck) {
+function draftRunnableStatus(plan, flow, runner, executionBlockers, selfCheck) {
     if (runner === "manual" || isVerificationOnlyFlow(flow)) {
         return "review-only";
     }
@@ -5462,9 +5462,8 @@ function draftRunnableStatus(plan, flow, runner, validationSummary, executionBlo
         return "review-only";
     }
     if (executionBlockers.length === 0 &&
-        validationSummary.blockingGapCount === 0 &&
         flow.missingTestability.length === 0 &&
-        (flow.fixtureReadiness.status === "ready" || flow.fixtureReadiness.status === "not-needed") &&
+        flow.fixtureReadiness.status !== "missing" &&
         selfCheck?.status === "pass") {
         return "runnable-candidate";
     }
@@ -5502,6 +5501,10 @@ function evaluatePlaywrightDraftSelfCheck(plan, flow, content, todoCount) {
     checks.push(draftSelfCheckItem("TODO comments", todoCount === 0 ? "pass" : "warning", todoCount === 0
         ? "No TODO comments remain in the generated draft."
         : `${todoCount} TODO marker${todoCount === 1 ? "" : "s"} remain for reviewer follow-up.`));
+    const weakSmokeAssertions = content.match(/expect\(page\.locator\(["']body["']\)\)\.toBeVisible\(\)/g)?.length ?? 0;
+    checks.push(draftSelfCheckItem("Domain assertions", weakSmokeAssertions === 0 ? "pass" : "warning", weakSmokeAssertions === 0
+        ? "The draft does not rely on body-only smoke assertions."
+        : `${weakSmokeAssertions} body-only smoke assertion${weakSmokeAssertions === 1 ? "" : "s"} must be replaced with observable domain outcomes.`));
     checks.push(draftSelfCheckItem("Execution profile", plan.executionProfile.confidence === "low" || plan.executionProfile.blockers.length > 0 ? "warning" : "pass", plan.executionProfile.confidence === "low" || plan.executionProfile.blockers.length > 0
         ? "Runner setup, base URL, launch command, or config evidence is incomplete."
         : "Runner setup evidence is strong enough for a local execution attempt."));
@@ -5846,8 +5849,10 @@ function buildPlaywrightDraft(plan, flow, addedDiffText = {}) {
     for (const step of draftExecutableSteps(flow, "playwright")) {
         const manifestCheck = manifestCheckForDraftStep(flow, step);
         const manifestBody = manifestCheck ? playwrightActionForManifestCheck(manifestCheck, step) : undefined;
-        const selector = manifestBody ? undefined : takeSelectorForStep(selectorQueue, step);
+        const failureBody = manifestBody ? undefined : playwrightFailureActionForStep(flow, step);
+        const selector = manifestBody || failureBody ? undefined : takeSelectorForStep(selectorQueue, step);
         const body = manifestBody ??
+            failureBody ??
             (selector
                 ? playwrightActionForStep(selector, playwrightLocator(selector), step)
                 : playwrightFallbackActionForStep(step));
@@ -6833,19 +6838,24 @@ function formatDraftActionKindSummary(summary) {
         .map((item) => `${item.kind} ${item.total} (${item.required} required, ${item.recommended} recommended)`)
         .join(", ");
 }
-function buildDraftNextSteps(plan, runner) {
+function buildDraftNextSteps(plan, runner, files) {
     const steps = [];
+    const hasTodos = files.some((file) => (file.todoCount ?? 0) > 0);
     if (runner === "maestro") {
         steps.push(plan.executionProfile.appId
             ? `Use app id ${plan.executionProfile.appId} or export APP_ID before running Maestro.`
             : "Replace ${APP_ID} or export APP_ID before running Maestro.");
-        steps.push("Replace TODO text selectors with visible copy, testID, or accessibilityLabel selectors.");
+        if (hasTodos) {
+            steps.push("Replace TODO text selectors with visible copy, testID, or accessibilityLabel selectors.");
+        }
         steps.push(plan.executionProfile.startCommand
             ? `Launch the app with \`${plan.executionProfile.startCommand}\`, then run \`${plan.executionProfile.testCommand ?? "maestro test .maestro"}\`.`
             : `Run the app with the launch command that matches your simulator or device, then run \`${plan.executionProfile.testCommand ?? "maestro test .maestro"}\`.`);
     }
     else if (runner === "playwright") {
-        steps.push("Replace TODO locators with role, text, or data-testid locators from the app.");
+        if (hasTodos) {
+            steps.push("Replace TODO locators with role, text, or data-testid locators from the app.");
+        }
         if (plan.executionProfile.blockers.some((blocker) => /No Playwright config/i.test(blocker))) {
             steps.push(playwrightConfigGuidance(plan.executionProfile));
         }
@@ -7124,6 +7134,53 @@ function playwrightActionForStep(selector, locator, step) {
     body.push(`await ${locator}.click();`);
     return body;
 }
+function playwrightFailureActionForStep(flow, step) {
+    if (!isFailurePathStep(step)) {
+        return undefined;
+    }
+    const endpoint = playwrightFailureMockEndpoint(flow);
+    const actionSelector = flow.selectors.find((selector) => !isInputSelector(selector) && selectorCanDriveInteraction(selector));
+    const outcomeSelector = flow.selectors.find((selector) => selectorCanSupportAssertion(selector) && isFailureOutcomeText(selector.value));
+    if (!endpoint || !actionSelector || !outcomeSelector) {
+        return undefined;
+    }
+    const routePattern = playwrightMockRoutePattern(endpoint);
+    const status = failureResponseStatus(step);
+    return [
+        `// Step intent: ${step}`,
+        `await page.unroute("${quoteJs(routePattern)}");`,
+        `await page.route("${quoteJs(routePattern)}", async (route) => {`,
+        "  await route.fulfill({",
+        `    status: ${status},`,
+        '    contentType: "application/json",',
+        '    body: JSON.stringify({ error: "QAMap simulated failure" }),',
+        "  });",
+        "});",
+        `await ${playwrightLocator(actionSelector)}.click();`,
+        `await expect(${playwrightLocator(outcomeSelector)}).toBeVisible();`,
+    ];
+}
+function playwrightFailureMockEndpoint(flow) {
+    const observedEndpoints = observedEndpointsForFlow(flow);
+    return flow.fixtureReadiness.apiEndpoints.find((endpoint) => !endpointMatchesAny(endpoint, observedEndpoints));
+}
+function isFailurePathStep(step) {
+    return /\b(?:blocked|declined|denied|empty|error|failed|failure|invalid|rejected|timeout|unauthori[sz]ed)\b/i.test(step) ||
+        /(?:오류|실패|거절|권한|잘못|할 수 없)/.test(step);
+}
+function isFailureOutcomeText(value) {
+    return /\b(?:cannot|could not|declined|denied|error|failed|failure|invalid|rejected|try again|unauthori[sz]ed|unavailable)\b/i.test(value) ||
+        /(?:오류|실패|거절|권한|잘못|할 수 없)/.test(value);
+}
+function failureResponseStatus(step) {
+    if (/\b(?:denied|unauthori[sz]ed)\b/i.test(step) || /권한/.test(step)) {
+        return 401;
+    }
+    if (/\b(?:invalid|validation)\b/i.test(step) || /잘못/.test(step)) {
+        return 422;
+    }
+    return 500;
+}
 function manifestCheckForDraftStep(flow, step) {
     const checks = flow.manifestCheckMatches ?? [];
     const stepKey = normalizeManifestStepKey(step);