@skyramp/mcp 0.1.0 → 0.1.2

package/build/prompts/enhance-assertions/uiAssertionsPrompt.js

@@ -1,90 +1,229 @@
 import { getPersonaPrefix } from "../personas.js";
-export const UI_ASSERTIONS_PROMPT = `${getPersonaPrefix()}Your task is to enhance assertions for the given UI test.
+import { maintenanceTaskSuffix, renderRule, } from "./sharedAssertionRules.js";
+const UI_ASSERTION_CATEGORIES = [
+    {
+        name: "Critical UI Assertions",
+        rules: [
+            {
+                title: "Selector constraints",
+                description: "Every assertion uses a selector already in the file. Never invent `data-testid`, role names, or classes.",
+                subPoints: [
+                    "No tautological assertions — locating an element by text X, then asserting it contains X.",
+                ],
+                examples: [
+                    {
+                        language: "javascript",
+                        code: `await expect(page.getByTestId('badge-count')).toHaveText('3');`,
+                    },
+                ],
+            },
+            {
+                title: "Page errors",
+                description: "Register `page.on('pageerror', ...)` before the first navigation and assert `expect(errors).toHaveLength(0)` at the end of the test.",
+                examples: [],
+            },
+            {
+                title: "Correct intended behavior",
+                description: "Assertions encode the expected outcome of the action, not the buggy runtime output observed in the trace.",
+                examples: [
+                    {
+                        language: "javascript",
+                        code: `await expect(page.locator('[data-testid="session-row"]')).toHaveCount(3);`,
+                    },
+                ],
+            },
+            {
+                title: "Tests target the changed behavior introduced by the pull request",
+                description: "At least one assertion targets the changed behavior on populated or updated state, not only empty or zero state.",
+                examples: [],
+            },
+            {
+                title: "Collection / repeated UI elements",
+                description: "For every collection page or selector matching multiple rendered nodes (items, rows, cards, badges, tabs, chips, definition rows, nav items), assert the exact count plus per-item content.",
+                subPoints: [
+                    "Use `toHaveCount(N)` where N comes from the trace array length or rendered DOM (`0` for empty state).",
+                    "Add at least one concrete per-item content assertion (`toHaveText`, `toHaveValue`, `toHaveAttribute`).",
+                    "Empty trace → `toHaveCount(0)` paired with the empty-state text.",
+                ],
+                examples: [
+                    {
+                        language: "javascript",
+                        code: `await expect(page.getByRole('row')).toHaveCount(2);
+await expect(page.getByRole('cell', { name: 'admin@example.com' })).toHaveText('admin@example.com');`,
+                    },
+                    {
+                        language: "javascript",
+                        code: `await expect(page.getByTestId('notification-badge')).toHaveText('3');
+await expect(page.getByTestId('notification-row')).toHaveCount(3);`,
+                    },
+                    {
+                        language: "javascript",
+                        code: `await expect(page.getByTestId('definition-row')).toHaveCount(3);
+await expect(page.getByTestId('definition-row').nth(0)).toHaveText('Version 10.2.1');`,
+                    },
+                ],
+            },
+            {
+                title: "Positive-path companion",
+                description: "Negative-only checks and URL-only routing tests must include at least one positive rendered-element assertion such as a heading, breadcrumb, title, link, count, or exact text.",
+                examples: [
+                    {
+                        language: "javascript",
+                        code: `await expect(page).toHaveURL(/\\/users/);
+await expect(page.getByRole('heading')).toHaveText('User Directory');`,
+                    },
+                ],
+            },
+            {
+                title: "Captured network responses",
+                description: "For every `page.waitForResponse`, `page.on`, or `page.route`, assert at least one status, body, or header field. Body values used downstream must also appear as visible UI text.",
+                examples: [
+                    {
+                        language: "javascript",
+                        code: `const loginResp = await page.waitForResponse('**/login');
+expect(loginResp.status()).toBe(200);
+expect((await loginResp.json()).user.email).toBe('admin@example.com');`,
+                    },
+                ],
+            },
+        ],
+    },
+    {
+        name: "Computed Values",
+        rules: [
+            {
+                title: "Exact rendered values",
+                description: "When the exact text, value, or attribute is knowable from the trace or source, use `toHaveText`, `toHaveValue`, or `toHaveAttribute` — never `toBeVisible` or `toContainText`.",
+                subPoints: [
+                    "Definition / info pages: assert the actual displayed values (version, count, date), not just the static label text.",
+                    "Images: assert `toHaveAttribute('src', ...)` with the exact src — never visibility alone.",
+                ],
+                examples: [
+                    {
+                        language: "javascript",
+                        code: `await expect(page.getByTestId('status')).toHaveText('Active');`,
+                    },
+                    {
+                        language: "javascript",
+                        code: `await expect(page.getByTestId('product-image')).toHaveAttribute('src', /expected-pattern/);`,
+                    },
+                ],
+            },
+        ],
+    },
+    {
+        name: "Post-edit State",
+        description: "Apply only when the test contains a state-changing action. Otherwise mark NOT APPLICABLE.",
+        rules: [
+            {
+                title: "Post-action visible state",
+                description: "After a state-changing action (form fill+submit, save / delete / create / toggle, checkbox click, hover / mouseout, refresh, reload, JS update, form edit), assert the visible updated outcome via `toHaveText`, `toHaveValue`, `toHaveAttribute`, or `toBeChecked`. The displayed values must reflect the edit.",
+                subPoints: [
+                    "Toggle / checkbox renames: assert the new label or state, not `toBeVisible` on the renamed element.",
+                    "Hover / mouseout: assert the overlay or state visible / hidden after the event.",
+                    "List edit: when the test submits N items, assert exactly N rows. When the count is unknown before the action, read it before and assert the delta after.",
+                ],
+                examples: [
+                    {
+                        language: "javascript",
+                        code: `await expect(page.getByTestId('description-checkbox')).toBeChecked();
+await expect(page.getByTestId('description-label')).toHaveText('check_box Description');`,
+                    },
+                    {
+                        language: "javascript",
+                        code: `await expect(page.getByTestId('total')).toHaveText('$19.98');`,
+                    },
+                    {
+                        language: "javascript",
+                        code: `await expect(page.getByTestId('version')).toHaveText('10.2.1');`,
+                    },
+                ],
+            },
+        ],
+    },
+];
+function renderCategories(categories) {
+    let index = 0;
+    return categories
+        .map((category) => {
+        const rendered = category.rules
+            .map((rule) => {
+            index += 1;
+            return renderRule(index, rule);
+        })
+            .join("\n\n");
+        const header = `#### ${category.name}`;
+        return category.description
+            ? `${header}\n${category.description}\n\n${rendered}`
+            : `${header}\n${rendered}`;
+    })
+        .join("\n\n");
+}
+function renderAssertionCategoriesTemplate(categories) {
+    return categories
+        .map((category) => {
+        const ruleKeys = category.rules
+            .map((rule) => `      "${rule.title}": []`)
+            .join(",\n");
+        return `    "${category.name}": {\n${ruleKeys}\n    }`;
+    })
+        .join(",\n");
+}
+export function getUIAssertionsPrompt(testFile, enhanceType) {
+    const categoryTemplate = renderAssertionCategoriesTemplate(UI_ASSERTION_CATEGORIES);
+    return `${getPersonaPrefix()}Your task is to enhance assertions for the given UI test file: \`${testFile}\`.${maintenanceTaskSuffix(enhanceType)}
+
 ### First Check
 If the generated test file has no \`expect()\` assertions, you MUST manually add them before anything else. Use \`import { expect } from '@skyramp/skyramp';\` — never from \`@playwright/test\`. If an existing import pulls \`expect\` from \`@playwright/test\`, move it to \`@skyramp/skyramp\` (keep \`test\` on the playwright line).
 
-### Top Priorities
-You MUST output a \`<thinking>\` block that explicitly confirms each of these for the file:
-1. **Selector inventory** — list every selector already present in the generated test file (\`data-testid\`, role + name, text, label, etc.). New assertions may use ONLY selectors from this list. Do NOT invent \`data-testid\` values, role names, or aria attributes.
-2. **Process — Replay → Identify → Fix or Add** — walk through these three steps explicitly:
-   a. **Replay the scenario mentally**: at each state-changing action (form submit, item add/edit/delete), ask: "What is the EXPECTED outcome based on the action performed?"
-   b. **Identify expectation mismatches**: if the recorded trace shows a result that contradicts the action (e.g. removing 1 of 2 items but the page shows 3, submitting a form but getting a blank page, editing a field but the old value persists), that is an app bug the test should catch. List every mismatch you find.
-   c. **Fix or add assertions** for each mismatch:
-      - If an existing assertion uses the WRONG (buggy) value, edit it to assert the CORRECT expected value.
-      - If no assertion exists for the buggy behavior, ADD one immediately after the action that triggers it.
-3. Collection / list / grid / table page → exact \`toHaveCount(N)\` OR explicit empty-state text + zero-count assertion. Heading or toolbar visibility is never enough.
-   a. **\`toHaveCount(N)\` is MANDATORY whenever the test interacts with ANY repeated element** — rows, cards, list items, badges, nav links, breadcrumbs, definition rows, tab items, chips, table rows, accordion panels, etc. Use exact \`toHaveCount(N)\` (or \`toHaveCount(0)\` + empty-state text for the empty case). Single-element pages are exempt. Heading or toolbar visibility is NEVER enough when repeated elements are on screen.
-4. Every known exact rendered value uses \`toHaveText('...')\`, not \`toBeVisible()\` or a regex.
-5. Every state-changing action (submit, add, edit, delete) has a post-action assertion of the UPDATED state — not the pre-action state.
-6. \`page.on('pageerror', ...)\` listener registered BEFORE the first navigation; \`expect(errors).toHaveLength(0)\` at the end.
-7. Computed/dynamic values (totals, counts, badges) asserted with exact expected values after each mutation.
-8. Assertions encode the intended correct behavior. Buggy runtime errors, stale state, or crash output are only asserted when the intended UX is explicitly an error state.
-9. At least one assertion targets the PR's core changed behavior. If the PR feature only activates with data (e.g. notifications badge, cart count, unread count), the test MUST create or seed that data — never assert only the empty/zero/default state.
-
-### Assertion Strength (use the strongest applicable)
-- ✅ STRONG: \`toHaveCount(N)\`, \`toHaveText('Exact Value')\`, \`toHaveValue('input value')\`, \`toHaveAttribute('src', ...)\`
-- ⚠️ PARTIAL: \`toContainText('foo')\` — only when the full string is genuinely dynamic
-- ❌ WEAKEST: \`toBeVisible()\` — only when presence/absence is the actual test
-
-### Strategic Placement
-1. Collection pages: exact item/row/card count, OR empty-state text + zero-count, plus at least one concrete content assertion (cell, card, or text).
-  ❌ BAD (collection page validated only by heading): \`expect(page.getByRole("heading", { name: "Activity Feed" })).toHaveText("Activity Feed")\`
-  ❌ BAD (collection page validated only by toolbar): \`expect(page.getByRole("button", { name: "search" })).toBeVisible()\`
-  ✅ GOOD: \`expect(page.getByRole("row")).toHaveCount(2)\` plus \`expect(page.getByRole("cell", { name: "admin@example.com" })).toHaveText("admin@example.com")\`
-2. Read-only collection traces: still required to use the strongest read-only assertions above. Do not fall back to heading-only.
-3. After each state-changing action: assert the visible UPDATED outcome (item appears, total recalculates, count changes).
-  ❌ BAD (visibility when the recalculated total is known): \`expect(page.getByTestId('total')).toBeVisible()\`
-  ✅ GOOD: \`expect(page.getByTestId('total')).toHaveText('$19.98')\`
+### Pre-Edit Assertion Analysis
+Before editing the given test file, you must output a \`<thinking>\` block. The aim of the \`<thinking>\` block is to analyze each in-scope item (action, selector, or captured network response) in the given test file and output a JSON array that ensures no assertion rule is overlooked. The JSON array should match the template below — every assertion category and every rule title under it must appear as a key, even when the value is \`[]\`.
+1. Selector inventory — list every selector already present in the generated test file (\`data-testid\`, role + name, text, label, etc.). New assertions may use only selectors from this list. Do not invent \`data-testid\` values, role names, or aria attributes. Also note captured network responses, repeated element patterns, exact rendered text/value/attribute from trace/source, and existing \`toBeVisible()\` assertions whose exact text is knowable.
+2. Process — Replay → Identify → Fix or Add — walk through these three steps explicitly.
+   a. Replay the scenario mentally. At each state-changing action (form submit, item add/edit/delete), ask: "What is the EXPECTED outcome based on the action performed?"
+   b. Identify expectation mismatches. If the recorded trace shows a result that contradicts the action (e.g. removing 1 of 2 items but the page shows 3, submitting a form but getting a blank page, editing a field but the old value persists), that is an app bug the test should catch. List every mismatch you find.
+   c. Fix or add assertions for each mismatch.
+      - If an existing assertion uses the wrong (buggy) value, edit it to assert the correct expected value.
+      - If no assertion exists for the buggy behavior, add one immediately after the action that triggers it.
+3. Classify each in-scope action / selector / captured response by its applicable assertion category, marking each category APPLICABLE or NOT APPLICABLE.
+   - Critical UI Assertions applies when there is a collection / repeated element (\`toHaveCount\`), a pageerror handler, a captured network response, a negative-only or URL-only test needing a positive-path companion, or a tautological locator-by-text + assert-text to replace.
+   - Computed Values applies when there is a knowable exact text / value / attribute, including a definition-page displayed value or an image \`src\`.
+   - Post-edit State applies only when the test contains a state-changing action (form fill+submit, save / delete / create / toggle, checkbox click, hover / mouseout, refresh, reload, JS update, form edit); otherwise NOT APPLICABLE.
+4. For each in-scope item, output one JSON object using the template below. The output is an array — repeat the object template below once per in-scope item.
+   - \`action_or_selector_or_response\`: the selector, action, or captured network response this entry covers.
+   - \`assertion_categories\`: an object that MUST contain every category name below as a key. The value of each category is itself an object that MUST contain every rule title under that category as a key. For each rule, the value is an array of assertion lines you will add for this item under that rule. Use \`[]\` only when the rule does not apply to this item — every category key and every rule key must still be present. This forces you to consider every rule for every item.
 
-  ❌ BAD (only confirms the placeholder is gone): \`expect(page.getByTestId('version')).not.toHaveText('—')\`
-  ✅ GOOD: \`expect(page.getByTestId('version')).toHaveText('10.2.1')\`
-4. Refresh / reload: assert the intended updated state still renders, not stale or error output.
-5. Forms: assert displayed values reflect the edit, not the pre-edit state.
-6. Lists modified by edits: assert exact item count after the edit (e.g. submitted 2 items → exactly 2 rows).
-7. Lists with unknown exact count (test does not control the data): read the count before the action and assert it changed by the expected delta after.
-8. Repeated UI elements (badges with counts, nav menu items, breadcrumb segments, definition rows on a panel/info page, tab list, chip group, accordion sections): assert \`toHaveCount(N)\` on the parent locator. Visibility on the container is never enough.
-  ❌ BAD (badge with a numeric count, only label asserted): \`expect(page.getByText('Notifications')).toBeVisible()\`
-  ✅ GOOD: \`expect(page.getByTestId('notification-badge')).toHaveText('3')\` PLUS \`expect(page.getByTestId('notification-row')).toHaveCount(3)\`
+\`\`\`json
+[{
+  "action_or_selector_or_response": "<selector | action | response>",
+  "assertion_categories": {
+${categoryTemplate}
+  }
+}]
+\`\`\`
 
-  ❌ BAD (info/definition page with multiple rows, no count assertion): \`expect(page.getByRole('heading', { name: 'Server Info' })).toBeVisible()\`
-  ✅ GOOD: \`expect(page.getByTestId('definition-row')).toHaveCount(3)\` PLUS \`toHaveText\` on each definition value (version string, Node version, OS string)
-9. Image-affecting PRs: assert \`src\` attribute, not visibility.
-  ❌ BAD (visibility does not verify the correct image): \`expect(page.getByTestId('product-image')).toBeVisible()\`
-  ✅ GOOD: \`expect(page.getByTestId('product-image')).toHaveAttribute('src', /expected-pattern/)\`
-10. Dynamic JS updates (no reload): assert the updated value immediately after the action.
-11. Captured network responses (\`page.waitForResponse\`, \`page.on('response', ...)\`, \`page.route()\`): assert at least one field from the captured response — status, body value, or header. A captured-but-unasserted response is a no-op.
-  ❌ BAD (response captured but never asserted): \`const loginResp = await page.waitForResponse('**/login')\`
-  ✅ GOOD: \`const loginResp = await page.waitForResponse('**/login'); expect(loginResp.status()).toBe(200); expect((await loginResp.json()).user.email).toBe('admin@example.com')\`
+### Assertion Rules with Examples
+Most-violated patterns — apply every time. (1) repeated elements → \`toHaveCount(N)\` + per-item \`toHaveText\`/\`toHaveValue\`/\`toHaveAttribute\` (Collection / repeated UI elements). (2) post-action state → \`toHaveText\`/\`toHaveValue\`/\`toHaveAttribute\`/\`toBeChecked\`, not \`toBeVisible\` when exact value is knowable (Post-action visible state). (3) routing/URL-only tests → at least one rendered-element exact text (Positive-path companion).
+Strength order. \`toHaveCount\`/\`toHaveText\`/\`toHaveValue\`/\`toHaveAttribute\` > \`toContainText\` (only when string is genuinely dynamic) > \`toBeVisible\` (only when presence is the actual test).
 
-### What NOT to Assert
-- Static page headings or boilerplate labels
-- Intermediate states (typing, dropdown opening)
-- Values already guaranteed by the action you just took
-- The same value with multiple selectors
-- Tautological assertions — locating an element by text X, then asserting it contains X
-  ❌ BAD: \`page.getByText('My Activity').toContainText('My Activity')\`
-  ✅ GOOD: \`expect(page.getByTestId('badge-count')).toHaveText('3')\`
-- Buggy error text or crash output as the expected result (unless the intended UX is explicitly an error state)
-  ❌ BAD (asserting the bug as expected output): \`expect(page.getByText('s.expires.toISOString is not')).toHaveText('s.expires.toISOString is not a function')\`
-  ✅ GOOD: assert the correct post-action UI state (e.g. \`expect(page.locator('[data-testid="session-row"]')).toHaveCount(3)\`) and let the test fail when the bug is present
+${renderCategories(UI_ASSERTION_CATEGORIES)}
 
-### Verification of enhanced assertions
-1. Every new assertion uses a selector that already appears in the generated file; no invented \`data-testid\`, role, or aria values
-2. Every action-vs-trace mismatch identified during replay is encoded as either a corrected existing assertion or a new assertion right after the triggering action
-3. **[CRITICAL] Every repeated-element selector the test interacted with has exact \`toHaveCount(N)\` (or \`toHaveCount(0)\` + empty-state text) PLUS at least one concrete content assertion.** Repeated elements include rows, cards, list items, badges, nav links, breadcrumbs, definition rows, tab items, chips, table rows, accordion panels. No bare visibility/heading-only assertions remain on repeated elements.
-4. Every known exact value uses \`toHaveText\` instead of \`toBeVisible\` or a regex
-5. Every state-changing action has a post-action assertion of the UPDATED state
-6. \`pageerror\` listener registered before first navigation; \`expect(errors).toHaveLength(0)\` at end
-7. Computed/dynamic values asserted with exact expected values after mutation
-8. Assertions encode the intended correct behavior; buggy output only asserted for explicit error-state UX
-9. At least one assertion targets the PR's core changed behavior
-10. Test exercises a state-mutating flow — not just the empty/zero/default state when the PR feature requires data to activate
-11. No tautological assertions (element found by text X then asserts it contains X)
-12. Image assertions use \`toHaveAttribute('src', ...)\` not just \`toBeVisible()\` when the PR affects images
-13. Every captured network response (via \`page.waitForResponse\` / \`page.on('response')\` / \`page.route()\`) has at least one field asserted — status, body value, or header
-14. \`expect\` imported from \`@skyramp/skyramp\`, not \`@playwright/test\`
+### What Not to Do
+- Do not invent selectors, values, attributes, or assertions
+- Do not use \`toBeVisible\` when exact text/value/state is knowable
+- Do not use tautological locator-by-text + assert-text
+- Do not use \`toContainText\` when the full string is known
+- Do not assert static page headings, intermediate states, or values guaranteed by the action you just took
+- Do not assert the same value with multiple selectors
+- Do not import \`expect\` from \`@playwright/test\`
+- Do not restructure, reorder, or modify existing code/imports/signatures
+- Do not add comments or docstrings
+- Do not remove or modify existing assertions
+- Do not assert buggy/error text as expected (unless the intended UX is an error state)
 
-An item passes verification only when the assertion is present AND is a good assertion per the rules above. If any item is not satisfied — assertion missing, OR present but a bad assertion — add or fix it per the rules before completing, preferring the strongest applicable matcher (see \`### Assertion Strength\`).
+### Verification of Assertions
+After adding all assertion lines in the given test file, verify that every applicable rule has been applied correctly to each in-scope action, selector, and captured response. If any are missing or weakly applied, fix them before completing.
 
 The goal is tests that FAIL when the app has bugs, not tests that simply replay what happened.
 `;
+}