@cementic/cementic-test 0.2.11 → 0.2.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -0
- package/dist/cli.js +1043 -70
- package/dist/cli.js.map +1 -1
- package/package.json +1 -1
package/dist/cli.js
CHANGED
|
@@ -501,39 +501,427 @@ function buildProviderConfigs(env) {
|
|
|
501
501
|
function getAvailableCtVarKeys() {
|
|
502
502
|
return Object.keys(process.env).filter((key) => /^CT_VAR_[A-Z0-9_]+$/.test(key)).sort();
|
|
503
503
|
}
|
|
504
|
+
var RULE_10_PLAYWRIGHT_KNOWLEDGE_BASE = `
|
|
505
|
+
RULE 10 - PLAYWRIGHT KNOWLEDGE BASE
|
|
506
|
+
(sourced from playwright.dev official documentation)
|
|
507
|
+
|
|
508
|
+
You are an expert in Playwright TypeScript. Apply this knowledge
|
|
509
|
+
when generating test code. Always use web-first assertions that
|
|
510
|
+
auto-wait. Never use manual waits or non-awaited assertions.
|
|
511
|
+
|
|
512
|
+
LOCATOR PRIORITY (use in this order)
|
|
513
|
+
1. page.getByRole('button', { name: 'Submit' }) <- best
|
|
514
|
+
2. page.getByLabel('Email address')
|
|
515
|
+
3. page.getByPlaceholder('Enter email')
|
|
516
|
+
4. page.getByText('Welcome')
|
|
517
|
+
5. page.getByAltText('logo')
|
|
518
|
+
6. page.getByTitle('Close')
|
|
519
|
+
7. page.getByTestId('submit-btn')
|
|
520
|
+
8. page.locator('#id') or page.locator('.class') <- last resort
|
|
521
|
+
|
|
522
|
+
Chain locators to narrow scope:
|
|
523
|
+
page.getByRole('listitem').filter({ hasText: 'Product 2' })
|
|
524
|
+
.getByRole('button', { name: 'Add to cart' }).click()
|
|
525
|
+
|
|
526
|
+
LOCATOR ASSERTIONS (always await, always web-first)
|
|
527
|
+
Visibility:
|
|
528
|
+
await expect(locator).toBeVisible()
|
|
529
|
+
await expect(locator).toBeHidden()
|
|
530
|
+
await expect(locator).toBeInViewport()
|
|
531
|
+
|
|
532
|
+
State:
|
|
533
|
+
await expect(locator).toBeEnabled()
|
|
534
|
+
await expect(locator).toBeDisabled()
|
|
535
|
+
await expect(locator).toBeChecked()
|
|
536
|
+
await expect(locator).not.toBeChecked()
|
|
537
|
+
await expect(locator).toBeFocused()
|
|
538
|
+
await expect(locator).toBeEditable()
|
|
539
|
+
await expect(locator).toBeEmpty()
|
|
540
|
+
await expect(locator).toBeAttached()
|
|
541
|
+
|
|
542
|
+
Text content:
|
|
543
|
+
await expect(locator).toHaveText('exact text')
|
|
544
|
+
await expect(locator).toHaveText(/regex/)
|
|
545
|
+
await expect(locator).toContainText('partial')
|
|
546
|
+
await expect(locator).toContainText(['item1', 'item2'])
|
|
547
|
+
|
|
548
|
+
Value and attributes:
|
|
549
|
+
await expect(locator).toHaveValue('input value')
|
|
550
|
+
await expect(locator).toHaveValues(['opt1', 'opt2']) // multi-select
|
|
551
|
+
await expect(locator).toHaveAttribute('href', /pattern/)
|
|
552
|
+
await expect(locator).toHaveClass(/active/)
|
|
553
|
+
await expect(locator).toHaveCSS('color', 'rgb(0,0,0)')
|
|
554
|
+
await expect(locator).toHaveId('submit-btn')
|
|
555
|
+
await expect(locator).toHaveAccessibleName('Submit form')
|
|
556
|
+
await expect(locator).toHaveAccessibleDescription('...')
|
|
557
|
+
|
|
558
|
+
Counting (PREFER toHaveCount over .count() to avoid flakiness):
|
|
559
|
+
await expect(locator).toHaveCount(3)
|
|
560
|
+
await expect(locator).toHaveCount(0) // none exist
|
|
561
|
+
// Only use .count() when you need the actual number:
|
|
562
|
+
const n = await page.getByRole('button').count();
|
|
563
|
+
console.log(\`Found \${n} buttons\`);
|
|
564
|
+
|
|
565
|
+
PAGE ASSERTIONS
|
|
566
|
+
await expect(page).toHaveTitle(/Playwright/)
|
|
567
|
+
await expect(page).toHaveTitle('Exact Title')
|
|
568
|
+
await expect(page).toHaveURL('https://example.com/dashboard')
|
|
569
|
+
await expect(page).toHaveURL(/\\/dashboard/)
|
|
570
|
+
|
|
571
|
+
ACTIONS (Playwright auto-waits before each action)
|
|
572
|
+
Navigation:
|
|
573
|
+
await page.goto('https://example.com')
|
|
574
|
+
await page.goBack()
|
|
575
|
+
await page.goForward()
|
|
576
|
+
await page.reload()
|
|
577
|
+
|
|
578
|
+
Clicking:
|
|
579
|
+
await locator.click()
|
|
580
|
+
await locator.dblclick()
|
|
581
|
+
await locator.click({ button: 'right' })
|
|
582
|
+
await locator.click({ modifiers: ['Shift'] })
|
|
583
|
+
|
|
584
|
+
Forms:
|
|
585
|
+
await locator.fill('text') // clears then types
|
|
586
|
+
await locator.clear()
|
|
587
|
+
await locator.pressSequentially('slow typing', { delay: 50 })
|
|
588
|
+
await locator.selectOption('value')
|
|
589
|
+
await locator.selectOption({ label: 'Blue' })
|
|
590
|
+
await locator.check()
|
|
591
|
+
await locator.uncheck()
|
|
592
|
+
await locator.setInputFiles('path/to/file.pdf')
|
|
593
|
+
|
|
594
|
+
Hover and focus:
|
|
595
|
+
await locator.hover()
|
|
596
|
+
await locator.focus()
|
|
597
|
+
await locator.blur()
|
|
598
|
+
|
|
599
|
+
Keyboard:
|
|
600
|
+
await page.keyboard.press('Enter')
|
|
601
|
+
await page.keyboard.press('Tab')
|
|
602
|
+
await page.keyboard.press('Escape')
|
|
603
|
+
await page.keyboard.press('Control+A')
|
|
604
|
+
|
|
605
|
+
Scroll:
|
|
606
|
+
await locator.scrollIntoViewIfNeeded()
|
|
607
|
+
await page.mouse.wheel(0, 500)
|
|
608
|
+
|
|
609
|
+
BEST PRACTICES (from playwright.dev/docs/best-practices)
|
|
610
|
+
DO use web-first assertions:
|
|
611
|
+
await expect(page.getByText('welcome')).toBeVisible()
|
|
612
|
+
|
|
613
|
+
NEVER use synchronous assertions:
|
|
614
|
+
expect(await page.getByText('welcome').isVisible()).toBe(true)
|
|
615
|
+
|
|
616
|
+
DO chain locators:
|
|
617
|
+
page.getByRole('listitem').filter({ hasText: 'Product 2' })
|
|
618
|
+
.getByRole('button', { name: 'Add to cart' })
|
|
619
|
+
|
|
620
|
+
NEVER use fragile CSS/XPath selectors:
|
|
621
|
+
page.locator('button.buttonIcon.episode-actions-later')
|
|
622
|
+
|
|
623
|
+
DO use role-based locators:
|
|
624
|
+
page.getByRole('button', { name: 'submit' })
|
|
625
|
+
|
|
626
|
+
INTENT -> PATTERN MAPPING
|
|
627
|
+
|
|
628
|
+
COUNTING:
|
|
629
|
+
"how many X" / "count X" / "number of X" / "return the count"
|
|
630
|
+
-> const n = await page.getByRole('X').count();
|
|
631
|
+
console.log(\`Found \${n} X elements\`);
|
|
632
|
+
expect(n).toBeGreaterThan(0);
|
|
633
|
+
|
|
634
|
+
"there are N X" / "exactly N X"
|
|
635
|
+
-> await expect(page.getByRole('X')).toHaveCount(N);
|
|
636
|
+
|
|
637
|
+
CRITICAL: Never use getByText("how many X") - count intent
|
|
638
|
+
means call .count() or toHaveCount(), not search for text.
|
|
639
|
+
|
|
640
|
+
PRESENCE:
|
|
641
|
+
"X is present/visible/shown/exists"
|
|
642
|
+
-> await expect(locator).toBeVisible()
|
|
643
|
+
|
|
644
|
+
"X is hidden/not present/gone"
|
|
645
|
+
-> await expect(locator).toBeHidden()
|
|
646
|
+
|
|
647
|
+
TEXT:
|
|
648
|
+
"text says X" / "shows X" / "message is X"
|
|
649
|
+
-> await expect(locator).toHaveText('X')
|
|
650
|
+
|
|
651
|
+
"contains X" / "includes X"
|
|
652
|
+
-> await expect(locator).toContainText('X')
|
|
653
|
+
|
|
654
|
+
"page title is X"
|
|
655
|
+
-> await expect(page).toHaveTitle(/X/i)
|
|
656
|
+
|
|
657
|
+
"heading says X"
|
|
658
|
+
-> await expect(page.getByRole('heading')).toContainText('X')
|
|
659
|
+
|
|
660
|
+
"error says X"
|
|
661
|
+
-> await expect(page.getByRole('alert')).toContainText('X')
|
|
662
|
+
|
|
663
|
+
STATE:
|
|
664
|
+
"X is enabled/disabled"
|
|
665
|
+
-> await expect(locator).toBeEnabled() / toBeDisabled()
|
|
666
|
+
|
|
667
|
+
"X is checked/unchecked"
|
|
668
|
+
-> await expect(locator).toBeChecked() / not.toBeChecked()
|
|
669
|
+
|
|
670
|
+
"X has value Y"
|
|
671
|
+
-> await expect(locator).toHaveValue('Y')
|
|
672
|
+
|
|
673
|
+
"X is active / has class Y"
|
|
674
|
+
-> await expect(locator).toHaveClass(/Y/)
|
|
675
|
+
|
|
676
|
+
NAVIGATION:
|
|
677
|
+
"redirects to X" / "goes to X" / "URL contains X"
|
|
678
|
+
-> await expect(page).toHaveURL(/X/)
|
|
679
|
+
|
|
680
|
+
"stays on same page"
|
|
681
|
+
-> await expect(page).toHaveURL(/currentPath/)
|
|
682
|
+
|
|
683
|
+
"opens new tab"
|
|
684
|
+
-> const [newPage] = await Promise.all([
|
|
685
|
+
context.waitForEvent('page'),
|
|
686
|
+
locator.click()
|
|
687
|
+
]);
|
|
688
|
+
|
|
689
|
+
FORMS:
|
|
690
|
+
"form submits successfully"
|
|
691
|
+
-> fill fields + click submit + assert URL change or success message
|
|
692
|
+
|
|
693
|
+
"validation error shown"
|
|
694
|
+
-> submit empty + await expect(page.getByRole('alert')).toBeVisible()
|
|
695
|
+
|
|
696
|
+
"required field X"
|
|
697
|
+
-> submit without X + assert error message for X visible
|
|
698
|
+
|
|
699
|
+
AUTH:
|
|
700
|
+
"login with valid credentials"
|
|
701
|
+
-> fill(CT_VAR_USERNAME) + fill(CT_VAR_PASSWORD) + click login
|
|
702
|
+
+ await expect(page).toHaveURL(/dashboard|home|app/)
|
|
703
|
+
|
|
704
|
+
"login fails / invalid credentials"
|
|
705
|
+
-> fill bad values + click login
|
|
706
|
+
+ await expect(page.getByRole('alert')).toBeVisible()
|
|
707
|
+
|
|
708
|
+
"logout works"
|
|
709
|
+
-> click logout + await expect(page).toHaveURL(/login|home/)
|
|
710
|
+
|
|
711
|
+
THEME / VISUAL:
|
|
712
|
+
"dark mode / theme toggle"
|
|
713
|
+
-> await locator.click()
|
|
714
|
+
+ await expect(page.locator('html')).toHaveAttribute('data-theme', 'dark')
|
|
715
|
+
// OR: await expect(page.locator('body')).toHaveClass(/dark/)
|
|
716
|
+
|
|
717
|
+
TIMING:
|
|
718
|
+
"X loads / X appears"
|
|
719
|
+
-> await expect(locator).toBeVisible({ timeout: 10000 })
|
|
720
|
+
|
|
721
|
+
"spinner disappears / loader gone"
|
|
722
|
+
-> await expect(spinner).toBeHidden({ timeout: 10000 })
|
|
723
|
+
|
|
724
|
+
"modal closes"
|
|
725
|
+
-> await expect(modal).toBeHidden()
|
|
726
|
+
|
|
727
|
+
ACCESSIBILITY:
|
|
728
|
+
"has alt text"
|
|
729
|
+
-> await expect(page.locator('img')).toHaveAttribute('alt', /.+/)
|
|
730
|
+
|
|
731
|
+
"keyboard navigable"
|
|
732
|
+
-> await page.keyboard.press('Tab')
|
|
733
|
+
+ await expect(firstFocusable).toBeFocused()
|
|
734
|
+
|
|
735
|
+
CRITICAL RULES
|
|
736
|
+
1. NEVER use intent words as locator text.
|
|
737
|
+
"count buttons" != getByText("count buttons")
|
|
738
|
+
"verify heading" != getByText("verify heading")
|
|
739
|
+
|
|
740
|
+
2. ALWAYS use web-first assertions (await expect).
|
|
741
|
+
NEVER use: expect(await locator.isVisible()).toBe(true)
|
|
742
|
+
|
|
743
|
+
3. For counting, prefer toHaveCount() over .count() unless
|
|
744
|
+
you need the actual number for logging.
|
|
745
|
+
|
|
746
|
+
4. Auto-waiting: Playwright automatically waits for elements
|
|
747
|
+
to be actionable before click/fill/etc. Do not add
|
|
748
|
+
manual waitForTimeout() unless testing timing specifically.
|
|
749
|
+
|
|
750
|
+
5. Use not. prefix for negative assertions:
|
|
751
|
+
await expect(locator).not.toBeVisible()
|
|
752
|
+
await expect(locator).not.toBeChecked()
|
|
753
|
+
`.trim();
|
|
754
|
+
var ADVANCED_DOC_MAP = {
|
|
755
|
+
upload: "https://playwright.dev/docs/input",
|
|
756
|
+
"file upload": "https://playwright.dev/docs/input",
|
|
757
|
+
intercept: "https://playwright.dev/docs/mock",
|
|
758
|
+
mock: "https://playwright.dev/docs/mock",
|
|
759
|
+
"api mock": "https://playwright.dev/docs/mock",
|
|
760
|
+
accessibility: "https://playwright.dev/docs/accessibility-testing",
|
|
761
|
+
"screen reader": "https://playwright.dev/docs/accessibility-testing",
|
|
762
|
+
visual: "https://playwright.dev/docs/screenshots",
|
|
763
|
+
screenshot: "https://playwright.dev/docs/screenshots",
|
|
764
|
+
mobile: "https://playwright.dev/docs/emulation",
|
|
765
|
+
responsive: "https://playwright.dev/docs/emulation",
|
|
766
|
+
viewport: "https://playwright.dev/docs/emulation",
|
|
767
|
+
network: "https://playwright.dev/docs/network",
|
|
768
|
+
"api call": "https://playwright.dev/docs/network",
|
|
769
|
+
iframe: "https://playwright.dev/docs/frames",
|
|
770
|
+
frame: "https://playwright.dev/docs/frames",
|
|
771
|
+
auth: "https://playwright.dev/docs/auth",
|
|
772
|
+
"sign in": "https://playwright.dev/docs/auth",
|
|
773
|
+
"stay logged": "https://playwright.dev/docs/auth",
|
|
774
|
+
cookie: "https://playwright.dev/docs/auth",
|
|
775
|
+
storage: "https://playwright.dev/docs/auth",
|
|
776
|
+
download: "https://playwright.dev/docs/downloads",
|
|
777
|
+
pdf: "https://playwright.dev/docs/downloads",
|
|
778
|
+
dialog: "https://playwright.dev/docs/dialogs",
|
|
779
|
+
alert: "https://playwright.dev/docs/dialogs",
|
|
780
|
+
confirm: "https://playwright.dev/docs/dialogs",
|
|
781
|
+
"new tab": "https://playwright.dev/docs/pages",
|
|
782
|
+
"new page": "https://playwright.dev/docs/pages",
|
|
783
|
+
popup: "https://playwright.dev/docs/pages"
|
|
784
|
+
};
|
|
785
|
+
async function fetchDocContext(intent) {
|
|
786
|
+
const intentLower = intent.toLowerCase();
|
|
787
|
+
const matched = /* @__PURE__ */ new Set();
|
|
788
|
+
for (const [keyword, url] of Object.entries(ADVANCED_DOC_MAP)) {
|
|
789
|
+
if (intentLower.includes(keyword)) {
|
|
790
|
+
matched.add(url);
|
|
791
|
+
}
|
|
792
|
+
}
|
|
793
|
+
if (matched.size === 0) return "";
|
|
794
|
+
const fetched = [];
|
|
795
|
+
for (const url of matched) {
|
|
796
|
+
try {
|
|
797
|
+
const res = await fetch(url, { signal: AbortSignal.timeout(5e3) });
|
|
798
|
+
const html = await res.text();
|
|
799
|
+
const text = html.replace(/<script[^>]*>[\s\S]*?<\/script>/gi, "").replace(/<style[^>]*>[\s\S]*?<\/style>/gi, "").replace(/<[^>]+>/g, " ").replace(/\s{3,}/g, "\n\n").slice(0, 3e3);
|
|
800
|
+
fetched.push(`
|
|
801
|
+
// From ${url}:
|
|
802
|
+
${text}`);
|
|
803
|
+
} catch {
|
|
804
|
+
}
|
|
805
|
+
}
|
|
806
|
+
if (fetched.length === 0) return "";
|
|
807
|
+
return `
|
|
808
|
+
|
|
809
|
+
\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501
|
|
810
|
+
ADDITIONAL PLAYWRIGHT DOCS (fetched for this intent)
|
|
811
|
+
\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501${fetched.join("\n")}`;
|
|
812
|
+
}
|
|
504
813
|
function buildSystemMessage() {
|
|
505
814
|
return `
|
|
506
|
-
You are a senior
|
|
815
|
+
You are a senior software test architect with 10 years of Playwright experience.
|
|
816
|
+
You translate user requirements into precise, minimal, executable test scenarios.
|
|
817
|
+
|
|
818
|
+
RULE 1 - INTENT IS THE REQUIREMENT
|
|
819
|
+
Parse the user's feature description into specific testable claims BEFORE examining page context.
|
|
820
|
+
Generate scenarios for the user's intent, not for every element on the page.
|
|
821
|
+
|
|
822
|
+
RULE 2 - GENERATE TESTS FOR THE INTENT, NOT THE PAGE INVENTORY
|
|
823
|
+
Only generate tests for claims extracted from the intent.
|
|
824
|
+
Never generate tests for page elements that were not requested.
|
|
825
|
+
Page context improves selector quality and wording, but it does not define test scope.
|
|
826
|
+
|
|
827
|
+
RULE 3 - SELECTOR HIERARCHY
|
|
828
|
+
When choosing selectors, prefer:
|
|
829
|
+
1. Exact page-context match when available
|
|
830
|
+
2. getByRole() with text from the intent
|
|
831
|
+
3. getByLabel() for form inputs
|
|
832
|
+
4. getByText() with text from the intent
|
|
833
|
+
5. getByPlaceholder() for inputs
|
|
834
|
+
6. locator('#id') only when the id appears stable
|
|
835
|
+
|
|
836
|
+
Tie-breaker when multiple elements match:
|
|
837
|
+
1. Prefer the highest-confidence page-context match
|
|
838
|
+
2. Prefer visible elements over hidden
|
|
839
|
+
3. Prefer interactive elements over static elements
|
|
840
|
+
4. If still ambiguous, use the first match and note it with a short comment in the step text
|
|
841
|
+
|
|
842
|
+
RULE 4 - TITLE DISAMBIGUATION
|
|
843
|
+
"verify title" means the main visible heading on the page.
|
|
844
|
+
Use a heading assertion first.
|
|
845
|
+
Do NOT treat "title" as the browser tab title unless the intent explicitly says "browser title", "tab title", or "page title".
|
|
846
|
+
|
|
847
|
+
RULE 5 - PRESENCE-ONLY INTENTS
|
|
848
|
+
For presence-only intents such as "verify X", "check X is present", or "X is visible":
|
|
849
|
+
- Only assert visibility
|
|
850
|
+
- Do NOT click the element
|
|
851
|
+
- Do NOT assert redirects, state changes, or side effects
|
|
852
|
+
|
|
853
|
+
RULE 6 - NEGATIVE CASES
|
|
854
|
+
Generate negative scenarios only when the intent implies interaction:
|
|
855
|
+
- Form testing: always include an invalid-input scenario
|
|
856
|
+
- Authentication: always include a wrong-credentials scenario
|
|
857
|
+
- Presence checks: no negative case needed
|
|
858
|
+
- Button click: negative case only if the button can be disabled
|
|
859
|
+
|
|
860
|
+
RULE 7 - SETUP DEPENDENCIES
|
|
861
|
+
If authentication is required to reach the page:
|
|
862
|
+
- Add setup steps using CT_VAR_USERNAME and CT_VAR_PASSWORD
|
|
863
|
+
- Tag the scenario with @requires-auth
|
|
864
|
+
- Mention the dependency in a short step or assertion note when needed
|
|
507
865
|
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
-
|
|
866
|
+
RULE 8 - TEST DATA
|
|
867
|
+
Never use the literal word 'value' as a placeholder.
|
|
868
|
+
When filling a field:
|
|
869
|
+
- If the user context lists a matching CT_VAR_* variable, write it exactly as '\${CT_VAR_FIELDNAME}` + `'
|
|
870
|
+
- Otherwise use a field-specific fallback such as 'test-username', 'test-email@example.com', 'test-password', or 'test-search'
|
|
512
871
|
|
|
513
|
-
|
|
872
|
+
RULE 9 - SCOPE DISCIPLINE
|
|
873
|
+
Match scenario count to intent complexity:
|
|
874
|
+
- 1 to 2 claims: 1 to 2 scenarios maximum
|
|
875
|
+
- 3 to 5 claims: 3 to 5 scenarios maximum
|
|
876
|
+
- Full flow: 4 to 8 scenarios maximum
|
|
877
|
+
Never exceed 8 scenarios.
|
|
878
|
+
|
|
879
|
+
${RULE_10_PLAYWRIGHT_KNOWLEDGE_BASE}
|
|
880
|
+
|
|
881
|
+
COMMON TESTING VOCABULARY
|
|
882
|
+
"verify/check X is present/visible" -> toBeVisible()
|
|
883
|
+
"verify title/heading" -> heading is visible
|
|
884
|
+
"button/link is present" -> button or link is visible
|
|
885
|
+
"can click X" -> click then assert visible outcome
|
|
886
|
+
"form submits" -> fill + click + assert success or URL
|
|
887
|
+
"error shows" -> trigger error + assert error visible
|
|
888
|
+
"redirects to X" -> URL contains or equals X
|
|
889
|
+
"text says X" -> text equals or contains X
|
|
890
|
+
"page loads" -> main heading or key element is visible
|
|
891
|
+
|
|
892
|
+
OUTPUT FORMAT - EXACT MARKDOWN SCHEMA
|
|
893
|
+
Output valid markdown in this exact format. No preamble. No explanation. Just the markdown.
|
|
894
|
+
|
|
895
|
+
# {PREFIX}-001 - {scenario-slug} - {Human readable title} @tag1 @tag2
|
|
896
|
+
|
|
897
|
+
## Steps
|
|
898
|
+
1. Navigate to {url}
|
|
899
|
+
2. {action verb} {target description}
|
|
900
|
+
|
|
901
|
+
## Expected Results
|
|
902
|
+
- {specific observable assertion}
|
|
903
|
+
- {specific observable assertion}
|
|
904
|
+
|
|
905
|
+
---
|
|
906
|
+
|
|
907
|
+
# {PREFIX}-002 - {scenario-slug} - {Human readable title} @tag1 @tag2
|
|
514
908
|
|
|
515
|
-
# <ID> \u2014 <Short title> @<tag1> @<tag2>
|
|
516
909
|
## Steps
|
|
517
|
-
1.
|
|
518
|
-
2.
|
|
910
|
+
1. Navigate to {url}
|
|
911
|
+
2. {action verb} {target description}
|
|
519
912
|
|
|
520
913
|
## Expected Results
|
|
521
|
-
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
-
|
|
526
|
-
-
|
|
527
|
-
-
|
|
528
|
-
-
|
|
529
|
-
-
|
|
530
|
-
-
|
|
531
|
-
-
|
|
532
|
-
- If you do not know the real value, use the pattern 'test-{fieldname}'
|
|
533
|
-
- Expected Results = verifiable UI outcomes ("Error message 'Invalid credentials' is visible", "Page redirects to /dashboard")
|
|
534
|
-
- Include both happy-path AND negative/edge-case tests
|
|
535
|
-
- Do NOT explain or add preamble \u2014 output ONLY the test cases
|
|
536
|
-
- No code blocks, no extra headings outside the pattern
|
|
914
|
+
- {specific observable assertion}
|
|
915
|
+
|
|
916
|
+
Rules for the markdown schema:
|
|
917
|
+
- id format: PREFIX-NNN using the provided prefix and numbering
|
|
918
|
+
- slug format: kebab-case, max 5 words, derived from the scenario subject
|
|
919
|
+
- title: human readable and aligned to the user's intent
|
|
920
|
+
- tags: include at least one of @smoke @regression @negative @happy-path plus a relevant domain tag such as @auth @ui @navigation @form @checkout
|
|
921
|
+
- steps: numbered, start with Navigate, then actions, written in plain English
|
|
922
|
+
- assertions: bullet points with observable outcomes only
|
|
923
|
+
- never write "test passes" or "works correctly"
|
|
924
|
+
- do NOT wrap the output in code fences
|
|
537
925
|
`.trim();
|
|
538
926
|
}
|
|
539
927
|
function buildUserMessage(ctx) {
|
|
@@ -542,9 +930,12 @@ function buildUserMessage(ctx) {
|
|
|
542
930
|
lines.push(`App / product description:`);
|
|
543
931
|
lines.push(ctx.appDescription || "N/A");
|
|
544
932
|
lines.push("");
|
|
545
|
-
lines.push(`
|
|
933
|
+
lines.push(`Primary test intent:`);
|
|
546
934
|
lines.push(ctx.feature);
|
|
547
935
|
lines.push("");
|
|
936
|
+
lines.push(`Interpret the feature text above as the source of truth.`);
|
|
937
|
+
lines.push(`Only generate scenarios for claims that are explicitly implied by that intent.`);
|
|
938
|
+
lines.push("");
|
|
548
939
|
if (ctx.url) {
|
|
549
940
|
lines.push(`Page URL: ${ctx.url}`);
|
|
550
941
|
lines.push("");
|
|
@@ -588,9 +979,9 @@ function buildUserMessage(ctx) {
|
|
|
588
979
|
}
|
|
589
980
|
lines.push(`Test ID prefix: ${ctx.prefix}`);
|
|
590
981
|
lines.push(`Start numbering from: ${String(ctx.startIndex).padStart(3, "0")}`);
|
|
591
|
-
lines.push(`
|
|
982
|
+
lines.push(`Requested number of test cases: ${ctx.numCases}`);
|
|
592
983
|
lines.push("");
|
|
593
|
-
lines.push(`
|
|
984
|
+
lines.push(`Use the requested count only when it fits the intent complexity. Prefer fewer scenarios when the request is simple.`);
|
|
594
985
|
lines.push(`Output ONLY the test cases. No explanation, no preamble.`);
|
|
595
986
|
return lines.join("\n");
|
|
596
987
|
}
|
|
@@ -645,7 +1036,8 @@ async function callOpenAiCompatible(apiKey, model, baseUrl, displayName, system,
|
|
|
645
1036
|
}
|
|
646
1037
|
async function generateTcMarkdownWithAi(ctx) {
|
|
647
1038
|
const providerConfig = resolveLlmProvider();
|
|
648
|
-
const
|
|
1039
|
+
const docContext = await fetchDocContext(ctx.feature);
|
|
1040
|
+
const system = buildSystemMessage() + docContext;
|
|
649
1041
|
const user = buildUserMessage(ctx);
|
|
650
1042
|
console.log(`\u{1F916} Using ${providerConfig.displayName} (${providerConfig.model})`);
|
|
651
1043
|
if (providerConfig.transport === "anthropic") {
|
|
@@ -662,13 +1054,324 @@ async function generateTcMarkdownWithAi(ctx) {
|
|
|
662
1054
|
}
|
|
663
1055
|
|
|
664
1056
|
// src/core/analyse.ts
|
|
1057
|
+
var RULE_10_PLAYWRIGHT_KNOWLEDGE_BASE2 = `
|
|
1058
|
+
RULE 10 - PLAYWRIGHT KNOWLEDGE BASE
|
|
1059
|
+
(sourced from playwright.dev official documentation)
|
|
1060
|
+
|
|
1061
|
+
You are an expert in Playwright TypeScript. Apply this knowledge
|
|
1062
|
+
when generating test code. Always use web-first assertions that
|
|
1063
|
+
auto-wait. Never use manual waits or non-awaited assertions.
|
|
1064
|
+
|
|
1065
|
+
LOCATOR PRIORITY (use in this order)
|
|
1066
|
+
1. page.getByRole('button', { name: 'Submit' }) <- best
|
|
1067
|
+
2. page.getByLabel('Email address')
|
|
1068
|
+
3. page.getByPlaceholder('Enter email')
|
|
1069
|
+
4. page.getByText('Welcome')
|
|
1070
|
+
5. page.getByAltText('logo')
|
|
1071
|
+
6. page.getByTitle('Close')
|
|
1072
|
+
7. page.getByTestId('submit-btn')
|
|
1073
|
+
8. page.locator('#id') or page.locator('.class') <- last resort
|
|
1074
|
+
|
|
1075
|
+
Chain locators to narrow scope:
|
|
1076
|
+
page.getByRole('listitem').filter({ hasText: 'Product 2' })
|
|
1077
|
+
.getByRole('button', { name: 'Add to cart' }).click()
|
|
1078
|
+
|
|
1079
|
+
LOCATOR ASSERTIONS (always await, always web-first)
|
|
1080
|
+
Visibility:
|
|
1081
|
+
await expect(locator).toBeVisible()
|
|
1082
|
+
await expect(locator).toBeHidden()
|
|
1083
|
+
await expect(locator).toBeInViewport()
|
|
1084
|
+
|
|
1085
|
+
State:
|
|
1086
|
+
await expect(locator).toBeEnabled()
|
|
1087
|
+
await expect(locator).toBeDisabled()
|
|
1088
|
+
await expect(locator).toBeChecked()
|
|
1089
|
+
await expect(locator).not.toBeChecked()
|
|
1090
|
+
await expect(locator).toBeFocused()
|
|
1091
|
+
await expect(locator).toBeEditable()
|
|
1092
|
+
await expect(locator).toBeEmpty()
|
|
1093
|
+
await expect(locator).toBeAttached()
|
|
1094
|
+
|
|
1095
|
+
Text content:
|
|
1096
|
+
await expect(locator).toHaveText('exact text')
|
|
1097
|
+
await expect(locator).toHaveText(/regex/)
|
|
1098
|
+
await expect(locator).toContainText('partial')
|
|
1099
|
+
await expect(locator).toContainText(['item1', 'item2'])
|
|
1100
|
+
|
|
1101
|
+
Value and attributes:
|
|
1102
|
+
await expect(locator).toHaveValue('input value')
|
|
1103
|
+
await expect(locator).toHaveValues(['opt1', 'opt2']) // multi-select
|
|
1104
|
+
await expect(locator).toHaveAttribute('href', /pattern/)
|
|
1105
|
+
await expect(locator).toHaveClass(/active/)
|
|
1106
|
+
await expect(locator).toHaveCSS('color', 'rgb(0,0,0)')
|
|
1107
|
+
await expect(locator).toHaveId('submit-btn')
|
|
1108
|
+
await expect(locator).toHaveAccessibleName('Submit form')
|
|
1109
|
+
await expect(locator).toHaveAccessibleDescription('...')
|
|
1110
|
+
|
|
1111
|
+
Counting (PREFER toHaveCount over .count() to avoid flakiness):
|
|
1112
|
+
await expect(locator).toHaveCount(3)
|
|
1113
|
+
await expect(locator).toHaveCount(0) // none exist
|
|
1114
|
+
// Only use .count() when you need the actual number:
|
|
1115
|
+
const n = await page.getByRole('button').count();
|
|
1116
|
+
console.log(\`Found \${n} buttons\`);
|
|
1117
|
+
|
|
1118
|
+
PAGE ASSERTIONS
|
|
1119
|
+
await expect(page).toHaveTitle(/Playwright/)
|
|
1120
|
+
await expect(page).toHaveTitle('Exact Title')
|
|
1121
|
+
await expect(page).toHaveURL('https://example.com/dashboard')
|
|
1122
|
+
await expect(page).toHaveURL(/\\/dashboard/)
|
|
1123
|
+
|
|
1124
|
+
ACTIONS (Playwright auto-waits before each action)
|
|
1125
|
+
Navigation:
|
|
1126
|
+
await page.goto('https://example.com')
|
|
1127
|
+
await page.goBack()
|
|
1128
|
+
await page.goForward()
|
|
1129
|
+
await page.reload()
|
|
1130
|
+
|
|
1131
|
+
Clicking:
|
|
1132
|
+
await locator.click()
|
|
1133
|
+
await locator.dblclick()
|
|
1134
|
+
await locator.click({ button: 'right' })
|
|
1135
|
+
await locator.click({ modifiers: ['Shift'] })
|
|
1136
|
+
|
|
1137
|
+
Forms:
|
|
1138
|
+
await locator.fill('text') // clears then types
|
|
1139
|
+
await locator.clear()
|
|
1140
|
+
await locator.pressSequentially('slow typing', { delay: 50 })
|
|
1141
|
+
await locator.selectOption('value')
|
|
1142
|
+
await locator.selectOption({ label: 'Blue' })
|
|
1143
|
+
await locator.check()
|
|
1144
|
+
await locator.uncheck()
|
|
1145
|
+
await locator.setInputFiles('path/to/file.pdf')
|
|
1146
|
+
|
|
1147
|
+
Hover and focus:
|
|
1148
|
+
await locator.hover()
|
|
1149
|
+
await locator.focus()
|
|
1150
|
+
await locator.blur()
|
|
1151
|
+
|
|
1152
|
+
Keyboard:
|
|
1153
|
+
await page.keyboard.press('Enter')
|
|
1154
|
+
await page.keyboard.press('Tab')
|
|
1155
|
+
await page.keyboard.press('Escape')
|
|
1156
|
+
await page.keyboard.press('Control+A')
|
|
1157
|
+
|
|
1158
|
+
Scroll:
|
|
1159
|
+
await locator.scrollIntoViewIfNeeded()
|
|
1160
|
+
await page.mouse.wheel(0, 500)
|
|
1161
|
+
|
|
1162
|
+
BEST PRACTICES (from playwright.dev/docs/best-practices)
|
|
1163
|
+
DO use web-first assertions:
|
|
1164
|
+
await expect(page.getByText('welcome')).toBeVisible()
|
|
1165
|
+
|
|
1166
|
+
NEVER use synchronous assertions:
|
|
1167
|
+
expect(await page.getByText('welcome').isVisible()).toBe(true)
|
|
1168
|
+
|
|
1169
|
+
DO chain locators:
|
|
1170
|
+
page.getByRole('listitem').filter({ hasText: 'Product 2' })
|
|
1171
|
+
.getByRole('button', { name: 'Add to cart' })
|
|
1172
|
+
|
|
1173
|
+
NEVER use fragile CSS/XPath selectors:
|
|
1174
|
+
page.locator('button.buttonIcon.episode-actions-later')
|
|
1175
|
+
|
|
1176
|
+
DO use role-based locators:
|
|
1177
|
+
page.getByRole('button', { name: 'submit' })
|
|
1178
|
+
|
|
1179
|
+
INTENT -> PATTERN MAPPING
|
|
1180
|
+
|
|
1181
|
+
COUNTING:
|
|
1182
|
+
"how many X" / "count X" / "number of X" / "return the count"
|
|
1183
|
+
-> const n = await page.getByRole('X').count();
|
|
1184
|
+
console.log(\`Found \${n} X elements\`);
|
|
1185
|
+
expect(n).toBeGreaterThan(0);
|
|
1186
|
+
|
|
1187
|
+
"there are N X" / "exactly N X"
|
|
1188
|
+
-> await expect(page.getByRole('X')).toHaveCount(N);
|
|
1189
|
+
|
|
1190
|
+
CRITICAL: Never use getByText("how many X") - count intent
|
|
1191
|
+
means call .count() or toHaveCount(), not search for text.
|
|
1192
|
+
|
|
1193
|
+
PRESENCE:
|
|
1194
|
+
"X is present/visible/shown/exists"
|
|
1195
|
+
-> await expect(locator).toBeVisible()
|
|
1196
|
+
|
|
1197
|
+
"X is hidden/not present/gone"
|
|
1198
|
+
-> await expect(locator).toBeHidden()
|
|
1199
|
+
|
|
1200
|
+
TEXT:
|
|
1201
|
+
"text says X" / "shows X" / "message is X"
|
|
1202
|
+
-> await expect(locator).toHaveText('X')
|
|
1203
|
+
|
|
1204
|
+
"contains X" / "includes X"
|
|
1205
|
+
-> await expect(locator).toContainText('X')
|
|
1206
|
+
|
|
1207
|
+
"page title is X"
|
|
1208
|
+
-> await expect(page).toHaveTitle(/X/i)
|
|
1209
|
+
|
|
1210
|
+
"heading says X"
|
|
1211
|
+
-> await expect(page.getByRole('heading')).toContainText('X')
|
|
1212
|
+
|
|
1213
|
+
"error says X"
|
|
1214
|
+
-> await expect(page.getByRole('alert')).toContainText('X')
|
|
1215
|
+
|
|
1216
|
+
STATE:
|
|
1217
|
+
"X is enabled/disabled"
|
|
1218
|
+
-> await expect(locator).toBeEnabled() / toBeDisabled()
|
|
1219
|
+
|
|
1220
|
+
"X is checked/unchecked"
|
|
1221
|
+
-> await expect(locator).toBeChecked() / not.toBeChecked()
|
|
1222
|
+
|
|
1223
|
+
"X has value Y"
|
|
1224
|
+
-> await expect(locator).toHaveValue('Y')
|
|
1225
|
+
|
|
1226
|
+
"X is active / has class Y"
|
|
1227
|
+
-> await expect(locator).toHaveClass(/Y/)
|
|
1228
|
+
|
|
1229
|
+
NAVIGATION:
|
|
1230
|
+
"redirects to X" / "goes to X" / "URL contains X"
|
|
1231
|
+
-> await expect(page).toHaveURL(/X/)
|
|
1232
|
+
|
|
1233
|
+
"stays on same page"
|
|
1234
|
+
-> await expect(page).toHaveURL(/currentPath/)
|
|
1235
|
+
|
|
1236
|
+
"opens new tab"
|
|
1237
|
+
-> const [newPage] = await Promise.all([
|
|
1238
|
+
context.waitForEvent('page'),
|
|
1239
|
+
locator.click()
|
|
1240
|
+
]);
|
|
1241
|
+
|
|
1242
|
+
FORMS:
|
|
1243
|
+
"form submits successfully"
|
|
1244
|
+
-> fill fields + click submit + assert URL change or success message
|
|
1245
|
+
|
|
1246
|
+
"validation error shown"
|
|
1247
|
+
-> submit empty + await expect(page.getByRole('alert')).toBeVisible()
|
|
1248
|
+
|
|
1249
|
+
"required field X"
|
|
1250
|
+
-> submit without X + assert error message for X visible
|
|
1251
|
+
|
|
1252
|
+
AUTH:
|
|
1253
|
+
"login with valid credentials"
|
|
1254
|
+
-> fill(CT_VAR_USERNAME) + fill(CT_VAR_PASSWORD) + click login
|
|
1255
|
+
+ await expect(page).toHaveURL(/dashboard|home|app/)
|
|
1256
|
+
|
|
1257
|
+
"login fails / invalid credentials"
|
|
1258
|
+
-> fill bad values + click login
|
|
1259
|
+
+ await expect(page.getByRole('alert')).toBeVisible()
|
|
1260
|
+
|
|
1261
|
+
"logout works"
|
|
1262
|
+
-> click logout + await expect(page).toHaveURL(/login|home/)
|
|
1263
|
+
|
|
1264
|
+
THEME / VISUAL:
|
|
1265
|
+
"dark mode / theme toggle"
|
|
1266
|
+
-> await locator.click()
|
|
1267
|
+
+ await expect(page.locator('html')).toHaveAttribute('data-theme', 'dark')
|
|
1268
|
+
// OR: await expect(page.locator('body')).toHaveClass(/dark/)
|
|
1269
|
+
|
|
1270
|
+
TIMING:
|
|
1271
|
+
"X loads / X appears"
|
|
1272
|
+
-> await expect(locator).toBeVisible({ timeout: 10000 })
|
|
1273
|
+
|
|
1274
|
+
"spinner disappears / loader gone"
|
|
1275
|
+
-> await expect(spinner).toBeHidden({ timeout: 10000 })
|
|
1276
|
+
|
|
1277
|
+
"modal closes"
|
|
1278
|
+
-> await expect(modal).toBeHidden()
|
|
1279
|
+
|
|
1280
|
+
ACCESSIBILITY:
|
|
1281
|
+
"has alt text"
|
|
1282
|
+
-> await expect(page.locator('img')).toHaveAttribute('alt', /.+/)
|
|
1283
|
+
|
|
1284
|
+
"keyboard navigable"
|
|
1285
|
+
-> await page.keyboard.press('Tab')
|
|
1286
|
+
+ await expect(firstFocusable).toBeFocused()
|
|
1287
|
+
|
|
1288
|
+
CRITICAL RULES
|
|
1289
|
+
1. NEVER use intent words as locator text.
|
|
1290
|
+
"count buttons" != getByText("count buttons")
|
|
1291
|
+
"verify heading" != getByText("verify heading")
|
|
1292
|
+
|
|
1293
|
+
2. ALWAYS use web-first assertions (await expect).
|
|
1294
|
+
NEVER use: expect(await locator.isVisible()).toBe(true)
|
|
1295
|
+
|
|
1296
|
+
3. For counting, prefer toHaveCount() over .count() unless
|
|
1297
|
+
you need the actual number for logging.
|
|
1298
|
+
|
|
1299
|
+
4. Auto-waiting: Playwright automatically waits for elements
|
|
1300
|
+
to be actionable before click/fill/etc. Do not add
|
|
1301
|
+
manual waitForTimeout() unless testing timing specifically.
|
|
1302
|
+
|
|
1303
|
+
5. Use not. prefix for negative assertions:
|
|
1304
|
+
await expect(locator).not.toBeVisible()
|
|
1305
|
+
await expect(locator).not.toBeChecked()
|
|
1306
|
+
`.trim();
|
|
1307
|
+
var ADVANCED_DOC_MAP2 = {
|
|
1308
|
+
upload: "https://playwright.dev/docs/input",
|
|
1309
|
+
"file upload": "https://playwright.dev/docs/input",
|
|
1310
|
+
intercept: "https://playwright.dev/docs/mock",
|
|
1311
|
+
mock: "https://playwright.dev/docs/mock",
|
|
1312
|
+
"api mock": "https://playwright.dev/docs/mock",
|
|
1313
|
+
accessibility: "https://playwright.dev/docs/accessibility-testing",
|
|
1314
|
+
"screen reader": "https://playwright.dev/docs/accessibility-testing",
|
|
1315
|
+
visual: "https://playwright.dev/docs/screenshots",
|
|
1316
|
+
screenshot: "https://playwright.dev/docs/screenshots",
|
|
1317
|
+
mobile: "https://playwright.dev/docs/emulation",
|
|
1318
|
+
responsive: "https://playwright.dev/docs/emulation",
|
|
1319
|
+
viewport: "https://playwright.dev/docs/emulation",
|
|
1320
|
+
network: "https://playwright.dev/docs/network",
|
|
1321
|
+
"api call": "https://playwright.dev/docs/network",
|
|
1322
|
+
iframe: "https://playwright.dev/docs/frames",
|
|
1323
|
+
frame: "https://playwright.dev/docs/frames",
|
|
1324
|
+
auth: "https://playwright.dev/docs/auth",
|
|
1325
|
+
"sign in": "https://playwright.dev/docs/auth",
|
|
1326
|
+
"stay logged": "https://playwright.dev/docs/auth",
|
|
1327
|
+
cookie: "https://playwright.dev/docs/auth",
|
|
1328
|
+
storage: "https://playwright.dev/docs/auth",
|
|
1329
|
+
download: "https://playwright.dev/docs/downloads",
|
|
1330
|
+
pdf: "https://playwright.dev/docs/downloads",
|
|
1331
|
+
dialog: "https://playwright.dev/docs/dialogs",
|
|
1332
|
+
alert: "https://playwright.dev/docs/dialogs",
|
|
1333
|
+
confirm: "https://playwright.dev/docs/dialogs",
|
|
1334
|
+
"new tab": "https://playwright.dev/docs/pages",
|
|
1335
|
+
"new page": "https://playwright.dev/docs/pages",
|
|
1336
|
+
popup: "https://playwright.dev/docs/pages"
|
|
1337
|
+
};
|
|
1338
|
+
async function fetchDocContext2(intent) {
|
|
1339
|
+
const intentLower = intent.toLowerCase();
|
|
1340
|
+
const matched = /* @__PURE__ */ new Set();
|
|
1341
|
+
for (const [keyword, url] of Object.entries(ADVANCED_DOC_MAP2)) {
|
|
1342
|
+
if (intentLower.includes(keyword)) {
|
|
1343
|
+
matched.add(url);
|
|
1344
|
+
}
|
|
1345
|
+
}
|
|
1346
|
+
if (matched.size === 0) return "";
|
|
1347
|
+
const fetched = [];
|
|
1348
|
+
for (const url of matched) {
|
|
1349
|
+
try {
|
|
1350
|
+
const res = await fetch(url, { signal: AbortSignal.timeout(5e3) });
|
|
1351
|
+
const html = await res.text();
|
|
1352
|
+
const text = html.replace(/<script[^>]*>[\s\S]*?<\/script>/gi, "").replace(/<style[^>]*>[\s\S]*?<\/style>/gi, "").replace(/<[^>]+>/g, " ").replace(/\s{3,}/g, "\n\n").slice(0, 3e3);
|
|
1353
|
+
fetched.push(`
|
|
1354
|
+
// From ${url}:
|
|
1355
|
+
${text}`);
|
|
1356
|
+
} catch {
|
|
1357
|
+
}
|
|
1358
|
+
}
|
|
1359
|
+
if (fetched.length === 0) return "";
|
|
1360
|
+
return `
|
|
1361
|
+
|
|
1362
|
+
\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501
|
|
1363
|
+
ADDITIONAL PLAYWRIGHT DOCS (fetched for this intent)
|
|
1364
|
+
\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501${fetched.join("\n")}`;
|
|
1365
|
+
}
|
|
665
1366
|
async function analyseElements(elementMap, options = {}) {
|
|
666
|
-
const { verbose = false } = options;
|
|
1367
|
+
const { verbose = false, feature } = options;
|
|
667
1368
|
const providerConfig = resolveLlmProvider();
|
|
1369
|
+
const requestedFeature = normalizeRequestedFeature(feature, elementMap);
|
|
668
1370
|
log(verbose, `
|
|
669
1371
|
[analyse] Sending capture to ${providerConfig.displayName} (${providerConfig.model})`);
|
|
670
|
-
const
|
|
671
|
-
const
|
|
1372
|
+
const docContext = await fetchDocContext2(requestedFeature);
|
|
1373
|
+
const systemPrompt = buildSystemPrompt() + docContext;
|
|
1374
|
+
const userPrompt = buildUserPrompt(elementMap, requestedFeature);
|
|
672
1375
|
const rawResponse = providerConfig.transport === "anthropic" ? await callAnthropic2(providerConfig.apiKey, providerConfig.model, systemPrompt, userPrompt) : await callOpenAiCompatible2(
|
|
673
1376
|
providerConfig.apiKey,
|
|
674
1377
|
providerConfig.model,
|
|
@@ -678,7 +1381,7 @@ async function analyseElements(elementMap, options = {}) {
|
|
|
678
1381
|
userPrompt
|
|
679
1382
|
);
|
|
680
1383
|
const parsed = parseAnalysisJson(rawResponse);
|
|
681
|
-
return sanitizeAnalysis(parsed, elementMap);
|
|
1384
|
+
return sanitizeAnalysis(parsed, elementMap, requestedFeature);
|
|
682
1385
|
}
|
|
683
1386
|
function buildSystemPrompt() {
|
|
684
1387
|
return `
|
|
@@ -688,16 +1391,21 @@ Your job is to analyse a structured map of interactive elements extracted from a
|
|
|
688
1391
|
then produce a complete set of Playwright test scenarios.
|
|
689
1392
|
|
|
690
1393
|
RULES:
|
|
691
|
-
1.
|
|
1394
|
+
1. The user's feature description is the PRIMARY requirement. Generate scenarios that test what the user asked for.
|
|
692
1395
|
2. Every assertion must include an exact "playwright" field with a complete await expect(...) statement.
|
|
693
|
-
3. Use
|
|
694
|
-
4. Include
|
|
1396
|
+
3. Use captured elements as the first selector source. If the intent names text that was not captured, you may fall back to a safe Playwright selector derived from the intent such as getByRole(), getByText(), getByLabel(), getByPlaceholder(), or locator('#id').
|
|
1397
|
+
4. Include happy-path and negative scenarios only when the intent implies interaction, and stay evidence-backed.
|
|
695
1398
|
5. Output only valid JSON matching the requested schema.
|
|
696
1399
|
6. Do not invent redirect targets, success pages, error text, password clearing, or security scenarios unless the capture explicitly supports them.
|
|
697
1400
|
7. If no status or alert region was captured, avoid scenarios that depend on unseen server-side validation messages.
|
|
698
|
-
8.
|
|
699
|
-
9.
|
|
700
|
-
|
|
1401
|
+
8. Scope must match intent complexity. Simple 1 to 2 claim requests should produce only 1 to 2 scenarios.
|
|
1402
|
+
9. For presence-only intents such as "verify X" or "X is present", only assert visibility. Do not click and do not assert outcomes after clicking.
|
|
1403
|
+
${RULE_10_PLAYWRIGHT_KNOWLEDGE_BASE2}
|
|
1404
|
+
11. For fill steps, never use the literal word "value". If matching CT_VAR_* variables are listed in the prompt, use them in human-readable step text and use a realistic non-generic runtime value in the JSON value field. Otherwise use a field-specific fallback such as test-username or test-password.
|
|
1405
|
+
12. The "selector" field must be a raw selector expression such as locator("#username") or getByRole('button', { name: "Login" }). Never wrap selectors in page. or page.locator("...") inside JSON.
|
|
1406
|
+
13. Tie-breaker for multiple matching elements: prefer highest confidence, then interactive elements, then the first visible-looking candidate.
|
|
1407
|
+
14. If a page clearly contains login or auth fields, do not create submit scenarios that only click the button. Include realistic fill steps for captured username/email and password inputs when the intent asks for auth interaction.
|
|
1408
|
+
15. "verify title" means the main visible heading on the page, not the browser tab title, unless the intent explicitly says browser title, tab title, or page title.
|
|
701
1409
|
|
|
702
1410
|
OUTPUT SCHEMA:
|
|
703
1411
|
{
|
|
@@ -734,9 +1442,16 @@ OUTPUT SCHEMA:
|
|
|
734
1442
|
"audioSummary": string
|
|
735
1443
|
}`.trim();
|
|
736
1444
|
}
|
|
737
|
-
function buildUserPrompt(elementMap) {
|
|
1445
|
+
function buildUserPrompt(elementMap, feature) {
|
|
738
1446
|
const lines = [];
|
|
739
1447
|
const authElements = detectAuthElements(elementMap);
|
|
1448
|
+
const availableCtVarKeys = getAvailableCtVarKeys2();
|
|
1449
|
+
const intentProfile = buildIntentProfile(feature);
|
|
1450
|
+
lines.push("USER INTENT");
|
|
1451
|
+
lines.push(`Feature description: ${feature}`);
|
|
1452
|
+
lines.push(`Intent mode: ${intentProfile.mode}`);
|
|
1453
|
+
lines.push(`Maximum scenarios for this request: ${intentProfile.maxScenarios}`);
|
|
1454
|
+
lines.push("");
|
|
740
1455
|
lines.push("PAGE INFORMATION");
|
|
741
1456
|
lines.push(`URL: ${elementMap.url}`);
|
|
742
1457
|
lines.push(`Title: ${elementMap.title}`);
|
|
@@ -762,6 +1477,13 @@ function buildUserPrompt(elementMap) {
|
|
|
762
1477
|
}
|
|
763
1478
|
lines.push("");
|
|
764
1479
|
}
|
|
1480
|
+
if (availableCtVarKeys.length > 0) {
|
|
1481
|
+
lines.push("AVAILABLE CT_VAR_* VARIABLES:");
|
|
1482
|
+
for (const envKey of availableCtVarKeys) {
|
|
1483
|
+
lines.push(` - ${envKey}`);
|
|
1484
|
+
}
|
|
1485
|
+
lines.push("");
|
|
1486
|
+
}
|
|
765
1487
|
const interactiveCount = elementMap.elements.filter((element) => element.category === "input" || element.category === "button" || element.category === "link").length;
|
|
766
1488
|
const statusCount = elementMap.elements.filter((element) => element.category === "status").length;
|
|
767
1489
|
lines.push("EVIDENCE CONSTRAINTS:");
|
|
@@ -769,6 +1491,7 @@ function buildUserPrompt(elementMap) {
|
|
|
769
1491
|
lines.push(` - Status or alert regions captured: ${statusCount}`);
|
|
770
1492
|
lines.push(" - If no redirect target is explicitly captured, do not assert a destination path.");
|
|
771
1493
|
lines.push(" - If no status region was captured, avoid exact server-side credential error claims.");
|
|
1494
|
+
lines.push(" - Generate scenarios only for the requested feature description.");
|
|
772
1495
|
if (authElements.usernameInput && authElements.passwordInput && authElements.submitButton) {
|
|
773
1496
|
lines.push(" - This page contains a captured auth form. Include fill steps for the username/email and password fields in credential-submission scenarios.");
|
|
774
1497
|
lines.push(" - For auth pages without captured post-submit evidence, prefer evidence-backed form-state assertions over invented success redirects.");
|
|
@@ -837,15 +1560,21 @@ ${raw.slice(0, 500)}`
|
|
|
837
1560
|
);
|
|
838
1561
|
}
|
|
839
1562
|
}
|
|
840
|
-
function sanitizeAnalysis(analysis, elementMap) {
|
|
1563
|
+
function sanitizeAnalysis(analysis, elementMap, requestedFeature) {
|
|
841
1564
|
const selectors = /* @__PURE__ */ new Set();
|
|
842
1565
|
for (const element of elementMap.elements) {
|
|
843
1566
|
selectors.add(element.selector);
|
|
844
1567
|
for (const alt of element.selectorAlt) selectors.add(alt);
|
|
845
1568
|
}
|
|
846
1569
|
const rawScenarios = Array.isArray(analysis.scenarios) ? analysis.scenarios : [];
|
|
1570
|
+
const intentProfile = buildIntentProfile(requestedFeature);
|
|
847
1571
|
const currentUrl = new URL(elementMap.url);
|
|
848
1572
|
const authElements = detectAuthElements(elementMap);
|
|
1573
|
+
const resolvedFeature = requestedFeature || analysis.feature || inferFeatureName(elementMap);
|
|
1574
|
+
const resolvedPrefix = (analysis.suggestedPrefix || inferPrefix({
|
|
1575
|
+
featureText: resolvedFeature,
|
|
1576
|
+
url: elementMap.url
|
|
1577
|
+
})).toUpperCase();
|
|
849
1578
|
const knownPaths = /* @__PURE__ */ new Set([
|
|
850
1579
|
currentUrl.pathname,
|
|
851
1580
|
...elementMap.elements.filter((element) => element.category === "link").map((element) => {
|
|
@@ -868,21 +1597,28 @@ function sanitizeAnalysis(analysis, elementMap) {
|
|
|
868
1597
|
if (!pathMatch) return true;
|
|
869
1598
|
return knownPaths.has(pathMatch[0]);
|
|
870
1599
|
})
|
|
871
|
-
})).filter((scenario) => scenario.assertions.length > 0)
|
|
872
|
-
const
|
|
873
|
-
|
|
1600
|
+
})).filter((scenario) => scenario.assertions.length > 0);
|
|
1601
|
+
const intentAlignedScenarios = applyIntentPolicy(
|
|
1602
|
+
sanitizedScenarios,
|
|
1603
|
+
elementMap,
|
|
1604
|
+
intentProfile,
|
|
1605
|
+
resolvedPrefix
|
|
1606
|
+
).slice(0, intentProfile.maxScenarios);
|
|
1607
|
+
const useAuthFallback = shouldUseAuthFallback(authElements, intentAlignedScenarios, intentProfile);
|
|
1608
|
+
const fallbackScenarios = intentProfile.mode === "presence" ? buildPresenceOnlyScenarios(elementMap, requestedFeature, resolvedPrefix) : useAuthFallback ? buildAuthFallbackScenarios(elementMap, resolvedPrefix, authElements) : buildFallbackScenarios(elementMap, resolvedPrefix);
|
|
1609
|
+
const finalScenarios = useAuthFallback ? fallbackScenarios.slice(0, intentProfile.maxScenarios) : intentAlignedScenarios.length > 0 ? intentAlignedScenarios : fallbackScenarios.slice(0, intentProfile.maxScenarios);
|
|
874
1610
|
return {
|
|
875
1611
|
...analysis,
|
|
876
1612
|
url: analysis.url || elementMap.url,
|
|
877
|
-
feature:
|
|
878
|
-
suggestedPrefix:
|
|
1613
|
+
feature: resolvedFeature,
|
|
1614
|
+
suggestedPrefix: resolvedPrefix,
|
|
879
1615
|
scenarios: finalScenarios,
|
|
880
1616
|
analysisNotes: [
|
|
881
1617
|
analysis.analysisNotes,
|
|
882
1618
|
useAuthFallback ? "Replaced low-evidence auth scenarios with deterministic evidence-backed auth coverage." : "",
|
|
883
1619
|
`Sanitized to ${finalScenarios.length} evidence-backed scenario(s) from ${rawScenarios.length} raw scenario(s).`
|
|
884
1620
|
].filter(Boolean).join(" "),
|
|
885
|
-
audioSummary: analysis.audioSummary || buildAudioSummary(
|
|
1621
|
+
audioSummary: analysis.audioSummary || buildAudioSummary(resolvedFeature, finalScenarios)
|
|
886
1622
|
};
|
|
887
1623
|
}
|
|
888
1624
|
function normalizeScenario(candidate, selectors, url) {
|
|
@@ -902,20 +1638,28 @@ function normalizeScenario(candidate, selectors, url) {
|
|
|
902
1638
|
function normalizeStep(candidate, selectors) {
|
|
903
1639
|
if (!candidate) return null;
|
|
904
1640
|
const selector = String(candidate.selector ?? "").trim();
|
|
905
|
-
if (selector
|
|
1641
|
+
if (!isAllowedSelector(selector, selectors)) return null;
|
|
906
1642
|
const action = String(candidate.action ?? "").trim();
|
|
907
1643
|
if (!["navigate", "fill", "click", "select", "check", "keyboard", "hover"].includes(action)) return null;
|
|
1644
|
+
const rawValue = String(candidate.value ?? "");
|
|
1645
|
+
const value = action === "fill" || action === "select" ? resolveFieldRuntimeValue(selector, rawValue) : rawValue;
|
|
1646
|
+
const human = normalizeStepHuman(
|
|
1647
|
+
String(candidate.human ?? "").trim() || defaultStepHuman(action, selector, value),
|
|
1648
|
+
action,
|
|
1649
|
+
selector,
|
|
1650
|
+
value
|
|
1651
|
+
);
|
|
908
1652
|
return {
|
|
909
1653
|
action,
|
|
910
1654
|
selector,
|
|
911
|
-
value
|
|
912
|
-
human
|
|
1655
|
+
value,
|
|
1656
|
+
human
|
|
913
1657
|
};
|
|
914
1658
|
}
|
|
915
1659
|
function normalizeAssertion(candidate, selectors) {
|
|
916
1660
|
if (!candidate) return null;
|
|
917
1661
|
const selector = String(candidate.selector ?? "").trim();
|
|
918
|
-
if (selector
|
|
1662
|
+
if (!isAllowedSelector(selector, selectors)) return null;
|
|
919
1663
|
const assertion = {
|
|
920
1664
|
type: String(candidate.type ?? "visible").trim() || "visible",
|
|
921
1665
|
selector,
|
|
@@ -961,6 +1705,7 @@ function buildFallbackScenarios(elementMap, prefix) {
|
|
|
961
1705
|
});
|
|
962
1706
|
}
|
|
963
1707
|
if (emailInput && passwordInput && submitButton) {
|
|
1708
|
+
const emailValue = resolveFieldRuntimeValue(emailInput.selector, "");
|
|
964
1709
|
scenarios.push({
|
|
965
1710
|
id: nextId(scenarios.length + 1),
|
|
966
1711
|
title: "Submitting without a password keeps the user on the form",
|
|
@@ -975,8 +1720,8 @@ function buildFallbackScenarios(elementMap, prefix) {
|
|
|
975
1720
|
{
|
|
976
1721
|
action: "fill",
|
|
977
1722
|
selector: emailInput.selector,
|
|
978
|
-
value:
|
|
979
|
-
human: "Fill in the email field"
|
|
1723
|
+
value: emailValue,
|
|
1724
|
+
human: buildFillHuman("Fill in the email field", emailInput.selector, emailValue)
|
|
980
1725
|
},
|
|
981
1726
|
{
|
|
982
1727
|
action: "click",
|
|
@@ -1077,13 +1822,13 @@ function buildAuthFallbackScenarios(elementMap, prefix, authElements) {
|
|
|
1077
1822
|
action: "fill",
|
|
1078
1823
|
selector: usernameInput.selector,
|
|
1079
1824
|
value: usernameValue,
|
|
1080
|
-
human: "Fill in the username field"
|
|
1825
|
+
human: buildFillHuman("Fill in the username field", usernameInput.selector, usernameValue)
|
|
1081
1826
|
},
|
|
1082
1827
|
{
|
|
1083
1828
|
action: "fill",
|
|
1084
1829
|
selector: passwordInput.selector,
|
|
1085
1830
|
value: passwordValue,
|
|
1086
|
-
human: "Fill in the password field"
|
|
1831
|
+
human: buildFillHuman("Fill in the password field", passwordInput.selector, passwordValue)
|
|
1087
1832
|
}
|
|
1088
1833
|
],
|
|
1089
1834
|
assertions: [
|
|
@@ -1127,7 +1872,7 @@ function buildAuthFallbackScenarios(elementMap, prefix, authElements) {
|
|
|
1127
1872
|
action: "fill",
|
|
1128
1873
|
selector: usernameInput.selector,
|
|
1129
1874
|
value: usernameValue,
|
|
1130
|
-
human: "Fill in the username field"
|
|
1875
|
+
human: buildFillHuman("Fill in the username field", usernameInput.selector, usernameValue)
|
|
1131
1876
|
},
|
|
1132
1877
|
{
|
|
1133
1878
|
action: "click",
|
|
@@ -1164,16 +1909,166 @@ function buildAuthFallbackScenarios(elementMap, prefix, authElements) {
|
|
|
1164
1909
|
});
|
|
1165
1910
|
return scenarios;
|
|
1166
1911
|
}
|
|
1912
|
+
function normalizeRequestedFeature(feature, elementMap) {
|
|
1913
|
+
return String(feature ?? "").trim() || inferFeatureName(elementMap);
|
|
1914
|
+
}
|
|
1167
1915
|
function inferFeatureName(elementMap) {
|
|
1168
1916
|
const heading = elementMap.elements.find((element) => element.category === "heading" && element.name);
|
|
1169
1917
|
return heading?.name || elementMap.title || "Captured page";
|
|
1170
1918
|
}
|
|
1171
|
-
function
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
return
|
|
1919
|
+
function getAvailableCtVarKeys2() {
|
|
1920
|
+
return Object.keys(process.env).filter((key) => /^CT_VAR_[A-Z0-9_]+$/.test(key)).sort();
|
|
1921
|
+
}
|
|
1922
|
+
function buildIntentProfile(feature) {
|
|
1923
|
+
const normalized = feature.toLowerCase();
|
|
1924
|
+
const countIntent = /\b(?:how many|count|number of|return the count|exactly\s+\d+|there (?:is|are)\s+\d+)/.test(normalized);
|
|
1925
|
+
const wantsHeading = /\b(?:title|heading)\b/.test(normalized) && !/\b(?:browser title|tab title|page title)\b/.test(normalized);
|
|
1926
|
+
const presenceOnly = isPresenceOnlyIntent(normalized);
|
|
1927
|
+
const authIntent = /\b(?:login|log in|sign in|auth|credentials?)\b/.test(normalized) && !presenceOnly;
|
|
1928
|
+
const formIntent = /\b(?:form|submit|field|input|enter|fill|type)\b/.test(normalized) && !presenceOnly && !authIntent;
|
|
1929
|
+
const flowIntent = /\b(?:flow|journey|checkout|purchase|complete|works?)\b/.test(normalized) && !presenceOnly && !authIntent && !formIntent;
|
|
1930
|
+
const mode = countIntent ? "count" : presenceOnly ? "presence" : authIntent ? "auth" : formIntent ? "form" : flowIntent ? "flow" : "generic";
|
|
1931
|
+
const claimCount = countIntentClaims(feature);
|
|
1932
|
+
return {
|
|
1933
|
+
feature,
|
|
1934
|
+
mode,
|
|
1935
|
+
maxScenarios: mode === "count" ? 1 : mode === "presence" ? Math.min(Math.max(claimCount, 1), 2) : mode === "auth" || mode === "form" ? Math.min(Math.max(claimCount, 3), 5) : mode === "flow" ? Math.min(Math.max(claimCount, 4), 6) : Math.min(Math.max(claimCount, 1), 3),
|
|
1936
|
+
wantsHeading
|
|
1937
|
+
};
|
|
1938
|
+
}
|
|
1939
|
+
function countIntentClaims(feature) {
|
|
1940
|
+
const quoted = Array.from(feature.matchAll(/["“”']([^"“”']+)["“”']/g)).length;
|
|
1941
|
+
const segments = feature.split(/\s+(?:and|&)\s+|,\s*/i).map((segment) => segment.trim()).filter(Boolean);
|
|
1942
|
+
return Math.max(quoted, segments.length || 1);
|
|
1943
|
+
}
|
|
1944
|
+
function isPresenceOnlyIntent(normalizedFeature) {
|
|
1945
|
+
const presenceWords = /\b(?:verify|check|confirm|ensure|present|visible|shown|showing|available|exists?|title|heading)\b/.test(normalizedFeature);
|
|
1946
|
+
const actionWords = /\b(?:click|submit|fill|type|enter|login|log in|sign in|checkout|purchase|complete|flow|journey|error|invalid|disabled|enabled)\b/.test(normalizedFeature);
|
|
1947
|
+
return presenceWords && !actionWords;
|
|
1948
|
+
}
|
|
1949
|
+
function applyIntentPolicy(scenarios, elementMap, intentProfile, prefix) {
|
|
1950
|
+
if (intentProfile.mode === "count") {
|
|
1951
|
+
return buildCountScenarios(elementMap, intentProfile.feature, prefix);
|
|
1952
|
+
}
|
|
1953
|
+
if (intentProfile.mode !== "presence") {
|
|
1954
|
+
return scenarios.slice(0, intentProfile.maxScenarios);
|
|
1955
|
+
}
|
|
1956
|
+
return buildPresenceOnlyScenarios(elementMap, intentProfile.feature, prefix);
|
|
1957
|
+
}
|
|
1958
|
+
function buildCountScenarios(elementMap, feature, prefix) {
|
|
1959
|
+
const normalized = feature.toLowerCase();
|
|
1960
|
+
const countedElements = pickCountedElements(elementMap, normalized);
|
|
1961
|
+
const selector = countSelectorForFeature(normalized, countedElements);
|
|
1962
|
+
const count = countedElements.length;
|
|
1963
|
+
return [{
|
|
1964
|
+
id: `${prefix}-${String(1).padStart(3, "0")}`,
|
|
1965
|
+
title: feature,
|
|
1966
|
+
tags: ["@smoke", "@ui"],
|
|
1967
|
+
steps: [
|
|
1968
|
+
{
|
|
1969
|
+
action: "navigate",
|
|
1970
|
+
selector: "page",
|
|
1971
|
+
value: elementMap.url,
|
|
1972
|
+
human: `Navigate to ${elementMap.url}`
|
|
1973
|
+
}
|
|
1974
|
+
],
|
|
1975
|
+
assertions: [{
|
|
1976
|
+
type: "count",
|
|
1977
|
+
selector,
|
|
1978
|
+
expected: String(count),
|
|
1979
|
+
human: `There are exactly ${count} matching elements`,
|
|
1980
|
+
playwright: `await expect(page.${selector}).toHaveCount(${count});`
|
|
1981
|
+
}],
|
|
1982
|
+
narrator: "We count only the element type requested by the user.",
|
|
1983
|
+
codeLevel: "beginner"
|
|
1984
|
+
}];
|
|
1985
|
+
}
|
|
1986
|
+
function buildPresenceOnlyScenarios(elementMap, feature, prefix) {
|
|
1987
|
+
const claims = extractPresenceClaims(feature, elementMap);
|
|
1988
|
+
if (claims.length === 0) return buildFallbackScenarios(elementMap, prefix).slice(0, 1);
|
|
1989
|
+
return [{
|
|
1990
|
+
id: `${prefix}-${String(1).padStart(3, "0")}`,
|
|
1991
|
+
title: feature,
|
|
1992
|
+
tags: ["@smoke", "@ui"],
|
|
1993
|
+
steps: [
|
|
1994
|
+
{
|
|
1995
|
+
action: "navigate",
|
|
1996
|
+
selector: "page",
|
|
1997
|
+
value: elementMap.url,
|
|
1998
|
+
human: `Navigate to ${elementMap.url}`
|
|
1999
|
+
}
|
|
2000
|
+
],
|
|
2001
|
+
assertions: claims.map((claim) => ({
|
|
2002
|
+
type: "visible",
|
|
2003
|
+
selector: claim.selector,
|
|
2004
|
+
expected: "visible",
|
|
2005
|
+
human: claim.human,
|
|
2006
|
+
playwright: visibleAssertion(claim.selector)
|
|
2007
|
+
})),
|
|
2008
|
+
narrator: "We verify only the elements explicitly requested by the user.",
|
|
2009
|
+
codeLevel: "beginner"
|
|
2010
|
+
}];
|
|
2011
|
+
}
|
|
2012
|
+
function extractPresenceClaims(feature, elementMap) {
|
|
2013
|
+
const segments = feature.split(/\s+(?:and|&)\s+|,\s*/i).map((segment) => segment.trim()).filter(Boolean);
|
|
2014
|
+
const claims = [];
|
|
2015
|
+
for (const segment of segments) {
|
|
2016
|
+
const normalized = segment.toLowerCase();
|
|
2017
|
+
if (/\b(?:title|heading)\b/.test(normalized) && !/\b(?:browser title|tab title|page title)\b/.test(normalized)) {
|
|
2018
|
+
claims.push({
|
|
2019
|
+
selector: `getByRole('heading')`,
|
|
2020
|
+
human: `${chooseHeadingElement(elementMap)?.name || "Main page heading"} is visible`
|
|
2021
|
+
});
|
|
2022
|
+
continue;
|
|
2023
|
+
}
|
|
2024
|
+
const text = extractIntentLabel(segment);
|
|
2025
|
+
if (!text) continue;
|
|
2026
|
+
const match = findBestMatchingElement(elementMap, text, normalized);
|
|
2027
|
+
const selector = match?.selector ?? buildFallbackSelector(text, normalized);
|
|
2028
|
+
claims.push({
|
|
2029
|
+
selector,
|
|
2030
|
+
human: `${text} is visible`
|
|
2031
|
+
});
|
|
2032
|
+
}
|
|
2033
|
+
return claims;
|
|
2034
|
+
}
|
|
2035
|
+
function chooseHeadingElement(elementMap) {
|
|
2036
|
+
return rankElements(
|
|
2037
|
+
elementMap.elements.filter((element) => element.category === "heading")
|
|
2038
|
+
)[0];
|
|
2039
|
+
}
|
|
2040
|
+
function extractIntentLabel(segment) {
|
|
2041
|
+
const unquoted = Array.from(segment.matchAll(/["“”']([^"“”']+)["“”']/g)).map((match) => match[1].trim());
|
|
2042
|
+
if (unquoted.length > 0) return unquoted[0];
|
|
2043
|
+
return segment.replace(/\b(?:verify|check|confirm|ensure)\b/gi, " ").replace(/\b(?:the|a|an)\b/gi, " ").replace(/\b(?:is|are|be|should be)\b/gi, " ").replace(/\b(?:present|visible|shown|showing|available|exists?)\b/gi, " ").replace(/\b(?:button|link|cta|label|text)\b/gi, " ").replace(/[.?!,:;]+/g, " ").replace(/\s+/g, " ").trim();
|
|
2044
|
+
}
|
|
2045
|
+
function findBestMatchingElement(elementMap, text, segment) {
|
|
2046
|
+
const preferredCategory = /\bbutton\b/.test(segment) ? "button" : /\blink\b/.test(segment) ? "link" : /\bheading\b/.test(segment) ? "heading" : void 0;
|
|
2047
|
+
const query = text.toLowerCase();
|
|
2048
|
+
const candidates = elementMap.elements.filter((element) => {
|
|
2049
|
+
if (preferredCategory && element.category !== preferredCategory) return false;
|
|
2050
|
+
const haystack = `${element.name ?? ""} ${element.purpose}`.toLowerCase();
|
|
2051
|
+
return haystack.includes(query);
|
|
2052
|
+
});
|
|
2053
|
+
return rankElements(candidates)[0];
|
|
2054
|
+
}
|
|
2055
|
+
function rankElements(elements) {
|
|
2056
|
+
const confidenceWeight = (confidence) => confidence === "high" ? 3 : confidence === "medium" ? 2 : 1;
|
|
2057
|
+
const interactiveWeight = (category) => category === "button" || category === "link" || category === "input" ? 2 : 1;
|
|
2058
|
+
return [...elements].sort((left, right) => {
|
|
2059
|
+
const confidenceDelta = confidenceWeight(right.confidence) - confidenceWeight(left.confidence);
|
|
2060
|
+
if (confidenceDelta !== 0) return confidenceDelta;
|
|
2061
|
+
const interactiveDelta = interactiveWeight(right.category) - interactiveWeight(left.category);
|
|
2062
|
+
if (interactiveDelta !== 0) return interactiveDelta;
|
|
2063
|
+
return 0;
|
|
2064
|
+
});
|
|
2065
|
+
}
|
|
2066
|
+
function buildFallbackSelector(text, segment) {
|
|
2067
|
+
const safeRegex = text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
2068
|
+
if (/\bbutton\b/.test(segment)) return `getByRole('button', { name: /${safeRegex}/i })`;
|
|
2069
|
+
if (/\blink\b/.test(segment)) return `getByRole('link', { name: /${safeRegex}/i })`;
|
|
2070
|
+
if (/\b(?:title|heading)\b/.test(segment)) return `getByRole('heading', { name: /${safeRegex}/i })`;
|
|
2071
|
+
return `getByText(${JSON.stringify(text)})`;
|
|
1177
2072
|
}
|
|
1178
2073
|
function buildAudioSummary(feature, scenarios) {
|
|
1179
2074
|
return `We finished validating ${feature || "the captured page"} with ${scenarios.length} evidence-backed scenario${scenarios.length === 1 ? "" : "s"}.`;
|
|
@@ -1189,7 +2084,8 @@ function detectAuthElements(elementMap) {
|
|
|
1189
2084
|
const heading = elementMap.elements.find((element) => element.category === "heading" && /login|sign in|auth/.test((element.name ?? "").toLowerCase())) ?? elementMap.elements.find((element) => element.category === "heading");
|
|
1190
2085
|
return { usernameInput, passwordInput, submitButton, heading };
|
|
1191
2086
|
}
|
|
1192
|
-
function shouldUseAuthFallback(authElements, scenarios) {
|
|
2087
|
+
function shouldUseAuthFallback(authElements, scenarios, intentProfile) {
|
|
2088
|
+
if (intentProfile.mode !== "auth" && intentProfile.mode !== "form") return false;
|
|
1193
2089
|
if (!authElements.usernameInput || !authElements.passwordInput || !authElements.submitButton) return false;
|
|
1194
2090
|
if (scenarios.length === 0) return true;
|
|
1195
2091
|
const hasCredentialEntry = scenarios.some((scenario) => {
|
|
@@ -1203,13 +2099,37 @@ function shouldUseAuthFallback(authElements, scenarios) {
|
|
|
1203
2099
|
}
|
|
1204
2100
|
function defaultStepHuman(action, selector, value) {
|
|
1205
2101
|
if (action === "navigate") return `Navigate to ${value || "the page"}`;
|
|
1206
|
-
if (action === "fill") return `Fill the field ${selector}
|
|
2102
|
+
if (action === "fill") return buildFillHuman(`Fill the field ${selector}`, selector, value);
|
|
1207
2103
|
if (action === "click") return `Click ${selector}`;
|
|
1208
2104
|
if (action === "select") return `Select ${value} in ${selector}`;
|
|
1209
2105
|
if (action === "check") return `Check ${selector}`;
|
|
1210
2106
|
if (action === "keyboard") return `Press ${value}`;
|
|
1211
2107
|
return `Interact with ${selector}`;
|
|
1212
2108
|
}
|
|
2109
|
+
function normalizeStepHuman(human, action, selector, value) {
|
|
2110
|
+
if (action !== "fill" && action !== "select") return human;
|
|
2111
|
+
if (!value) return human;
|
|
2112
|
+
const explicitTemplate = /\$\{CT_VAR_[A-Z0-9_]+\}/.test(human);
|
|
2113
|
+
const quotedValue = human.match(/\bwith\s+['"`]([^'"`]+)['"`]/i)?.[1];
|
|
2114
|
+
if (explicitTemplate) return human;
|
|
2115
|
+
if (quotedValue && !isGenericFillValue(quotedValue)) return human;
|
|
2116
|
+
const rewrittenBase = human.replace(/\s+with\s+['"`][^'"`]+['"`]\s*$/i, "").trim() || human;
|
|
2117
|
+
return buildFillHuman(rewrittenBase, selector, value);
|
|
2118
|
+
}
|
|
2119
|
+
function pickCountedElements(elementMap, normalizedFeature) {
|
|
2120
|
+
const category = /\bbutton/.test(normalizedFeature) ? "button" : /\blink/.test(normalizedFeature) ? "link" : /\bheading|title/.test(normalizedFeature) ? "heading" : /\binput|field|textbox|text box/.test(normalizedFeature) ? "input" : void 0;
|
|
2121
|
+
if (category) {
|
|
2122
|
+
return elementMap.elements.filter((element) => element.category === category);
|
|
2123
|
+
}
|
|
2124
|
+
return elementMap.elements.filter((element) => element.category === "button" || element.category === "link" || element.category === "input");
|
|
2125
|
+
}
|
|
2126
|
+
function countSelectorForFeature(normalizedFeature, elements) {
|
|
2127
|
+
if (/\bbutton/.test(normalizedFeature)) return `getByRole('button')`;
|
|
2128
|
+
if (/\blink/.test(normalizedFeature)) return `getByRole('link')`;
|
|
2129
|
+
if (/\bheading|title/.test(normalizedFeature)) return `getByRole('heading')`;
|
|
2130
|
+
if (/\binput|field|textbox|text box/.test(normalizedFeature)) return `getByRole('textbox')`;
|
|
2131
|
+
return elements[0]?.selector ?? `locator('*')`;
|
|
2132
|
+
}
|
|
1213
2133
|
function normalizePlaywrightAssertion(assertion) {
|
|
1214
2134
|
const repaired = scopeBareSelectorsInAssertion(unwrapWrappedSelectorAssertion(assertion.playwright));
|
|
1215
2135
|
if (repaired) return ensureStatement(repaired);
|
|
@@ -1264,10 +2184,54 @@ function valueAssertion(selector, value) {
|
|
|
1264
2184
|
return `await expect(${selectorTarget(selector)}).toHaveValue(${JSON.stringify(value)});`;
|
|
1265
2185
|
}
|
|
1266
2186
|
function inferAuthValue(element, kind) {
|
|
1267
|
-
if (kind === "password") return "
|
|
1268
|
-
|
|
1269
|
-
|
|
1270
|
-
|
|
2187
|
+
if (kind === "password") return resolveFieldRuntimeValue(element.selector, "test-password");
|
|
2188
|
+
return resolveFieldRuntimeValue(element.selector, "");
|
|
2189
|
+
}
|
|
2190
|
+
function buildFillHuman(base, selector, runtimeValue) {
|
|
2191
|
+
const displayValue = resolveFieldDisplayValue(selector, runtimeValue);
|
|
2192
|
+
return `${base} with '${displayValue}'`;
|
|
2193
|
+
}
|
|
2194
|
+
function resolveFieldRuntimeValue(selector, currentValue) {
|
|
2195
|
+
if (currentValue && !isGenericFillValue(currentValue) && !extractCtVarTemplate(currentValue)) {
|
|
2196
|
+
return currentValue;
|
|
2197
|
+
}
|
|
2198
|
+
const envKey = matchCtVarKey(selector);
|
|
2199
|
+
if (envKey && process.env[envKey]) {
|
|
2200
|
+
return String(process.env[envKey]);
|
|
2201
|
+
}
|
|
2202
|
+
return fallbackFieldValue(selector);
|
|
2203
|
+
}
|
|
2204
|
+
function resolveFieldDisplayValue(selector, runtimeValue) {
|
|
2205
|
+
const envKey = matchCtVarKey(selector);
|
|
2206
|
+
if (envKey) return `\${${envKey}}`;
|
|
2207
|
+
return runtimeValue;
|
|
2208
|
+
}
|
|
2209
|
+
function matchCtVarKey(selectorOrText) {
|
|
2210
|
+
const haystack = selectorOrText.toLowerCase();
|
|
2211
|
+
const preferred = /\bemail\b/.test(haystack) ? "CT_VAR_EMAIL" : /\bpassword\b/.test(haystack) ? "CT_VAR_PASSWORD" : /\bsearch\b/.test(haystack) ? "CT_VAR_SEARCH" : /\bphone|tel\b/.test(haystack) ? "CT_VAR_PHONE" : /\bname\b/.test(haystack) ? "CT_VAR_NAME" : /\buser(?:name)?|login\b/.test(haystack) ? "CT_VAR_USERNAME" : void 0;
|
|
2212
|
+
if (preferred && process.env[preferred] !== void 0) return preferred;
|
|
2213
|
+
return getAvailableCtVarKeys2().find((key) => new RegExp(key.replace(/^CT_VAR_/, "").toLowerCase()).test(haystack));
|
|
2214
|
+
}
|
|
2215
|
+
function fallbackFieldValue(selectorOrText) {
|
|
2216
|
+
const haystack = selectorOrText.toLowerCase();
|
|
2217
|
+
if (/\bemail\b/.test(haystack)) return "test-email@example.com";
|
|
2218
|
+
if (/\bpassword\b/.test(haystack)) return "test-password";
|
|
2219
|
+
if (/\bsearch\b/.test(haystack)) return "test-search";
|
|
2220
|
+
if (/\bphone|tel\b/.test(haystack)) return "test-phone";
|
|
2221
|
+
if (/\bname\b/.test(haystack) && !/\buser(?:name)?\b/.test(haystack)) return "test-name";
|
|
2222
|
+
if (/\buser(?:name)?|login\b/.test(haystack)) return "test-username";
|
|
2223
|
+
return "test-input";
|
|
2224
|
+
}
|
|
2225
|
+
function isGenericFillValue(value) {
|
|
2226
|
+
return /^(?:value|text|input|option|selection|selected value|default)$/i.test(value.trim());
|
|
2227
|
+
}
|
|
2228
|
+
function extractCtVarTemplate(value) {
|
|
2229
|
+
return value.match(/^\$\{(CT_VAR_[A-Z0-9_]+)\}$/)?.[1];
|
|
2230
|
+
}
|
|
2231
|
+
function isAllowedSelector(selector, selectors) {
|
|
2232
|
+
if (selector === "page") return true;
|
|
2233
|
+
if (selectors.has(selector)) return true;
|
|
2234
|
+
return /^(?:getByRole|getByText|getByLabel|getByPlaceholder|getByTestId)\(/.test(selector) || /^locator\((['"])(#.+?)\1\)$/.test(selector);
|
|
1271
2235
|
}
|
|
1272
2236
|
function ensureStatement(value) {
|
|
1273
2237
|
const trimmed = String(value ?? "").trim();
|
|
@@ -1958,7 +2922,7 @@ function saveSpecPreview(analysis, outputDir = "tests/preview") {
|
|
|
1958
2922
|
if (step.action === "hover") lines.push(` await ${selector}.hover();`);
|
|
1959
2923
|
}
|
|
1960
2924
|
for (const assertion of scenario.assertions) {
|
|
1961
|
-
lines.push(` ${
|
|
2925
|
+
lines.push(` ${renderPreviewAssertion(assertion.playwright, assertion.human)}`);
|
|
1962
2926
|
}
|
|
1963
2927
|
lines.push("});");
|
|
1964
2928
|
lines.push("");
|
|
@@ -1982,6 +2946,15 @@ function ensureStatement2(value) {
|
|
|
1982
2946
|
if (!trimmed) return "// TODO: add assertion";
|
|
1983
2947
|
return trimmed.endsWith(";") ? trimmed : `${trimmed};`;
|
|
1984
2948
|
}
|
|
2949
|
+
function renderPreviewAssertion(playwright, human) {
|
|
2950
|
+
const statement = ensureStatement2(playwright);
|
|
2951
|
+
const headingMatch = statement.match(/^await expect\(page\.getByRole\((['"])heading\1\)\)\.toBeVisible\(\);$/);
|
|
2952
|
+
const visibleSubject = String(human ?? "").match(/^(.+?) is visible$/)?.[1]?.trim();
|
|
2953
|
+
if (headingMatch && visibleSubject && !/main page heading/i.test(visibleSubject)) {
|
|
2954
|
+
return `await expect(page.getByRole("heading", { name: ${JSON.stringify(visibleSubject)} })).toBeVisible();`;
|
|
2955
|
+
}
|
|
2956
|
+
return statement;
|
|
2957
|
+
}
|
|
1985
2958
|
|
|
1986
2959
|
// src/commands/tc.ts
|
|
1987
2960
|
function slugify2(text) {
|
|
@@ -2074,7 +3047,7 @@ async function runTcInteractive(params) {
|
|
|
2074
3047
|
console.log("Capture-only mode requested. No test cases were generated.");
|
|
2075
3048
|
return;
|
|
2076
3049
|
}
|
|
2077
|
-
const analysis = await analyseElements(elementMap, { verbose: true });
|
|
3050
|
+
const analysis = await analyseElements(elementMap, { verbose: true, feature });
|
|
2078
3051
|
printCaptureReport(elementMap, analysis);
|
|
2079
3052
|
const previewPath = saveSpecPreview(analysis);
|
|
2080
3053
|
if (previewPath) {
|