@cementic/cementic-test 0.2.12 → 0.2.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/cli.js CHANGED
@@ -501,6 +501,315 @@ function buildProviderConfigs(env) {
501
501
  function getAvailableCtVarKeys() {
502
502
  return Object.keys(process.env).filter((key) => /^CT_VAR_[A-Z0-9_]+$/.test(key)).sort();
503
503
  }
504
+ var RULE_10_PLAYWRIGHT_KNOWLEDGE_BASE = `
505
+ RULE 10 - PLAYWRIGHT KNOWLEDGE BASE
506
+ (sourced from playwright.dev official documentation)
507
+
508
+ You are an expert in Playwright TypeScript. Apply this knowledge
509
+ when generating test code. Always use web-first assertions that
510
+ auto-wait. Never use manual waits or non-awaited assertions.
511
+
512
+ LOCATOR PRIORITY (use in this order)
513
+ 1. page.getByRole('button', { name: 'Submit' }) <- best
514
+ 2. page.getByLabel('Email address')
515
+ 3. page.getByPlaceholder('Enter email')
516
+ 4. page.getByText('Welcome')
517
+ 5. page.getByAltText('logo')
518
+ 6. page.getByTitle('Close')
519
+ 7. page.getByTestId('submit-btn')
520
+ 8. page.locator('#id') or page.locator('.class') <- last resort
521
+
522
+ Chain locators to narrow scope:
523
+ page.getByRole('listitem').filter({ hasText: 'Product 2' })
524
+ .getByRole('button', { name: 'Add to cart' }).click()
525
+
526
+ LOCATOR ASSERTIONS (always await, always web-first)
527
+ Visibility:
528
+ await expect(locator).toBeVisible()
529
+ await expect(locator).toBeHidden()
530
+ await expect(locator).toBeInViewport()
531
+
532
+ State:
533
+ await expect(locator).toBeEnabled()
534
+ await expect(locator).toBeDisabled()
535
+ await expect(locator).toBeChecked()
536
+ await expect(locator).not.toBeChecked()
537
+ await expect(locator).toBeFocused()
538
+ await expect(locator).toBeEditable()
539
+ await expect(locator).toBeEmpty()
540
+ await expect(locator).toBeAttached()
541
+
542
+ Text content:
543
+ await expect(locator).toHaveText('exact text')
544
+ await expect(locator).toHaveText(/regex/)
545
+ await expect(locator).toContainText('partial')
546
+ await expect(locator).toContainText(['item1', 'item2'])
547
+
548
+ Value and attributes:
549
+ await expect(locator).toHaveValue('input value')
550
+ await expect(locator).toHaveValues(['opt1', 'opt2']) // multi-select
551
+ await expect(locator).toHaveAttribute('href', /pattern/)
552
+ await expect(locator).toHaveClass(/active/)
553
+ await expect(locator).toHaveCSS('color', 'rgb(0,0,0)')
554
+ await expect(locator).toHaveId('submit-btn')
555
+ await expect(locator).toHaveAccessibleName('Submit form')
556
+ await expect(locator).toHaveAccessibleDescription('...')
557
+
558
+ Counting (PREFER toHaveCount over .count() to avoid flakiness):
559
+ await expect(locator).toHaveCount(3)
560
+ await expect(locator).toHaveCount(0) // none exist
561
+ // Only use .count() when you need the actual number:
562
+ const n = await page.getByRole('button').count();
563
+ console.log(\`Found \${n} buttons\`);
564
+
565
+ PAGE ASSERTIONS
566
+ await expect(page).toHaveTitle(/Playwright/)
567
+ await expect(page).toHaveTitle('Exact Title')
568
+ await expect(page).toHaveURL('https://example.com/dashboard')
569
+ await expect(page).toHaveURL(/\\/dashboard/)
570
+
571
+ ACTIONS (Playwright auto-waits before each action)
572
+ Navigation:
573
+ await page.goto('https://example.com')
574
+ await page.goBack()
575
+ await page.goForward()
576
+ await page.reload()
577
+
578
+ Clicking:
579
+ await locator.click()
580
+ await locator.dblclick()
581
+ await locator.click({ button: 'right' })
582
+ await locator.click({ modifiers: ['Shift'] })
583
+
584
+ Forms:
585
+ await locator.fill('text') // clears then types
586
+ await locator.clear()
587
+ await locator.pressSequentially('slow typing', { delay: 50 })
588
+ await locator.selectOption('value')
589
+ await locator.selectOption({ label: 'Blue' })
590
+ await locator.check()
591
+ await locator.uncheck()
592
+ await locator.setInputFiles('path/to/file.pdf')
593
+
594
+ Hover and focus:
595
+ await locator.hover()
596
+ await locator.focus()
597
+ await locator.blur()
598
+
599
+ Keyboard:
600
+ await page.keyboard.press('Enter')
601
+ await page.keyboard.press('Tab')
602
+ await page.keyboard.press('Escape')
603
+ await page.keyboard.press('Control+A')
604
+
605
+ Scroll:
606
+ await locator.scrollIntoViewIfNeeded()
607
+ await page.mouse.wheel(0, 500)
608
+
609
+ BEST PRACTICES (from playwright.dev/docs/best-practices)
610
+ DO use web-first assertions:
611
+ await expect(page.getByText('welcome')).toBeVisible()
612
+
613
+ NEVER use synchronous assertions:
614
+ expect(await page.getByText('welcome').isVisible()).toBe(true)
615
+
616
+ DO chain locators:
617
+ page.getByRole('listitem').filter({ hasText: 'Product 2' })
618
+ .getByRole('button', { name: 'Add to cart' })
619
+
620
+ NEVER use fragile CSS/XPath selectors:
621
+ page.locator('button.buttonIcon.episode-actions-later')
622
+
623
+ DO use role-based locators:
624
+ page.getByRole('button', { name: 'submit' })
625
+
626
+ INTENT -> PATTERN MAPPING
627
+
628
+ COUNTING:
629
+ "how many X" / "count X" / "number of X" / "return the count"
630
+ -> const n = await page.getByRole('X').count();
631
+ console.log(\`Found \${n} X elements\`);
632
+ expect(n).toBeGreaterThan(0);
633
+
634
+ "there are N X" / "exactly N X"
635
+ -> await expect(page.getByRole('X')).toHaveCount(N);
636
+
637
+ CRITICAL: Never use getByText("how many X") - count intent
638
+ means call .count() or toHaveCount(), not search for text.
639
+
640
+ PRESENCE:
641
+ "X is present/visible/shown/exists"
642
+ -> await expect(locator).toBeVisible()
643
+
644
+ "X is hidden/not present/gone"
645
+ -> await expect(locator).toBeHidden()
646
+
647
+ TEXT:
648
+ "text says X" / "shows X" / "message is X"
649
+ -> await expect(locator).toHaveText('X')
650
+
651
+ "contains X" / "includes X"
652
+ -> await expect(locator).toContainText('X')
653
+
654
+ "page title is X"
655
+ -> await expect(page).toHaveTitle(/X/i)
656
+
657
+ "heading says X"
658
+ -> await expect(page.getByRole('heading')).toContainText('X')
659
+
660
+ "error says X"
661
+ -> await expect(page.getByRole('alert')).toContainText('X')
662
+
663
+ STATE:
664
+ "X is enabled/disabled"
665
+ -> await expect(locator).toBeEnabled() / toBeDisabled()
666
+
667
+ "X is checked/unchecked"
668
+ -> await expect(locator).toBeChecked() / not.toBeChecked()
669
+
670
+ "X has value Y"
671
+ -> await expect(locator).toHaveValue('Y')
672
+
673
+ "X is active / has class Y"
674
+ -> await expect(locator).toHaveClass(/Y/)
675
+
676
+ NAVIGATION:
677
+ "redirects to X" / "goes to X" / "URL contains X"
678
+ -> await expect(page).toHaveURL(/X/)
679
+
680
+ "stays on same page"
681
+ -> await expect(page).toHaveURL(/currentPath/)
682
+
683
+ "opens new tab"
684
+ -> const [newPage] = await Promise.all([
685
+ context.waitForEvent('page'),
686
+ locator.click()
687
+ ]);
688
+
689
+ FORMS:
690
+ "form submits successfully"
691
+ -> fill fields + click submit + assert URL change or success message
692
+
693
+ "validation error shown"
694
+ -> submit empty + await expect(page.getByRole('alert')).toBeVisible()
695
+
696
+ "required field X"
697
+ -> submit without X + assert error message for X visible
698
+
699
+ AUTH:
700
+ "login with valid credentials"
701
+ -> fill(CT_VAR_USERNAME) + fill(CT_VAR_PASSWORD) + click login
702
+ + await expect(page).toHaveURL(/dashboard|home|app/)
703
+
704
+ "login fails / invalid credentials"
705
+ -> fill bad values + click login
706
+ + await expect(page.getByRole('alert')).toBeVisible()
707
+
708
+ "logout works"
709
+ -> click logout + await expect(page).toHaveURL(/login|home/)
710
+
711
+ THEME / VISUAL:
712
+ "dark mode / theme toggle"
713
+ -> await locator.click()
714
+ + await expect(page.locator('html')).toHaveAttribute('data-theme', 'dark')
715
+ // OR: await expect(page.locator('body')).toHaveClass(/dark/)
716
+
717
+ TIMING:
718
+ "X loads / X appears"
719
+ -> await expect(locator).toBeVisible({ timeout: 10000 })
720
+
721
+ "spinner disappears / loader gone"
722
+ -> await expect(spinner).toBeHidden({ timeout: 10000 })
723
+
724
+ "modal closes"
725
+ -> await expect(modal).toBeHidden()
726
+
727
+ ACCESSIBILITY:
728
+ "has alt text"
729
+ -> await expect(page.locator('img')).toHaveAttribute('alt', /.+/)
730
+
731
+ "keyboard navigable"
732
+ -> await page.keyboard.press('Tab')
733
+ + await expect(firstFocusable).toBeFocused()
734
+
735
+ CRITICAL RULES
736
+ 1. NEVER use intent words as locator text.
737
+ "count buttons" != getByText("count buttons")
738
+ "verify heading" != getByText("verify heading")
739
+
740
+ 2. ALWAYS use web-first assertions (await expect).
741
+ NEVER use: expect(await locator.isVisible()).toBe(true)
742
+
743
+ 3. For counting, prefer toHaveCount() over .count() unless
744
+ you need the actual number for logging.
745
+
746
+ 4. Auto-waiting: Playwright automatically waits for elements
747
+ to be actionable before click/fill/etc. Do not add
748
+ manual waitForTimeout() unless testing timing specifically.
749
+
750
+ 5. Use not. prefix for negative assertions:
751
+ await expect(locator).not.toBeVisible()
752
+ await expect(locator).not.toBeChecked()
753
+ `.trim();
754
+ var ADVANCED_DOC_MAP = {
755
+ upload: "https://playwright.dev/docs/input",
756
+ "file upload": "https://playwright.dev/docs/input",
757
+ intercept: "https://playwright.dev/docs/mock",
758
+ mock: "https://playwright.dev/docs/mock",
759
+ "api mock": "https://playwright.dev/docs/mock",
760
+ accessibility: "https://playwright.dev/docs/accessibility-testing",
761
+ "screen reader": "https://playwright.dev/docs/accessibility-testing",
762
+ visual: "https://playwright.dev/docs/screenshots",
763
+ screenshot: "https://playwright.dev/docs/screenshots",
764
+ mobile: "https://playwright.dev/docs/emulation",
765
+ responsive: "https://playwright.dev/docs/emulation",
766
+ viewport: "https://playwright.dev/docs/emulation",
767
+ network: "https://playwright.dev/docs/network",
768
+ "api call": "https://playwright.dev/docs/network",
769
+ iframe: "https://playwright.dev/docs/frames",
770
+ frame: "https://playwright.dev/docs/frames",
771
+ auth: "https://playwright.dev/docs/auth",
772
+ "sign in": "https://playwright.dev/docs/auth",
773
+ "stay logged": "https://playwright.dev/docs/auth",
774
+ cookie: "https://playwright.dev/docs/auth",
775
+ storage: "https://playwright.dev/docs/auth",
776
+ download: "https://playwright.dev/docs/downloads",
777
+ pdf: "https://playwright.dev/docs/downloads",
778
+ dialog: "https://playwright.dev/docs/dialogs",
779
+ alert: "https://playwright.dev/docs/dialogs",
780
+ confirm: "https://playwright.dev/docs/dialogs",
781
+ "new tab": "https://playwright.dev/docs/pages",
782
+ "new page": "https://playwright.dev/docs/pages",
783
+ popup: "https://playwright.dev/docs/pages"
784
+ };
785
+ async function fetchDocContext(intent) {
786
+ const intentLower = intent.toLowerCase();
787
+ const matched = /* @__PURE__ */ new Set();
788
+ for (const [keyword, url] of Object.entries(ADVANCED_DOC_MAP)) {
789
+ if (intentLower.includes(keyword)) {
790
+ matched.add(url);
791
+ }
792
+ }
793
+ if (matched.size === 0) return "";
794
+ const fetched = [];
795
+ for (const url of matched) {
796
+ try {
797
+ const res = await fetch(url, { signal: AbortSignal.timeout(5e3) });
798
+ const html = await res.text();
799
+ const text = html.replace(/<script[^>]*>[\s\S]*?<\/script>/gi, "").replace(/<style[^>]*>[\s\S]*?<\/style>/gi, "").replace(/<[^>]+>/g, " ").replace(/\s{3,}/g, "\n\n").slice(0, 3e3);
800
+ fetched.push(`
801
+ // From ${url}:
802
+ ${text}`);
803
+ } catch {
804
+ }
805
+ }
806
+ if (fetched.length === 0) return "";
807
+ return `
808
+
809
+ \u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501
810
+ ADDITIONAL PLAYWRIGHT DOCS (fetched for this intent)
811
+ \u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501${fetched.join("\n")}`;
812
+ }
504
813
  function buildSystemMessage() {
505
814
  return `
506
815
  You are a senior software test architect with 10 years of Playwright experience.
@@ -567,6 +876,8 @@ Match scenario count to intent complexity:
567
876
  - Full flow: 4 to 8 scenarios maximum
568
877
  Never exceed 8 scenarios.
569
878
 
879
+ ${RULE_10_PLAYWRIGHT_KNOWLEDGE_BASE}
880
+
570
881
  COMMON TESTING VOCABULARY
571
882
  "verify/check X is present/visible" -> toBeVisible()
572
883
  "verify title/heading" -> heading is visible
@@ -725,7 +1036,8 @@ async function callOpenAiCompatible(apiKey, model, baseUrl, displayName, system,
725
1036
  }
726
1037
  async function generateTcMarkdownWithAi(ctx) {
727
1038
  const providerConfig = resolveLlmProvider();
728
- const system = buildSystemMessage();
1039
+ const docContext = await fetchDocContext(ctx.feature);
1040
+ const system = buildSystemMessage() + docContext;
729
1041
  const user = buildUserMessage(ctx);
730
1042
  console.log(`\u{1F916} Using ${providerConfig.displayName} (${providerConfig.model})`);
731
1043
  if (providerConfig.transport === "anthropic") {
@@ -742,13 +1054,323 @@ async function generateTcMarkdownWithAi(ctx) {
742
1054
  }
743
1055
 
744
1056
  // src/core/analyse.ts
1057
+ var RULE_10_PLAYWRIGHT_KNOWLEDGE_BASE2 = `
1058
+ RULE 10 - PLAYWRIGHT KNOWLEDGE BASE
1059
+ (sourced from playwright.dev official documentation)
1060
+
1061
+ You are an expert in Playwright TypeScript. Apply this knowledge
1062
+ when generating test code. Always use web-first assertions that
1063
+ auto-wait. Never use manual waits or non-awaited assertions.
1064
+
1065
+ LOCATOR PRIORITY (use in this order)
1066
+ 1. page.getByRole('button', { name: 'Submit' }) <- best
1067
+ 2. page.getByLabel('Email address')
1068
+ 3. page.getByPlaceholder('Enter email')
1069
+ 4. page.getByText('Welcome')
1070
+ 5. page.getByAltText('logo')
1071
+ 6. page.getByTitle('Close')
1072
+ 7. page.getByTestId('submit-btn')
1073
+ 8. page.locator('#id') or page.locator('.class') <- last resort
1074
+
1075
+ Chain locators to narrow scope:
1076
+ page.getByRole('listitem').filter({ hasText: 'Product 2' })
1077
+ .getByRole('button', { name: 'Add to cart' }).click()
1078
+
1079
+ LOCATOR ASSERTIONS (always await, always web-first)
1080
+ Visibility:
1081
+ await expect(locator).toBeVisible()
1082
+ await expect(locator).toBeHidden()
1083
+ await expect(locator).toBeInViewport()
1084
+
1085
+ State:
1086
+ await expect(locator).toBeEnabled()
1087
+ await expect(locator).toBeDisabled()
1088
+ await expect(locator).toBeChecked()
1089
+ await expect(locator).not.toBeChecked()
1090
+ await expect(locator).toBeFocused()
1091
+ await expect(locator).toBeEditable()
1092
+ await expect(locator).toBeEmpty()
1093
+ await expect(locator).toBeAttached()
1094
+
1095
+ Text content:
1096
+ await expect(locator).toHaveText('exact text')
1097
+ await expect(locator).toHaveText(/regex/)
1098
+ await expect(locator).toContainText('partial')
1099
+ await expect(locator).toContainText(['item1', 'item2'])
1100
+
1101
+ Value and attributes:
1102
+ await expect(locator).toHaveValue('input value')
1103
+ await expect(locator).toHaveValues(['opt1', 'opt2']) // multi-select
1104
+ await expect(locator).toHaveAttribute('href', /pattern/)
1105
+ await expect(locator).toHaveClass(/active/)
1106
+ await expect(locator).toHaveCSS('color', 'rgb(0,0,0)')
1107
+ await expect(locator).toHaveId('submit-btn')
1108
+ await expect(locator).toHaveAccessibleName('Submit form')
1109
+ await expect(locator).toHaveAccessibleDescription('...')
1110
+
1111
+ Counting (PREFER toHaveCount over .count() to avoid flakiness):
1112
+ await expect(locator).toHaveCount(3)
1113
+ await expect(locator).toHaveCount(0) // none exist
1114
+ // Only use .count() when you need the actual number:
1115
+ const n = await page.getByRole('button').count();
1116
+ console.log(\`Found \${n} buttons\`);
1117
+
1118
+ PAGE ASSERTIONS
1119
+ await expect(page).toHaveTitle(/Playwright/)
1120
+ await expect(page).toHaveTitle('Exact Title')
1121
+ await expect(page).toHaveURL('https://example.com/dashboard')
1122
+ await expect(page).toHaveURL(/\\/dashboard/)
1123
+
1124
+ ACTIONS (Playwright auto-waits before each action)
1125
+ Navigation:
1126
+ await page.goto('https://example.com')
1127
+ await page.goBack()
1128
+ await page.goForward()
1129
+ await page.reload()
1130
+
1131
+ Clicking:
1132
+ await locator.click()
1133
+ await locator.dblclick()
1134
+ await locator.click({ button: 'right' })
1135
+ await locator.click({ modifiers: ['Shift'] })
1136
+
1137
+ Forms:
1138
+ await locator.fill('text') // clears then types
1139
+ await locator.clear()
1140
+ await locator.pressSequentially('slow typing', { delay: 50 })
1141
+ await locator.selectOption('value')
1142
+ await locator.selectOption({ label: 'Blue' })
1143
+ await locator.check()
1144
+ await locator.uncheck()
1145
+ await locator.setInputFiles('path/to/file.pdf')
1146
+
1147
+ Hover and focus:
1148
+ await locator.hover()
1149
+ await locator.focus()
1150
+ await locator.blur()
1151
+
1152
+ Keyboard:
1153
+ await page.keyboard.press('Enter')
1154
+ await page.keyboard.press('Tab')
1155
+ await page.keyboard.press('Escape')
1156
+ await page.keyboard.press('Control+A')
1157
+
1158
+ Scroll:
1159
+ await locator.scrollIntoViewIfNeeded()
1160
+ await page.mouse.wheel(0, 500)
1161
+
1162
+ BEST PRACTICES (from playwright.dev/docs/best-practices)
1163
+ DO use web-first assertions:
1164
+ await expect(page.getByText('welcome')).toBeVisible()
1165
+
1166
+ NEVER use synchronous assertions:
1167
+ expect(await page.getByText('welcome').isVisible()).toBe(true)
1168
+
1169
+ DO chain locators:
1170
+ page.getByRole('listitem').filter({ hasText: 'Product 2' })
1171
+ .getByRole('button', { name: 'Add to cart' })
1172
+
1173
+ NEVER use fragile CSS/XPath selectors:
1174
+ page.locator('button.buttonIcon.episode-actions-later')
1175
+
1176
+ DO use role-based locators:
1177
+ page.getByRole('button', { name: 'submit' })
1178
+
1179
+ INTENT -> PATTERN MAPPING
1180
+
1181
+ COUNTING:
1182
+ "how many X" / "count X" / "number of X" / "return the count"
1183
+ -> const n = await page.getByRole('X').count();
1184
+ console.log(\`Found \${n} X elements\`);
1185
+ expect(n).toBeGreaterThan(0);
1186
+
1187
+ "there are N X" / "exactly N X"
1188
+ -> await expect(page.getByRole('X')).toHaveCount(N);
1189
+
1190
+ CRITICAL: Never use getByText("how many X") - count intent
1191
+ means call .count() or toHaveCount(), not search for text.
1192
+
1193
+ PRESENCE:
1194
+ "X is present/visible/shown/exists"
1195
+ -> await expect(locator).toBeVisible()
1196
+
1197
+ "X is hidden/not present/gone"
1198
+ -> await expect(locator).toBeHidden()
1199
+
1200
+ TEXT:
1201
+ "text says X" / "shows X" / "message is X"
1202
+ -> await expect(locator).toHaveText('X')
1203
+
1204
+ "contains X" / "includes X"
1205
+ -> await expect(locator).toContainText('X')
1206
+
1207
+ "page title is X"
1208
+ -> await expect(page).toHaveTitle(/X/i)
1209
+
1210
+ "heading says X"
1211
+ -> await expect(page.getByRole('heading')).toContainText('X')
1212
+
1213
+ "error says X"
1214
+ -> await expect(page.getByRole('alert')).toContainText('X')
1215
+
1216
+ STATE:
1217
+ "X is enabled/disabled"
1218
+ -> await expect(locator).toBeEnabled() / toBeDisabled()
1219
+
1220
+ "X is checked/unchecked"
1221
+ -> await expect(locator).toBeChecked() / not.toBeChecked()
1222
+
1223
+ "X has value Y"
1224
+ -> await expect(locator).toHaveValue('Y')
1225
+
1226
+ "X is active / has class Y"
1227
+ -> await expect(locator).toHaveClass(/Y/)
1228
+
1229
+ NAVIGATION:
1230
+ "redirects to X" / "goes to X" / "URL contains X"
1231
+ -> await expect(page).toHaveURL(/X/)
1232
+
1233
+ "stays on same page"
1234
+ -> await expect(page).toHaveURL(/currentPath/)
1235
+
1236
+ "opens new tab"
1237
+ -> const [newPage] = await Promise.all([
1238
+ context.waitForEvent('page'),
1239
+ locator.click()
1240
+ ]);
1241
+
1242
+ FORMS:
1243
+ "form submits successfully"
1244
+ -> fill fields + click submit + assert URL change or success message
1245
+
1246
+ "validation error shown"
1247
+ -> submit empty + await expect(page.getByRole('alert')).toBeVisible()
1248
+
1249
+ "required field X"
1250
+ -> submit without X + assert error message for X visible
1251
+
1252
+ AUTH:
1253
+ "login with valid credentials"
1254
+ -> fill(CT_VAR_USERNAME) + fill(CT_VAR_PASSWORD) + click login
1255
+ + await expect(page).toHaveURL(/dashboard|home|app/)
1256
+
1257
+ "login fails / invalid credentials"
1258
+ -> fill bad values + click login
1259
+ + await expect(page.getByRole('alert')).toBeVisible()
1260
+
1261
+ "logout works"
1262
+ -> click logout + await expect(page).toHaveURL(/login|home/)
1263
+
1264
+ THEME / VISUAL:
1265
+ "dark mode / theme toggle"
1266
+ -> await locator.click()
1267
+ + await expect(page.locator('html')).toHaveAttribute('data-theme', 'dark')
1268
+ // OR: await expect(page.locator('body')).toHaveClass(/dark/)
1269
+
1270
+ TIMING:
1271
+ "X loads / X appears"
1272
+ -> await expect(locator).toBeVisible({ timeout: 10000 })
1273
+
1274
+ "spinner disappears / loader gone"
1275
+ -> await expect(spinner).toBeHidden({ timeout: 10000 })
1276
+
1277
+ "modal closes"
1278
+ -> await expect(modal).toBeHidden()
1279
+
1280
+ ACCESSIBILITY:
1281
+ "has alt text"
1282
+ -> await expect(page.locator('img')).toHaveAttribute('alt', /.+/)
1283
+
1284
+ "keyboard navigable"
1285
+ -> await page.keyboard.press('Tab')
1286
+ + await expect(firstFocusable).toBeFocused()
1287
+
1288
+ CRITICAL RULES
1289
+ 1. NEVER use intent words as locator text.
1290
+ "count buttons" != getByText("count buttons")
1291
+ "verify heading" != getByText("verify heading")
1292
+
1293
+ 2. ALWAYS use web-first assertions (await expect).
1294
+ NEVER use: expect(await locator.isVisible()).toBe(true)
1295
+
1296
+ 3. For counting, prefer toHaveCount() over .count() unless
1297
+ you need the actual number for logging.
1298
+
1299
+ 4. Auto-waiting: Playwright automatically waits for elements
1300
+ to be actionable before click/fill/etc. Do not add
1301
+ manual waitForTimeout() unless testing timing specifically.
1302
+
1303
+ 5. Use not. prefix for negative assertions:
1304
+ await expect(locator).not.toBeVisible()
1305
+ await expect(locator).not.toBeChecked()
1306
+ `.trim();
1307
+ var ADVANCED_DOC_MAP2 = {
1308
+ upload: "https://playwright.dev/docs/input",
1309
+ "file upload": "https://playwright.dev/docs/input",
1310
+ intercept: "https://playwright.dev/docs/mock",
1311
+ mock: "https://playwright.dev/docs/mock",
1312
+ "api mock": "https://playwright.dev/docs/mock",
1313
+ accessibility: "https://playwright.dev/docs/accessibility-testing",
1314
+ "screen reader": "https://playwright.dev/docs/accessibility-testing",
1315
+ visual: "https://playwright.dev/docs/screenshots",
1316
+ screenshot: "https://playwright.dev/docs/screenshots",
1317
+ mobile: "https://playwright.dev/docs/emulation",
1318
+ responsive: "https://playwright.dev/docs/emulation",
1319
+ viewport: "https://playwright.dev/docs/emulation",
1320
+ network: "https://playwright.dev/docs/network",
1321
+ "api call": "https://playwright.dev/docs/network",
1322
+ iframe: "https://playwright.dev/docs/frames",
1323
+ frame: "https://playwright.dev/docs/frames",
1324
+ auth: "https://playwright.dev/docs/auth",
1325
+ "sign in": "https://playwright.dev/docs/auth",
1326
+ "stay logged": "https://playwright.dev/docs/auth",
1327
+ cookie: "https://playwright.dev/docs/auth",
1328
+ storage: "https://playwright.dev/docs/auth",
1329
+ download: "https://playwright.dev/docs/downloads",
1330
+ pdf: "https://playwright.dev/docs/downloads",
1331
+ dialog: "https://playwright.dev/docs/dialogs",
1332
+ alert: "https://playwright.dev/docs/dialogs",
1333
+ confirm: "https://playwright.dev/docs/dialogs",
1334
+ "new tab": "https://playwright.dev/docs/pages",
1335
+ "new page": "https://playwright.dev/docs/pages",
1336
+ popup: "https://playwright.dev/docs/pages"
1337
+ };
1338
+ async function fetchDocContext2(intent) {
1339
+ const intentLower = intent.toLowerCase();
1340
+ const matched = /* @__PURE__ */ new Set();
1341
+ for (const [keyword, url] of Object.entries(ADVANCED_DOC_MAP2)) {
1342
+ if (intentLower.includes(keyword)) {
1343
+ matched.add(url);
1344
+ }
1345
+ }
1346
+ if (matched.size === 0) return "";
1347
+ const fetched = [];
1348
+ for (const url of matched) {
1349
+ try {
1350
+ const res = await fetch(url, { signal: AbortSignal.timeout(5e3) });
1351
+ const html = await res.text();
1352
+ const text = html.replace(/<script[^>]*>[\s\S]*?<\/script>/gi, "").replace(/<style[^>]*>[\s\S]*?<\/style>/gi, "").replace(/<[^>]+>/g, " ").replace(/\s{3,}/g, "\n\n").slice(0, 3e3);
1353
+ fetched.push(`
1354
+ // From ${url}:
1355
+ ${text}`);
1356
+ } catch {
1357
+ }
1358
+ }
1359
+ if (fetched.length === 0) return "";
1360
+ return `
1361
+
1362
+ \u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501
1363
+ ADDITIONAL PLAYWRIGHT DOCS (fetched for this intent)
1364
+ \u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501${fetched.join("\n")}`;
1365
+ }
745
1366
  async function analyseElements(elementMap, options = {}) {
746
1367
  const { verbose = false, feature } = options;
747
1368
  const providerConfig = resolveLlmProvider();
748
1369
  const requestedFeature = normalizeRequestedFeature(feature, elementMap);
749
1370
  log(verbose, `
750
1371
  [analyse] Sending capture to ${providerConfig.displayName} (${providerConfig.model})`);
751
- const systemPrompt = buildSystemPrompt();
1372
+ const docContext = await fetchDocContext2(requestedFeature);
1373
+ const systemPrompt = buildSystemPrompt() + docContext;
752
1374
  const userPrompt = buildUserPrompt(elementMap, requestedFeature);
753
1375
  const rawResponse = providerConfig.transport === "anthropic" ? await callAnthropic2(providerConfig.apiKey, providerConfig.model, systemPrompt, userPrompt) : await callOpenAiCompatible2(
754
1376
  providerConfig.apiKey,
@@ -778,11 +1400,12 @@ RULES:
778
1400
  7. If no status or alert region was captured, avoid scenarios that depend on unseen server-side validation messages.
779
1401
  8. Scope must match intent complexity. Simple 1 to 2 claim requests should produce only 1 to 2 scenarios.
780
1402
  9. For presence-only intents such as "verify X" or "X is present", only assert visibility. Do not click and do not assert outcomes after clicking.
781
- 10. For fill steps, never use the literal word "value". If matching CT_VAR_* variables are listed in the prompt, use them in human-readable step text and use a realistic non-generic runtime value in the JSON value field. Otherwise use a field-specific fallback such as test-username or test-password.
782
- 11. The "selector" field must be a raw selector expression such as locator("#username") or getByRole('button', { name: "Login" }). Never wrap selectors in page. or page.locator("...") inside JSON.
783
- 12. Tie-breaker for multiple matching elements: prefer highest confidence, then interactive elements, then the first visible-looking candidate.
784
- 13. If a page clearly contains login or auth fields, do not create submit scenarios that only click the button. Include realistic fill steps for captured username/email and password inputs when the intent asks for auth interaction.
785
- 14. "verify title" means the main visible heading on the page, not the browser tab title, unless the intent explicitly says browser title, tab title, or page title.
1403
+ ${RULE_10_PLAYWRIGHT_KNOWLEDGE_BASE2}
1404
+ 11. For fill steps, never use the literal word "value". If matching CT_VAR_* variables are listed in the prompt, use them in human-readable step text and use a realistic non-generic runtime value in the JSON value field. Otherwise use a field-specific fallback such as test-username or test-password.
1405
+ 12. The "selector" field must be a raw selector expression such as locator("#username") or getByRole('button', { name: "Login" }). Never wrap selectors in page. or page.locator("...") inside JSON.
1406
+ 13. Tie-breaker for multiple matching elements: prefer highest confidence, then interactive elements, then the first visible-looking candidate.
1407
+ 14. If a page clearly contains login or auth fields, do not create submit scenarios that only click the button. Include realistic fill steps for captured username/email and password inputs when the intent asks for auth interaction.
1408
+ 15. "verify title" means the main visible heading on the page, not the browser tab title, unless the intent explicitly says browser title, tab title, or page title.
786
1409
 
787
1410
  OUTPUT SCHEMA:
788
1411
  {
@@ -1298,17 +1921,18 @@ function getAvailableCtVarKeys2() {
1298
1921
  }
1299
1922
  function buildIntentProfile(feature) {
1300
1923
  const normalized = feature.toLowerCase();
1924
+ const countIntent = /\b(?:how many|count|number of|return the count|exactly\s+\d+|there (?:is|are)\s+\d+)/.test(normalized);
1301
1925
  const wantsHeading = /\b(?:title|heading)\b/.test(normalized) && !/\b(?:browser title|tab title|page title)\b/.test(normalized);
1302
1926
  const presenceOnly = isPresenceOnlyIntent(normalized);
1303
1927
  const authIntent = /\b(?:login|log in|sign in|auth|credentials?)\b/.test(normalized) && !presenceOnly;
1304
1928
  const formIntent = /\b(?:form|submit|field|input|enter|fill|type)\b/.test(normalized) && !presenceOnly && !authIntent;
1305
1929
  const flowIntent = /\b(?:flow|journey|checkout|purchase|complete|works?)\b/.test(normalized) && !presenceOnly && !authIntent && !formIntent;
1306
- const mode = presenceOnly ? "presence" : authIntent ? "auth" : formIntent ? "form" : flowIntent ? "flow" : "generic";
1930
+ const mode = countIntent ? "count" : presenceOnly ? "presence" : authIntent ? "auth" : formIntent ? "form" : flowIntent ? "flow" : "generic";
1307
1931
  const claimCount = countIntentClaims(feature);
1308
1932
  return {
1309
1933
  feature,
1310
1934
  mode,
1311
- maxScenarios: mode === "presence" ? Math.min(Math.max(claimCount, 1), 2) : mode === "auth" || mode === "form" ? Math.min(Math.max(claimCount, 3), 5) : mode === "flow" ? Math.min(Math.max(claimCount, 4), 6) : Math.min(Math.max(claimCount, 1), 3),
1935
+ maxScenarios: mode === "count" ? 1 : mode === "presence" ? Math.min(Math.max(claimCount, 1), 2) : mode === "auth" || mode === "form" ? Math.min(Math.max(claimCount, 3), 5) : mode === "flow" ? Math.min(Math.max(claimCount, 4), 6) : Math.min(Math.max(claimCount, 1), 3),
1312
1936
  wantsHeading
1313
1937
  };
1314
1938
  }
@@ -1323,11 +1947,42 @@ function isPresenceOnlyIntent(normalizedFeature) {
1323
1947
  return presenceWords && !actionWords;
1324
1948
  }
1325
1949
  function applyIntentPolicy(scenarios, elementMap, intentProfile, prefix) {
1950
+ if (intentProfile.mode === "count") {
1951
+ return buildCountScenarios(elementMap, intentProfile.feature, prefix);
1952
+ }
1326
1953
  if (intentProfile.mode !== "presence") {
1327
1954
  return scenarios.slice(0, intentProfile.maxScenarios);
1328
1955
  }
1329
1956
  return buildPresenceOnlyScenarios(elementMap, intentProfile.feature, prefix);
1330
1957
  }
1958
+ function buildCountScenarios(elementMap, feature, prefix) {
1959
+ const normalized = feature.toLowerCase();
1960
+ const countedElements = pickCountedElements(elementMap, normalized);
1961
+ const selector = countSelectorForFeature(normalized, countedElements);
1962
+ const count = countedElements.length;
1963
+ return [{
1964
+ id: `${prefix}-${String(1).padStart(3, "0")}`,
1965
+ title: feature,
1966
+ tags: ["@smoke", "@ui"],
1967
+ steps: [
1968
+ {
1969
+ action: "navigate",
1970
+ selector: "page",
1971
+ value: elementMap.url,
1972
+ human: `Navigate to ${elementMap.url}`
1973
+ }
1974
+ ],
1975
+ assertions: [{
1976
+ type: "count",
1977
+ selector,
1978
+ expected: String(count),
1979
+ human: `There are exactly ${count} matching elements`,
1980
+ playwright: `await expect(page.${selector}).toHaveCount(${count});`
1981
+ }],
1982
+ narrator: "We count only the element type requested by the user.",
1983
+ codeLevel: "beginner"
1984
+ }];
1985
+ }
1331
1986
  function buildPresenceOnlyScenarios(elementMap, feature, prefix) {
1332
1987
  const claims = extractPresenceClaims(feature, elementMap);
1333
1988
  if (claims.length === 0) return buildFallbackScenarios(elementMap, prefix).slice(0, 1);
@@ -1360,11 +2015,9 @@ function extractPresenceClaims(feature, elementMap) {
1360
2015
  for (const segment of segments) {
1361
2016
  const normalized = segment.toLowerCase();
1362
2017
  if (/\b(?:title|heading)\b/.test(normalized) && !/\b(?:browser title|tab title|page title)\b/.test(normalized)) {
1363
- const heading = chooseHeadingElement(elementMap);
1364
- const selector2 = heading?.selector ?? `getByRole('heading')`;
1365
2018
  claims.push({
1366
- selector: selector2,
1367
- human: `${heading?.name || "Main page heading"} is visible`
2019
+ selector: `getByRole('heading')`,
2020
+ human: `${chooseHeadingElement(elementMap)?.name || "Main page heading"} is visible`
1368
2021
  });
1369
2022
  continue;
1370
2023
  }
@@ -1432,7 +2085,7 @@ function detectAuthElements(elementMap) {
1432
2085
  return { usernameInput, passwordInput, submitButton, heading };
1433
2086
  }
1434
2087
  function shouldUseAuthFallback(authElements, scenarios, intentProfile) {
1435
- if (intentProfile.mode === "presence") return false;
2088
+ if (intentProfile.mode !== "auth" && intentProfile.mode !== "form") return false;
1436
2089
  if (!authElements.usernameInput || !authElements.passwordInput || !authElements.submitButton) return false;
1437
2090
  if (scenarios.length === 0) return true;
1438
2091
  const hasCredentialEntry = scenarios.some((scenario) => {
@@ -1456,8 +2109,26 @@ function defaultStepHuman(action, selector, value) {
1456
2109
  function normalizeStepHuman(human, action, selector, value) {
1457
2110
  if (action !== "fill" && action !== "select") return human;
1458
2111
  if (!value) return human;
1459
- if (/\bwith\s+['"`]/i.test(human) || /\$\{CT_VAR_[A-Z0-9_]+\}/.test(human)) return human;
1460
- return buildFillHuman(human, selector, value);
2112
+ const explicitTemplate = /\$\{CT_VAR_[A-Z0-9_]+\}/.test(human);
2113
+ const quotedValue = human.match(/\bwith\s+['"`]([^'"`]+)['"`]/i)?.[1];
2114
+ if (explicitTemplate) return human;
2115
+ if (quotedValue && !isGenericFillValue(quotedValue)) return human;
2116
+ const rewrittenBase = human.replace(/\s+with\s+['"`][^'"`]+['"`]\s*$/i, "").trim() || human;
2117
+ return buildFillHuman(rewrittenBase, selector, value);
2118
+ }
2119
+ function pickCountedElements(elementMap, normalizedFeature) {
2120
+ const category = /\bbutton/.test(normalizedFeature) ? "button" : /\blink/.test(normalizedFeature) ? "link" : /\bheading|title/.test(normalizedFeature) ? "heading" : /\binput|field|textbox|text box/.test(normalizedFeature) ? "input" : void 0;
2121
+ if (category) {
2122
+ return elementMap.elements.filter((element) => element.category === category);
2123
+ }
2124
+ return elementMap.elements.filter((element) => element.category === "button" || element.category === "link" || element.category === "input");
2125
+ }
2126
+ function countSelectorForFeature(normalizedFeature, elements) {
2127
+ if (/\bbutton/.test(normalizedFeature)) return `getByRole('button')`;
2128
+ if (/\blink/.test(normalizedFeature)) return `getByRole('link')`;
2129
+ if (/\bheading|title/.test(normalizedFeature)) return `getByRole('heading')`;
2130
+ if (/\binput|field|textbox|text box/.test(normalizedFeature)) return `getByRole('textbox')`;
2131
+ return elements[0]?.selector ?? `locator('*')`;
1461
2132
  }
1462
2133
  function normalizePlaywrightAssertion(assertion) {
1463
2134
  const repaired = scopeBareSelectorsInAssertion(unwrapWrappedSelectorAssertion(assertion.playwright));
@@ -2251,7 +2922,7 @@ function saveSpecPreview(analysis, outputDir = "tests/preview") {
2251
2922
  if (step.action === "hover") lines.push(` await ${selector}.hover();`);
2252
2923
  }
2253
2924
  for (const assertion of scenario.assertions) {
2254
- lines.push(` ${ensureStatement2(assertion.playwright)}`);
2925
+ lines.push(` ${renderPreviewAssertion(assertion.playwright, assertion.human)}`);
2255
2926
  }
2256
2927
  lines.push("});");
2257
2928
  lines.push("");
@@ -2275,6 +2946,15 @@ function ensureStatement2(value) {
2275
2946
  if (!trimmed) return "// TODO: add assertion";
2276
2947
  return trimmed.endsWith(";") ? trimmed : `${trimmed};`;
2277
2948
  }
2949
+ function renderPreviewAssertion(playwright, human) {
2950
+ const statement = ensureStatement2(playwright);
2951
+ const headingMatch = statement.match(/^await expect\(page\.getByRole\((['"])heading\1\)\)\.toBeVisible\(\);$/);
2952
+ const visibleSubject = String(human ?? "").match(/^(.+?) is visible$/)?.[1]?.trim();
2953
+ if (headingMatch && visibleSubject && !/main page heading/i.test(visibleSubject)) {
2954
+ return `await expect(page.getByRole("heading", { name: ${JSON.stringify(visibleSubject)} })).toBeVisible();`;
2955
+ }
2956
+ return statement;
2957
+ }
2278
2958
 
2279
2959
  // src/commands/tc.ts
2280
2960
  function slugify2(text) {