@cementic/cementic-test 0.2.11 → 0.2.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -0
- package/dist/cli.js +359 -66
- package/dist/cli.js.map +1 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -365,6 +365,13 @@ And start automating.
|
|
|
365
365
|
|
|
366
366
|
## Changelog
|
|
367
367
|
|
|
368
|
+
### v0.2.12
|
|
369
|
+
|
|
370
|
+
- made both AI generation paths intent-first so markdown and capture-based scenario generation follow the same scope and presence-only rules
|
|
371
|
+
- tightened capture-based scenario generation to preserve presence-only checks as visibility assertions, honor intent-based selector fallback, and avoid generic `'value'` placeholders
|
|
372
|
+
- added regression coverage for the new markdown prompt contract and capture-path intent enforcement
|
|
373
|
+
- breaking change: none
|
|
374
|
+
|
|
368
375
|
### v0.2.11
|
|
369
376
|
|
|
370
377
|
- AI no longer writes `'value'` as a step input and now prefers matching `CT_VAR_*` references or `test-{fieldname}` fallbacks
|
package/dist/cli.js
CHANGED
|
@@ -503,37 +503,114 @@ function getAvailableCtVarKeys() {
|
|
|
503
503
|
}
|
|
504
504
|
function buildSystemMessage() {
|
|
505
505
|
return `
|
|
506
|
-
You are a senior
|
|
506
|
+
You are a senior software test architect with 10 years of Playwright experience.
|
|
507
|
+
You translate user requirements into precise, minimal, executable test scenarios.
|
|
507
508
|
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
- Format them in strict Markdown that a downstream parser will process
|
|
509
|
+
RULE 1 - INTENT IS THE REQUIREMENT
|
|
510
|
+
Parse the user's feature description into specific testable claims BEFORE examining page context.
|
|
511
|
+
Generate scenarios for the user's intent, not for every element on the page.
|
|
512
512
|
|
|
513
|
-
|
|
513
|
+
RULE 2 - GENERATE TESTS FOR THE INTENT, NOT THE PAGE INVENTORY
|
|
514
|
+
Only generate tests for claims extracted from the intent.
|
|
515
|
+
Never generate tests for page elements that were not requested.
|
|
516
|
+
Page context improves selector quality and wording, but it does not define test scope.
|
|
517
|
+
|
|
518
|
+
RULE 3 - SELECTOR HIERARCHY
|
|
519
|
+
When choosing selectors, prefer:
|
|
520
|
+
1. Exact page-context match when available
|
|
521
|
+
2. getByRole() with text from the intent
|
|
522
|
+
3. getByLabel() for form inputs
|
|
523
|
+
4. getByText() with text from the intent
|
|
524
|
+
5. getByPlaceholder() for inputs
|
|
525
|
+
6. locator('#id') only when the id appears stable
|
|
526
|
+
|
|
527
|
+
Tie-breaker when multiple elements match:
|
|
528
|
+
1. Prefer the highest-confidence page-context match
|
|
529
|
+
2. Prefer visible elements over hidden
|
|
530
|
+
3. Prefer interactive elements over static elements
|
|
531
|
+
4. If still ambiguous, use the first match and note it with a short comment in the step text
|
|
532
|
+
|
|
533
|
+
RULE 4 - TITLE DISAMBIGUATION
|
|
534
|
+
"verify title" means the main visible heading on the page.
|
|
535
|
+
Use a heading assertion first.
|
|
536
|
+
Do NOT treat "title" as the browser tab title unless the intent explicitly says "browser title", "tab title", or "page title".
|
|
537
|
+
|
|
538
|
+
RULE 5 - PRESENCE-ONLY INTENTS
|
|
539
|
+
For presence-only intents such as "verify X", "check X is present", or "X is visible":
|
|
540
|
+
- Only assert visibility
|
|
541
|
+
- Do NOT click the element
|
|
542
|
+
- Do NOT assert redirects, state changes, or side effects
|
|
543
|
+
|
|
544
|
+
RULE 6 - NEGATIVE CASES
|
|
545
|
+
Generate negative scenarios only when the intent implies interaction:
|
|
546
|
+
- Form testing: always include an invalid-input scenario
|
|
547
|
+
- Authentication: always include a wrong-credentials scenario
|
|
548
|
+
- Presence checks: no negative case needed
|
|
549
|
+
- Button click: negative case only if the button can be disabled
|
|
550
|
+
|
|
551
|
+
RULE 7 - SETUP DEPENDENCIES
|
|
552
|
+
If authentication is required to reach the page:
|
|
553
|
+
- Add setup steps using CT_VAR_USERNAME and CT_VAR_PASSWORD
|
|
554
|
+
- Tag the scenario with @requires-auth
|
|
555
|
+
- Mention the dependency in a short step or assertion note when needed
|
|
556
|
+
|
|
557
|
+
RULE 8 - TEST DATA
|
|
558
|
+
Never use the literal word 'value' as a placeholder.
|
|
559
|
+
When filling a field:
|
|
560
|
+
- If the user context lists a matching CT_VAR_* variable, write it exactly as '\${CT_VAR_FIELDNAME}` + `'
|
|
561
|
+
- Otherwise use a field-specific fallback such as 'test-username', 'test-email@example.com', 'test-password', or 'test-search'
|
|
562
|
+
|
|
563
|
+
RULE 9 - SCOPE DISCIPLINE
|
|
564
|
+
Match scenario count to intent complexity:
|
|
565
|
+
- 1 to 2 claims: 1 to 2 scenarios maximum
|
|
566
|
+
- 3 to 5 claims: 3 to 5 scenarios maximum
|
|
567
|
+
- Full flow: 4 to 8 scenarios maximum
|
|
568
|
+
Never exceed 8 scenarios.
|
|
569
|
+
|
|
570
|
+
COMMON TESTING VOCABULARY
|
|
571
|
+
"verify/check X is present/visible" -> toBeVisible()
|
|
572
|
+
"verify title/heading" -> heading is visible
|
|
573
|
+
"button/link is present" -> button or link is visible
|
|
574
|
+
"can click X" -> click then assert visible outcome
|
|
575
|
+
"form submits" -> fill + click + assert success or URL
|
|
576
|
+
"error shows" -> trigger error + assert error visible
|
|
577
|
+
"redirects to X" -> URL contains or equals X
|
|
578
|
+
"text says X" -> text equals or contains X
|
|
579
|
+
"page loads" -> main heading or key element is visible
|
|
580
|
+
|
|
581
|
+
OUTPUT FORMAT - EXACT MARKDOWN SCHEMA
|
|
582
|
+
Output valid markdown in this exact format. No preamble. No explanation. Just the markdown.
|
|
583
|
+
|
|
584
|
+
# {PREFIX}-001 - {scenario-slug} - {Human readable title} @tag1 @tag2
|
|
585
|
+
|
|
586
|
+
## Steps
|
|
587
|
+
1. Navigate to {url}
|
|
588
|
+
2. {action verb} {target description}
|
|
589
|
+
|
|
590
|
+
## Expected Results
|
|
591
|
+
- {specific observable assertion}
|
|
592
|
+
- {specific observable assertion}
|
|
593
|
+
|
|
594
|
+
---
|
|
595
|
+
|
|
596
|
+
# {PREFIX}-002 - {scenario-slug} - {Human readable title} @tag1 @tag2
|
|
514
597
|
|
|
515
|
-
# <ID> \u2014 <Short title> @<tag1> @<tag2>
|
|
516
598
|
## Steps
|
|
517
|
-
1.
|
|
518
|
-
2.
|
|
599
|
+
1. Navigate to {url}
|
|
600
|
+
2. {action verb} {target description}
|
|
519
601
|
|
|
520
602
|
## Expected Results
|
|
521
|
-
-
|
|
522
|
-
- <verifiable outcome>
|
|
603
|
+
- {specific observable assertion}
|
|
523
604
|
|
|
524
|
-
Rules:
|
|
525
|
-
-
|
|
526
|
-
-
|
|
527
|
-
-
|
|
528
|
-
-
|
|
529
|
-
-
|
|
530
|
-
-
|
|
531
|
-
-
|
|
532
|
-
-
|
|
533
|
-
- Expected Results = verifiable UI outcomes ("Error message 'Invalid credentials' is visible", "Page redirects to /dashboard")
|
|
534
|
-
- Include both happy-path AND negative/edge-case tests
|
|
535
|
-
- Do NOT explain or add preamble \u2014 output ONLY the test cases
|
|
536
|
-
- No code blocks, no extra headings outside the pattern
|
|
605
|
+
Rules for the markdown schema:
|
|
606
|
+
- id format: PREFIX-NNN using the provided prefix and numbering
|
|
607
|
+
- slug format: kebab-case, max 5 words, derived from the scenario subject
|
|
608
|
+
- title: human readable and aligned to the user's intent
|
|
609
|
+
- tags: include at least one of @smoke @regression @negative @happy-path plus a relevant domain tag such as @auth @ui @navigation @form @checkout
|
|
610
|
+
- steps: numbered, start with Navigate, then actions, written in plain English
|
|
611
|
+
- assertions: bullet points with observable outcomes only
|
|
612
|
+
- never write "test passes" or "works correctly"
|
|
613
|
+
- do NOT wrap the output in code fences
|
|
537
614
|
`.trim();
|
|
538
615
|
}
|
|
539
616
|
function buildUserMessage(ctx) {
|
|
@@ -542,9 +619,12 @@ function buildUserMessage(ctx) {
|
|
|
542
619
|
lines.push(`App / product description:`);
|
|
543
620
|
lines.push(ctx.appDescription || "N/A");
|
|
544
621
|
lines.push("");
|
|
545
|
-
lines.push(`
|
|
622
|
+
lines.push(`Primary test intent:`);
|
|
546
623
|
lines.push(ctx.feature);
|
|
547
624
|
lines.push("");
|
|
625
|
+
lines.push(`Interpret the feature text above as the source of truth.`);
|
|
626
|
+
lines.push(`Only generate scenarios for claims that are explicitly implied by that intent.`);
|
|
627
|
+
lines.push("");
|
|
548
628
|
if (ctx.url) {
|
|
549
629
|
lines.push(`Page URL: ${ctx.url}`);
|
|
550
630
|
lines.push("");
|
|
@@ -588,9 +668,9 @@ function buildUserMessage(ctx) {
|
|
|
588
668
|
}
|
|
589
669
|
lines.push(`Test ID prefix: ${ctx.prefix}`);
|
|
590
670
|
lines.push(`Start numbering from: ${String(ctx.startIndex).padStart(3, "0")}`);
|
|
591
|
-
lines.push(`
|
|
671
|
+
lines.push(`Requested number of test cases: ${ctx.numCases}`);
|
|
592
672
|
lines.push("");
|
|
593
|
-
lines.push(`
|
|
673
|
+
lines.push(`Use the requested count only when it fits the intent complexity. Prefer fewer scenarios when the request is simple.`);
|
|
594
674
|
lines.push(`Output ONLY the test cases. No explanation, no preamble.`);
|
|
595
675
|
return lines.join("\n");
|
|
596
676
|
}
|
|
@@ -663,12 +743,13 @@ async function generateTcMarkdownWithAi(ctx) {
|
|
|
663
743
|
|
|
664
744
|
// src/core/analyse.ts
|
|
665
745
|
async function analyseElements(elementMap, options = {}) {
|
|
666
|
-
const { verbose = false } = options;
|
|
746
|
+
const { verbose = false, feature } = options;
|
|
667
747
|
const providerConfig = resolveLlmProvider();
|
|
748
|
+
const requestedFeature = normalizeRequestedFeature(feature, elementMap);
|
|
668
749
|
log(verbose, `
|
|
669
750
|
[analyse] Sending capture to ${providerConfig.displayName} (${providerConfig.model})`);
|
|
670
751
|
const systemPrompt = buildSystemPrompt();
|
|
671
|
-
const userPrompt = buildUserPrompt(elementMap);
|
|
752
|
+
const userPrompt = buildUserPrompt(elementMap, requestedFeature);
|
|
672
753
|
const rawResponse = providerConfig.transport === "anthropic" ? await callAnthropic2(providerConfig.apiKey, providerConfig.model, systemPrompt, userPrompt) : await callOpenAiCompatible2(
|
|
673
754
|
providerConfig.apiKey,
|
|
674
755
|
providerConfig.model,
|
|
@@ -678,7 +759,7 @@ async function analyseElements(elementMap, options = {}) {
|
|
|
678
759
|
userPrompt
|
|
679
760
|
);
|
|
680
761
|
const parsed = parseAnalysisJson(rawResponse);
|
|
681
|
-
return sanitizeAnalysis(parsed, elementMap);
|
|
762
|
+
return sanitizeAnalysis(parsed, elementMap, requestedFeature);
|
|
682
763
|
}
|
|
683
764
|
function buildSystemPrompt() {
|
|
684
765
|
return `
|
|
@@ -688,16 +769,20 @@ Your job is to analyse a structured map of interactive elements extracted from a
|
|
|
688
769
|
then produce a complete set of Playwright test scenarios.
|
|
689
770
|
|
|
690
771
|
RULES:
|
|
691
|
-
1.
|
|
772
|
+
1. The user's feature description is the PRIMARY requirement. Generate scenarios that test what the user asked for.
|
|
692
773
|
2. Every assertion must include an exact "playwright" field with a complete await expect(...) statement.
|
|
693
|
-
3. Use
|
|
694
|
-
4. Include
|
|
774
|
+
3. Use captured elements as the first selector source. If the intent names text that was not captured, you may fall back to a safe Playwright selector derived from the intent such as getByRole(), getByText(), getByLabel(), getByPlaceholder(), or locator('#id').
|
|
775
|
+
4. Include happy-path and negative scenarios only when the intent implies interaction, and stay evidence-backed.
|
|
695
776
|
5. Output only valid JSON matching the requested schema.
|
|
696
777
|
6. Do not invent redirect targets, success pages, error text, password clearing, or security scenarios unless the capture explicitly supports them.
|
|
697
778
|
7. If no status or alert region was captured, avoid scenarios that depend on unseen server-side validation messages.
|
|
698
|
-
8.
|
|
699
|
-
9.
|
|
700
|
-
10. If
|
|
779
|
+
8. Scope must match intent complexity. Simple 1 to 2 claim requests should produce only 1 to 2 scenarios.
|
|
780
|
+
9. For presence-only intents such as "verify X" or "X is present", only assert visibility. Do not click and do not assert outcomes after clicking.
|
|
781
|
+
10. For fill steps, never use the literal word "value". If matching CT_VAR_* variables are listed in the prompt, use them in human-readable step text and use a realistic non-generic runtime value in the JSON value field. Otherwise use a field-specific fallback such as test-username or test-password.
|
|
782
|
+
11. The "selector" field must be a raw selector expression such as locator("#username") or getByRole('button', { name: "Login" }). Never wrap selectors in page. or page.locator("...") inside JSON.
|
|
783
|
+
12. Tie-breaker for multiple matching elements: prefer highest confidence, then interactive elements, then the first visible-looking candidate.
|
|
784
|
+
13. If a page clearly contains login or auth fields, do not create submit scenarios that only click the button. Include realistic fill steps for captured username/email and password inputs when the intent asks for auth interaction.
|
|
785
|
+
14. "verify title" means the main visible heading on the page, not the browser tab title, unless the intent explicitly says browser title, tab title, or page title.
|
|
701
786
|
|
|
702
787
|
OUTPUT SCHEMA:
|
|
703
788
|
{
|
|
@@ -734,9 +819,16 @@ OUTPUT SCHEMA:
|
|
|
734
819
|
"audioSummary": string
|
|
735
820
|
}`.trim();
|
|
736
821
|
}
|
|
737
|
-
function buildUserPrompt(elementMap) {
|
|
822
|
+
function buildUserPrompt(elementMap, feature) {
|
|
738
823
|
const lines = [];
|
|
739
824
|
const authElements = detectAuthElements(elementMap);
|
|
825
|
+
const availableCtVarKeys = getAvailableCtVarKeys2();
|
|
826
|
+
const intentProfile = buildIntentProfile(feature);
|
|
827
|
+
lines.push("USER INTENT");
|
|
828
|
+
lines.push(`Feature description: ${feature}`);
|
|
829
|
+
lines.push(`Intent mode: ${intentProfile.mode}`);
|
|
830
|
+
lines.push(`Maximum scenarios for this request: ${intentProfile.maxScenarios}`);
|
|
831
|
+
lines.push("");
|
|
740
832
|
lines.push("PAGE INFORMATION");
|
|
741
833
|
lines.push(`URL: ${elementMap.url}`);
|
|
742
834
|
lines.push(`Title: ${elementMap.title}`);
|
|
@@ -762,6 +854,13 @@ function buildUserPrompt(elementMap) {
|
|
|
762
854
|
}
|
|
763
855
|
lines.push("");
|
|
764
856
|
}
|
|
857
|
+
if (availableCtVarKeys.length > 0) {
|
|
858
|
+
lines.push("AVAILABLE CT_VAR_* VARIABLES:");
|
|
859
|
+
for (const envKey of availableCtVarKeys) {
|
|
860
|
+
lines.push(` - ${envKey}`);
|
|
861
|
+
}
|
|
862
|
+
lines.push("");
|
|
863
|
+
}
|
|
765
864
|
const interactiveCount = elementMap.elements.filter((element) => element.category === "input" || element.category === "button" || element.category === "link").length;
|
|
766
865
|
const statusCount = elementMap.elements.filter((element) => element.category === "status").length;
|
|
767
866
|
lines.push("EVIDENCE CONSTRAINTS:");
|
|
@@ -769,6 +868,7 @@ function buildUserPrompt(elementMap) {
|
|
|
769
868
|
lines.push(` - Status or alert regions captured: ${statusCount}`);
|
|
770
869
|
lines.push(" - If no redirect target is explicitly captured, do not assert a destination path.");
|
|
771
870
|
lines.push(" - If no status region was captured, avoid exact server-side credential error claims.");
|
|
871
|
+
lines.push(" - Generate scenarios only for the requested feature description.");
|
|
772
872
|
if (authElements.usernameInput && authElements.passwordInput && authElements.submitButton) {
|
|
773
873
|
lines.push(" - This page contains a captured auth form. Include fill steps for the username/email and password fields in credential-submission scenarios.");
|
|
774
874
|
lines.push(" - For auth pages without captured post-submit evidence, prefer evidence-backed form-state assertions over invented success redirects.");
|
|
@@ -837,15 +937,21 @@ ${raw.slice(0, 500)}`
|
|
|
837
937
|
);
|
|
838
938
|
}
|
|
839
939
|
}
|
|
840
|
-
function sanitizeAnalysis(analysis, elementMap) {
|
|
940
|
+
function sanitizeAnalysis(analysis, elementMap, requestedFeature) {
|
|
841
941
|
const selectors = /* @__PURE__ */ new Set();
|
|
842
942
|
for (const element of elementMap.elements) {
|
|
843
943
|
selectors.add(element.selector);
|
|
844
944
|
for (const alt of element.selectorAlt) selectors.add(alt);
|
|
845
945
|
}
|
|
846
946
|
const rawScenarios = Array.isArray(analysis.scenarios) ? analysis.scenarios : [];
|
|
947
|
+
const intentProfile = buildIntentProfile(requestedFeature);
|
|
847
948
|
const currentUrl = new URL(elementMap.url);
|
|
848
949
|
const authElements = detectAuthElements(elementMap);
|
|
950
|
+
const resolvedFeature = requestedFeature || analysis.feature || inferFeatureName(elementMap);
|
|
951
|
+
const resolvedPrefix = (analysis.suggestedPrefix || inferPrefix({
|
|
952
|
+
featureText: resolvedFeature,
|
|
953
|
+
url: elementMap.url
|
|
954
|
+
})).toUpperCase();
|
|
849
955
|
const knownPaths = /* @__PURE__ */ new Set([
|
|
850
956
|
currentUrl.pathname,
|
|
851
957
|
...elementMap.elements.filter((element) => element.category === "link").map((element) => {
|
|
@@ -868,21 +974,28 @@ function sanitizeAnalysis(analysis, elementMap) {
|
|
|
868
974
|
if (!pathMatch) return true;
|
|
869
975
|
return knownPaths.has(pathMatch[0]);
|
|
870
976
|
})
|
|
871
|
-
})).filter((scenario) => scenario.assertions.length > 0)
|
|
872
|
-
const
|
|
873
|
-
|
|
977
|
+
})).filter((scenario) => scenario.assertions.length > 0);
|
|
978
|
+
const intentAlignedScenarios = applyIntentPolicy(
|
|
979
|
+
sanitizedScenarios,
|
|
980
|
+
elementMap,
|
|
981
|
+
intentProfile,
|
|
982
|
+
resolvedPrefix
|
|
983
|
+
).slice(0, intentProfile.maxScenarios);
|
|
984
|
+
const useAuthFallback = shouldUseAuthFallback(authElements, intentAlignedScenarios, intentProfile);
|
|
985
|
+
const fallbackScenarios = intentProfile.mode === "presence" ? buildPresenceOnlyScenarios(elementMap, requestedFeature, resolvedPrefix) : useAuthFallback ? buildAuthFallbackScenarios(elementMap, resolvedPrefix, authElements) : buildFallbackScenarios(elementMap, resolvedPrefix);
|
|
986
|
+
const finalScenarios = useAuthFallback ? fallbackScenarios.slice(0, intentProfile.maxScenarios) : intentAlignedScenarios.length > 0 ? intentAlignedScenarios : fallbackScenarios.slice(0, intentProfile.maxScenarios);
|
|
874
987
|
return {
|
|
875
988
|
...analysis,
|
|
876
989
|
url: analysis.url || elementMap.url,
|
|
877
|
-
feature:
|
|
878
|
-
suggestedPrefix:
|
|
990
|
+
feature: resolvedFeature,
|
|
991
|
+
suggestedPrefix: resolvedPrefix,
|
|
879
992
|
scenarios: finalScenarios,
|
|
880
993
|
analysisNotes: [
|
|
881
994
|
analysis.analysisNotes,
|
|
882
995
|
useAuthFallback ? "Replaced low-evidence auth scenarios with deterministic evidence-backed auth coverage." : "",
|
|
883
996
|
`Sanitized to ${finalScenarios.length} evidence-backed scenario(s) from ${rawScenarios.length} raw scenario(s).`
|
|
884
997
|
].filter(Boolean).join(" "),
|
|
885
|
-
audioSummary: analysis.audioSummary || buildAudioSummary(
|
|
998
|
+
audioSummary: analysis.audioSummary || buildAudioSummary(resolvedFeature, finalScenarios)
|
|
886
999
|
};
|
|
887
1000
|
}
|
|
888
1001
|
function normalizeScenario(candidate, selectors, url) {
|
|
@@ -902,20 +1015,28 @@ function normalizeScenario(candidate, selectors, url) {
|
|
|
902
1015
|
function normalizeStep(candidate, selectors) {
|
|
903
1016
|
if (!candidate) return null;
|
|
904
1017
|
const selector = String(candidate.selector ?? "").trim();
|
|
905
|
-
if (selector
|
|
1018
|
+
if (!isAllowedSelector(selector, selectors)) return null;
|
|
906
1019
|
const action = String(candidate.action ?? "").trim();
|
|
907
1020
|
if (!["navigate", "fill", "click", "select", "check", "keyboard", "hover"].includes(action)) return null;
|
|
1021
|
+
const rawValue = String(candidate.value ?? "");
|
|
1022
|
+
const value = action === "fill" || action === "select" ? resolveFieldRuntimeValue(selector, rawValue) : rawValue;
|
|
1023
|
+
const human = normalizeStepHuman(
|
|
1024
|
+
String(candidate.human ?? "").trim() || defaultStepHuman(action, selector, value),
|
|
1025
|
+
action,
|
|
1026
|
+
selector,
|
|
1027
|
+
value
|
|
1028
|
+
);
|
|
908
1029
|
return {
|
|
909
1030
|
action,
|
|
910
1031
|
selector,
|
|
911
|
-
value
|
|
912
|
-
human
|
|
1032
|
+
value,
|
|
1033
|
+
human
|
|
913
1034
|
};
|
|
914
1035
|
}
|
|
915
1036
|
function normalizeAssertion(candidate, selectors) {
|
|
916
1037
|
if (!candidate) return null;
|
|
917
1038
|
const selector = String(candidate.selector ?? "").trim();
|
|
918
|
-
if (selector
|
|
1039
|
+
if (!isAllowedSelector(selector, selectors)) return null;
|
|
919
1040
|
const assertion = {
|
|
920
1041
|
type: String(candidate.type ?? "visible").trim() || "visible",
|
|
921
1042
|
selector,
|
|
@@ -961,6 +1082,7 @@ function buildFallbackScenarios(elementMap, prefix) {
|
|
|
961
1082
|
});
|
|
962
1083
|
}
|
|
963
1084
|
if (emailInput && passwordInput && submitButton) {
|
|
1085
|
+
const emailValue = resolveFieldRuntimeValue(emailInput.selector, "");
|
|
964
1086
|
scenarios.push({
|
|
965
1087
|
id: nextId(scenarios.length + 1),
|
|
966
1088
|
title: "Submitting without a password keeps the user on the form",
|
|
@@ -975,8 +1097,8 @@ function buildFallbackScenarios(elementMap, prefix) {
|
|
|
975
1097
|
{
|
|
976
1098
|
action: "fill",
|
|
977
1099
|
selector: emailInput.selector,
|
|
978
|
-
value:
|
|
979
|
-
human: "Fill in the email field"
|
|
1100
|
+
value: emailValue,
|
|
1101
|
+
human: buildFillHuman("Fill in the email field", emailInput.selector, emailValue)
|
|
980
1102
|
},
|
|
981
1103
|
{
|
|
982
1104
|
action: "click",
|
|
@@ -1077,13 +1199,13 @@ function buildAuthFallbackScenarios(elementMap, prefix, authElements) {
|
|
|
1077
1199
|
action: "fill",
|
|
1078
1200
|
selector: usernameInput.selector,
|
|
1079
1201
|
value: usernameValue,
|
|
1080
|
-
human: "Fill in the username field"
|
|
1202
|
+
human: buildFillHuman("Fill in the username field", usernameInput.selector, usernameValue)
|
|
1081
1203
|
},
|
|
1082
1204
|
{
|
|
1083
1205
|
action: "fill",
|
|
1084
1206
|
selector: passwordInput.selector,
|
|
1085
1207
|
value: passwordValue,
|
|
1086
|
-
human: "Fill in the password field"
|
|
1208
|
+
human: buildFillHuman("Fill in the password field", passwordInput.selector, passwordValue)
|
|
1087
1209
|
}
|
|
1088
1210
|
],
|
|
1089
1211
|
assertions: [
|
|
@@ -1127,7 +1249,7 @@ function buildAuthFallbackScenarios(elementMap, prefix, authElements) {
|
|
|
1127
1249
|
action: "fill",
|
|
1128
1250
|
selector: usernameInput.selector,
|
|
1129
1251
|
value: usernameValue,
|
|
1130
|
-
human: "Fill in the username field"
|
|
1252
|
+
human: buildFillHuman("Fill in the username field", usernameInput.selector, usernameValue)
|
|
1131
1253
|
},
|
|
1132
1254
|
{
|
|
1133
1255
|
action: "click",
|
|
@@ -1164,16 +1286,136 @@ function buildAuthFallbackScenarios(elementMap, prefix, authElements) {
|
|
|
1164
1286
|
});
|
|
1165
1287
|
return scenarios;
|
|
1166
1288
|
}
|
|
1289
|
+
function normalizeRequestedFeature(feature, elementMap) {
|
|
1290
|
+
return String(feature ?? "").trim() || inferFeatureName(elementMap);
|
|
1291
|
+
}
|
|
1167
1292
|
function inferFeatureName(elementMap) {
|
|
1168
1293
|
const heading = elementMap.elements.find((element) => element.category === "heading" && element.name);
|
|
1169
1294
|
return heading?.name || elementMap.title || "Captured page";
|
|
1170
1295
|
}
|
|
1171
|
-
function
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
|
|
1296
|
+
function getAvailableCtVarKeys2() {
|
|
1297
|
+
return Object.keys(process.env).filter((key) => /^CT_VAR_[A-Z0-9_]+$/.test(key)).sort();
|
|
1298
|
+
}
|
|
1299
|
+
function buildIntentProfile(feature) {
|
|
1300
|
+
const normalized = feature.toLowerCase();
|
|
1301
|
+
const wantsHeading = /\b(?:title|heading)\b/.test(normalized) && !/\b(?:browser title|tab title|page title)\b/.test(normalized);
|
|
1302
|
+
const presenceOnly = isPresenceOnlyIntent(normalized);
|
|
1303
|
+
const authIntent = /\b(?:login|log in|sign in|auth|credentials?)\b/.test(normalized) && !presenceOnly;
|
|
1304
|
+
const formIntent = /\b(?:form|submit|field|input|enter|fill|type)\b/.test(normalized) && !presenceOnly && !authIntent;
|
|
1305
|
+
const flowIntent = /\b(?:flow|journey|checkout|purchase|complete|works?)\b/.test(normalized) && !presenceOnly && !authIntent && !formIntent;
|
|
1306
|
+
const mode = presenceOnly ? "presence" : authIntent ? "auth" : formIntent ? "form" : flowIntent ? "flow" : "generic";
|
|
1307
|
+
const claimCount = countIntentClaims(feature);
|
|
1308
|
+
return {
|
|
1309
|
+
feature,
|
|
1310
|
+
mode,
|
|
1311
|
+
maxScenarios: mode === "presence" ? Math.min(Math.max(claimCount, 1), 2) : mode === "auth" || mode === "form" ? Math.min(Math.max(claimCount, 3), 5) : mode === "flow" ? Math.min(Math.max(claimCount, 4), 6) : Math.min(Math.max(claimCount, 1), 3),
|
|
1312
|
+
wantsHeading
|
|
1313
|
+
};
|
|
1314
|
+
}
|
|
1315
|
+
function countIntentClaims(feature) {
|
|
1316
|
+
const quoted = Array.from(feature.matchAll(/["“”']([^"“”']+)["“”']/g)).length;
|
|
1317
|
+
const segments = feature.split(/\s+(?:and|&)\s+|,\s*/i).map((segment) => segment.trim()).filter(Boolean);
|
|
1318
|
+
return Math.max(quoted, segments.length || 1);
|
|
1319
|
+
}
|
|
1320
|
+
function isPresenceOnlyIntent(normalizedFeature) {
|
|
1321
|
+
const presenceWords = /\b(?:verify|check|confirm|ensure|present|visible|shown|showing|available|exists?|title|heading)\b/.test(normalizedFeature);
|
|
1322
|
+
const actionWords = /\b(?:click|submit|fill|type|enter|login|log in|sign in|checkout|purchase|complete|flow|journey|error|invalid|disabled|enabled)\b/.test(normalizedFeature);
|
|
1323
|
+
return presenceWords && !actionWords;
|
|
1324
|
+
}
|
|
1325
|
+
function applyIntentPolicy(scenarios, elementMap, intentProfile, prefix) {
|
|
1326
|
+
if (intentProfile.mode !== "presence") {
|
|
1327
|
+
return scenarios.slice(0, intentProfile.maxScenarios);
|
|
1328
|
+
}
|
|
1329
|
+
return buildPresenceOnlyScenarios(elementMap, intentProfile.feature, prefix);
|
|
1330
|
+
}
|
|
1331
|
+
function buildPresenceOnlyScenarios(elementMap, feature, prefix) {
|
|
1332
|
+
const claims = extractPresenceClaims(feature, elementMap);
|
|
1333
|
+
if (claims.length === 0) return buildFallbackScenarios(elementMap, prefix).slice(0, 1);
|
|
1334
|
+
return [{
|
|
1335
|
+
id: `${prefix}-${String(1).padStart(3, "0")}`,
|
|
1336
|
+
title: feature,
|
|
1337
|
+
tags: ["@smoke", "@ui"],
|
|
1338
|
+
steps: [
|
|
1339
|
+
{
|
|
1340
|
+
action: "navigate",
|
|
1341
|
+
selector: "page",
|
|
1342
|
+
value: elementMap.url,
|
|
1343
|
+
human: `Navigate to ${elementMap.url}`
|
|
1344
|
+
}
|
|
1345
|
+
],
|
|
1346
|
+
assertions: claims.map((claim) => ({
|
|
1347
|
+
type: "visible",
|
|
1348
|
+
selector: claim.selector,
|
|
1349
|
+
expected: "visible",
|
|
1350
|
+
human: claim.human,
|
|
1351
|
+
playwright: visibleAssertion(claim.selector)
|
|
1352
|
+
})),
|
|
1353
|
+
narrator: "We verify only the elements explicitly requested by the user.",
|
|
1354
|
+
codeLevel: "beginner"
|
|
1355
|
+
}];
|
|
1356
|
+
}
|
|
1357
|
+
function extractPresenceClaims(feature, elementMap) {
|
|
1358
|
+
const segments = feature.split(/\s+(?:and|&)\s+|,\s*/i).map((segment) => segment.trim()).filter(Boolean);
|
|
1359
|
+
const claims = [];
|
|
1360
|
+
for (const segment of segments) {
|
|
1361
|
+
const normalized = segment.toLowerCase();
|
|
1362
|
+
if (/\b(?:title|heading)\b/.test(normalized) && !/\b(?:browser title|tab title|page title)\b/.test(normalized)) {
|
|
1363
|
+
const heading = chooseHeadingElement(elementMap);
|
|
1364
|
+
const selector2 = heading?.selector ?? `getByRole('heading')`;
|
|
1365
|
+
claims.push({
|
|
1366
|
+
selector: selector2,
|
|
1367
|
+
human: `${heading?.name || "Main page heading"} is visible`
|
|
1368
|
+
});
|
|
1369
|
+
continue;
|
|
1370
|
+
}
|
|
1371
|
+
const text = extractIntentLabel(segment);
|
|
1372
|
+
if (!text) continue;
|
|
1373
|
+
const match = findBestMatchingElement(elementMap, text, normalized);
|
|
1374
|
+
const selector = match?.selector ?? buildFallbackSelector(text, normalized);
|
|
1375
|
+
claims.push({
|
|
1376
|
+
selector,
|
|
1377
|
+
human: `${text} is visible`
|
|
1378
|
+
});
|
|
1379
|
+
}
|
|
1380
|
+
return claims;
|
|
1381
|
+
}
|
|
1382
|
+
function chooseHeadingElement(elementMap) {
|
|
1383
|
+
return rankElements(
|
|
1384
|
+
elementMap.elements.filter((element) => element.category === "heading")
|
|
1385
|
+
)[0];
|
|
1386
|
+
}
|
|
1387
|
+
function extractIntentLabel(segment) {
|
|
1388
|
+
const unquoted = Array.from(segment.matchAll(/["“”']([^"“”']+)["“”']/g)).map((match) => match[1].trim());
|
|
1389
|
+
if (unquoted.length > 0) return unquoted[0];
|
|
1390
|
+
return segment.replace(/\b(?:verify|check|confirm|ensure)\b/gi, " ").replace(/\b(?:the|a|an)\b/gi, " ").replace(/\b(?:is|are|be|should be)\b/gi, " ").replace(/\b(?:present|visible|shown|showing|available|exists?)\b/gi, " ").replace(/\b(?:button|link|cta|label|text)\b/gi, " ").replace(/[.?!,:;]+/g, " ").replace(/\s+/g, " ").trim();
|
|
1391
|
+
}
|
|
1392
|
+
function findBestMatchingElement(elementMap, text, segment) {
|
|
1393
|
+
const preferredCategory = /\bbutton\b/.test(segment) ? "button" : /\blink\b/.test(segment) ? "link" : /\bheading\b/.test(segment) ? "heading" : void 0;
|
|
1394
|
+
const query = text.toLowerCase();
|
|
1395
|
+
const candidates = elementMap.elements.filter((element) => {
|
|
1396
|
+
if (preferredCategory && element.category !== preferredCategory) return false;
|
|
1397
|
+
const haystack = `${element.name ?? ""} ${element.purpose}`.toLowerCase();
|
|
1398
|
+
return haystack.includes(query);
|
|
1399
|
+
});
|
|
1400
|
+
return rankElements(candidates)[0];
|
|
1401
|
+
}
|
|
1402
|
+
function rankElements(elements) {
|
|
1403
|
+
const confidenceWeight = (confidence) => confidence === "high" ? 3 : confidence === "medium" ? 2 : 1;
|
|
1404
|
+
const interactiveWeight = (category) => category === "button" || category === "link" || category === "input" ? 2 : 1;
|
|
1405
|
+
return [...elements].sort((left, right) => {
|
|
1406
|
+
const confidenceDelta = confidenceWeight(right.confidence) - confidenceWeight(left.confidence);
|
|
1407
|
+
if (confidenceDelta !== 0) return confidenceDelta;
|
|
1408
|
+
const interactiveDelta = interactiveWeight(right.category) - interactiveWeight(left.category);
|
|
1409
|
+
if (interactiveDelta !== 0) return interactiveDelta;
|
|
1410
|
+
return 0;
|
|
1411
|
+
});
|
|
1412
|
+
}
|
|
1413
|
+
function buildFallbackSelector(text, segment) {
|
|
1414
|
+
const safeRegex = text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
1415
|
+
if (/\bbutton\b/.test(segment)) return `getByRole('button', { name: /${safeRegex}/i })`;
|
|
1416
|
+
if (/\blink\b/.test(segment)) return `getByRole('link', { name: /${safeRegex}/i })`;
|
|
1417
|
+
if (/\b(?:title|heading)\b/.test(segment)) return `getByRole('heading', { name: /${safeRegex}/i })`;
|
|
1418
|
+
return `getByText(${JSON.stringify(text)})`;
|
|
1177
1419
|
}
|
|
1178
1420
|
function buildAudioSummary(feature, scenarios) {
|
|
1179
1421
|
return `We finished validating ${feature || "the captured page"} with ${scenarios.length} evidence-backed scenario${scenarios.length === 1 ? "" : "s"}.`;
|
|
@@ -1189,7 +1431,8 @@ function detectAuthElements(elementMap) {
|
|
|
1189
1431
|
const heading = elementMap.elements.find((element) => element.category === "heading" && /login|sign in|auth/.test((element.name ?? "").toLowerCase())) ?? elementMap.elements.find((element) => element.category === "heading");
|
|
1190
1432
|
return { usernameInput, passwordInput, submitButton, heading };
|
|
1191
1433
|
}
|
|
1192
|
-
function shouldUseAuthFallback(authElements, scenarios) {
|
|
1434
|
+
function shouldUseAuthFallback(authElements, scenarios, intentProfile) {
|
|
1435
|
+
if (intentProfile.mode === "presence") return false;
|
|
1193
1436
|
if (!authElements.usernameInput || !authElements.passwordInput || !authElements.submitButton) return false;
|
|
1194
1437
|
if (scenarios.length === 0) return true;
|
|
1195
1438
|
const hasCredentialEntry = scenarios.some((scenario) => {
|
|
@@ -1203,13 +1446,19 @@ function shouldUseAuthFallback(authElements, scenarios) {
|
|
|
1203
1446
|
}
|
|
1204
1447
|
function defaultStepHuman(action, selector, value) {
|
|
1205
1448
|
if (action === "navigate") return `Navigate to ${value || "the page"}`;
|
|
1206
|
-
if (action === "fill") return `Fill the field ${selector}
|
|
1449
|
+
if (action === "fill") return buildFillHuman(`Fill the field ${selector}`, selector, value);
|
|
1207
1450
|
if (action === "click") return `Click ${selector}`;
|
|
1208
1451
|
if (action === "select") return `Select ${value} in ${selector}`;
|
|
1209
1452
|
if (action === "check") return `Check ${selector}`;
|
|
1210
1453
|
if (action === "keyboard") return `Press ${value}`;
|
|
1211
1454
|
return `Interact with ${selector}`;
|
|
1212
1455
|
}
|
|
1456
|
+
function normalizeStepHuman(human, action, selector, value) {
|
|
1457
|
+
if (action !== "fill" && action !== "select") return human;
|
|
1458
|
+
if (!value) return human;
|
|
1459
|
+
if (/\bwith\s+['"`]/i.test(human) || /\$\{CT_VAR_[A-Z0-9_]+\}/.test(human)) return human;
|
|
1460
|
+
return buildFillHuman(human, selector, value);
|
|
1461
|
+
}
|
|
1213
1462
|
function normalizePlaywrightAssertion(assertion) {
|
|
1214
1463
|
const repaired = scopeBareSelectorsInAssertion(unwrapWrappedSelectorAssertion(assertion.playwright));
|
|
1215
1464
|
if (repaired) return ensureStatement(repaired);
|
|
@@ -1264,10 +1513,54 @@ function valueAssertion(selector, value) {
|
|
|
1264
1513
|
return `await expect(${selectorTarget(selector)}).toHaveValue(${JSON.stringify(value)});`;
|
|
1265
1514
|
}
|
|
1266
1515
|
function inferAuthValue(element, kind) {
|
|
1267
|
-
if (kind === "password") return "
|
|
1268
|
-
|
|
1269
|
-
|
|
1270
|
-
|
|
1516
|
+
if (kind === "password") return resolveFieldRuntimeValue(element.selector, "test-password");
|
|
1517
|
+
return resolveFieldRuntimeValue(element.selector, "");
|
|
1518
|
+
}
|
|
1519
|
+
function buildFillHuman(base, selector, runtimeValue) {
|
|
1520
|
+
const displayValue = resolveFieldDisplayValue(selector, runtimeValue);
|
|
1521
|
+
return `${base} with '${displayValue}'`;
|
|
1522
|
+
}
|
|
1523
|
+
function resolveFieldRuntimeValue(selector, currentValue) {
|
|
1524
|
+
if (currentValue && !isGenericFillValue(currentValue) && !extractCtVarTemplate(currentValue)) {
|
|
1525
|
+
return currentValue;
|
|
1526
|
+
}
|
|
1527
|
+
const envKey = matchCtVarKey(selector);
|
|
1528
|
+
if (envKey && process.env[envKey]) {
|
|
1529
|
+
return String(process.env[envKey]);
|
|
1530
|
+
}
|
|
1531
|
+
return fallbackFieldValue(selector);
|
|
1532
|
+
}
|
|
1533
|
+
function resolveFieldDisplayValue(selector, runtimeValue) {
|
|
1534
|
+
const envKey = matchCtVarKey(selector);
|
|
1535
|
+
if (envKey) return `\${${envKey}}`;
|
|
1536
|
+
return runtimeValue;
|
|
1537
|
+
}
|
|
1538
|
+
function matchCtVarKey(selectorOrText) {
|
|
1539
|
+
const haystack = selectorOrText.toLowerCase();
|
|
1540
|
+
const preferred = /\bemail\b/.test(haystack) ? "CT_VAR_EMAIL" : /\bpassword\b/.test(haystack) ? "CT_VAR_PASSWORD" : /\bsearch\b/.test(haystack) ? "CT_VAR_SEARCH" : /\bphone|tel\b/.test(haystack) ? "CT_VAR_PHONE" : /\bname\b/.test(haystack) ? "CT_VAR_NAME" : /\buser(?:name)?|login\b/.test(haystack) ? "CT_VAR_USERNAME" : void 0;
|
|
1541
|
+
if (preferred && process.env[preferred] !== void 0) return preferred;
|
|
1542
|
+
return getAvailableCtVarKeys2().find((key) => new RegExp(key.replace(/^CT_VAR_/, "").toLowerCase()).test(haystack));
|
|
1543
|
+
}
|
|
1544
|
+
function fallbackFieldValue(selectorOrText) {
|
|
1545
|
+
const haystack = selectorOrText.toLowerCase();
|
|
1546
|
+
if (/\bemail\b/.test(haystack)) return "test-email@example.com";
|
|
1547
|
+
if (/\bpassword\b/.test(haystack)) return "test-password";
|
|
1548
|
+
if (/\bsearch\b/.test(haystack)) return "test-search";
|
|
1549
|
+
if (/\bphone|tel\b/.test(haystack)) return "test-phone";
|
|
1550
|
+
if (/\bname\b/.test(haystack) && !/\buser(?:name)?\b/.test(haystack)) return "test-name";
|
|
1551
|
+
if (/\buser(?:name)?|login\b/.test(haystack)) return "test-username";
|
|
1552
|
+
return "test-input";
|
|
1553
|
+
}
|
|
1554
|
+
function isGenericFillValue(value) {
|
|
1555
|
+
return /^(?:value|text|input|option|selection|selected value|default)$/i.test(value.trim());
|
|
1556
|
+
}
|
|
1557
|
+
function extractCtVarTemplate(value) {
|
|
1558
|
+
return value.match(/^\$\{(CT_VAR_[A-Z0-9_]+)\}$/)?.[1];
|
|
1559
|
+
}
|
|
1560
|
+
function isAllowedSelector(selector, selectors) {
|
|
1561
|
+
if (selector === "page") return true;
|
|
1562
|
+
if (selectors.has(selector)) return true;
|
|
1563
|
+
return /^(?:getByRole|getByText|getByLabel|getByPlaceholder|getByTestId)\(/.test(selector) || /^locator\((['"])(#.+?)\1\)$/.test(selector);
|
|
1271
1564
|
}
|
|
1272
1565
|
function ensureStatement(value) {
|
|
1273
1566
|
const trimmed = String(value ?? "").trim();
|
|
@@ -2074,7 +2367,7 @@ async function runTcInteractive(params) {
|
|
|
2074
2367
|
console.log("Capture-only mode requested. No test cases were generated.");
|
|
2075
2368
|
return;
|
|
2076
2369
|
}
|
|
2077
|
-
const analysis = await analyseElements(elementMap, { verbose: true });
|
|
2370
|
+
const analysis = await analyseElements(elementMap, { verbose: true, feature });
|
|
2078
2371
|
printCaptureReport(elementMap, analysis);
|
|
2079
2372
|
const previewPath = saveSpecPreview(analysis);
|
|
2080
2373
|
if (previewPath) {
|