ccqa 0.1.6 → 0.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +12 -7
- package/dist/bin/ccqa.d.mts +1 -0
- package/dist/bin/ccqa.mjs +1702 -0
- package/dist/package.json +48 -0
- package/dist/runtime/test-helpers.d.mts +22 -0
- package/dist/runtime/test-helpers.mjs +144 -0
- package/dist/runtime/vitest.config.d.mts +9837 -0
- package/dist/runtime/vitest.config.mjs +8 -0
- package/package.json +32 -11
- package/bin/ccqa.ts +0 -2
- package/src/claude/invoke.test.ts +0 -167
- package/src/claude/invoke.ts +0 -238
- package/src/cli/generate-setup.ts +0 -215
- package/src/cli/generate.ts +0 -224
- package/src/cli/index.ts +0 -25
- package/src/cli/logger.ts +0 -45
- package/src/cli/run.ts +0 -280
- package/src/cli/trace-setup.ts +0 -124
- package/src/cli/trace.test.ts +0 -233
- package/src/cli/trace.ts +0 -244
- package/src/codegen/actions-to-script.ts +0 -185
- package/src/prompts/codegen.ts +0 -73
- package/src/prompts/trace.ts +0 -278
- package/src/runtime/test-helpers.ts +0 -133
- package/src/runtime/vitest.config.ts +0 -15
- package/src/spec/parser.test.ts +0 -135
- package/src/spec/parser.ts +0 -96
- package/src/store/index.test.ts +0 -107
- package/src/store/index.ts +0 -193
- package/src/types.test.ts +0 -96
- package/src/types.ts +0 -91
package/src/prompts/trace.ts
DELETED
|
@@ -1,278 +0,0 @@
|
|
|
1
|
-
import type { TestSpec, SetupSpec } from "../types.ts";
|
|
2
|
-
|
|
3
|
-
export function generateSessionName(): string {
|
|
4
|
-
return `ccqa-trace-${new Date().toISOString().replace(/[:.]/g, "-")}`;
|
|
5
|
-
}
|
|
6
|
-
|
|
7
|
-
export function buildTraceSystemPrompt(spec: TestSpec, options?: { sessionName?: string; skipCookiesClear?: boolean }): string {
|
|
8
|
-
const sessionName = options?.sessionName ?? generateSessionName();
|
|
9
|
-
const skipCookiesClear = options?.skipCookiesClear ?? false;
|
|
10
|
-
|
|
11
|
-
const stepsText = spec.steps
|
|
12
|
-
.map(
|
|
13
|
-
(step) => `### ${step.id}: ${step.title}
|
|
14
|
-
- **Instruction**: ${step.instruction}
|
|
15
|
-
- **Expected**: ${step.expected}`,
|
|
16
|
-
)
|
|
17
|
-
.join("\n\n");
|
|
18
|
-
|
|
19
|
-
const prereqText = spec.prerequisites
|
|
20
|
-
? `## Prerequisites\n${spec.prerequisites}\n\n`
|
|
21
|
-
: "";
|
|
22
|
-
|
|
23
|
-
return `You are an expert QA engineer executing a browser E2E test. Execute each step precisely and record every browser action as a structured log line.
|
|
24
|
-
|
|
25
|
-
## Session
|
|
26
|
-
|
|
27
|
-
SESSION NAME: \`${sessionName}\`
|
|
28
|
-
|
|
29
|
-
Always pass \`--session ${sessionName}\` to every \`agent-browser\` command.
|
|
30
|
-
|
|
31
|
-
## Browser Commands
|
|
32
|
-
|
|
33
|
-
\`\`\`
|
|
34
|
-
agent-browser --session SESSION open <url>
|
|
35
|
-
agent-browser --session SESSION snapshot
|
|
36
|
-
agent-browser --session SESSION click "<selector>"
|
|
37
|
-
agent-browser --session SESSION fill "<selector>" "<value>"
|
|
38
|
-
agent-browser --session SESSION check "<selector>"
|
|
39
|
-
agent-browser --session SESSION uncheck "<selector>"
|
|
40
|
-
agent-browser --session SESSION press <Key>
|
|
41
|
-
agent-browser --session SESSION select "<selector>" "<value>"
|
|
42
|
-
agent-browser --session SESSION hover "<selector>"
|
|
43
|
-
agent-browser --session SESSION wait --text "<text>"
|
|
44
|
-
agent-browser --session SESSION cookies clear
|
|
45
|
-
\`\`\`
|
|
46
|
-
|
|
47
|
-
## Selector Rules
|
|
48
|
-
|
|
49
|
-
**ALLOWED — these formats only:**
|
|
50
|
-
|
|
51
|
-
| Format | Use when |
|
|
52
|
-
|--------|----------|
|
|
53
|
-
| \`[aria-label='label']\` | Element has aria-label (check snapshot output) — **FIRST CHOICE** |
|
|
54
|
-
| \`text=visible text\` | Unique visible text, no aria-label |
|
|
55
|
-
| \`[placeholder='text']\` | Input identified by placeholder |
|
|
56
|
-
| \`[type='password']\` | Password inputs only |
|
|
57
|
-
| \`a[href*='pattern']\` | Links where \`text=\` fails — use the URL pattern from the ARIA snapshot (e.g. \`a[href*='/settings']\`) |
|
|
58
|
-
|
|
59
|
-
**FORBIDDEN — these will break recorded tests or are not valid commands:**
|
|
60
|
-
|
|
61
|
-
- \`@ref\` / \`@e1\` / \`e14\` — reference IDs are session-specific and change every run; never use them
|
|
62
|
-
- \`[role='button']\` or \`[type='checkbox']\` alone — matches too many elements
|
|
63
|
-
- Bare tag selectors: \`button\`, \`td\`, \`tr\`, \`main a\`, \`table tbody tr:nth-child(N)\` — these are positional/non-deterministic and will fail on replay
|
|
64
|
-
- \`find ...\`, \`textbox ...\`, \`label ...\` — not valid agent-browser commands; these are **blocked** and will fail
|
|
65
|
-
- JavaScript execution (\`eval\`, \`js\`) — **blocked** at the hook level; cannot bypass this restriction
|
|
66
|
-
|
|
67
|
-
**Selector workflow:**
|
|
68
|
-
1. Run \`snapshot\` — read the ARIA tree output carefully
|
|
69
|
-
2. Find the element; note its exact \`aria-label\` value if present
|
|
70
|
-
3. If aria-label present → use \`[aria-label='...']\`; otherwise → use \`text=...\`
|
|
71
|
-
4. If \`text=...\` fails for a link → look at the ARIA snapshot for the link's URL, then use \`a[href*='...']\` with a distinctive URL substring (e.g. \`a[href*='/dashboard']\`, \`a[href*='filter=active']\`)
|
|
72
|
-
5. If clicking a table row → look for \`<a>\` links inside the row in the ARIA snapshot, then use \`a[href*='...']\` targeting that link's URL pattern
|
|
73
|
-
6. For checkboxes: try \`check "text=Label"\` or \`check "[aria-label='Label']"\`
|
|
74
|
-
7. Never guess — if a selector fails once, take a fresh snapshot before retrying
|
|
75
|
-
|
|
76
|
-
## Test Specification
|
|
77
|
-
|
|
78
|
-
Title: ${spec.title}
|
|
79
|
-
Base URL: ${spec.baseUrl}
|
|
80
|
-
|
|
81
|
-
${prereqText}## Steps
|
|
82
|
-
|
|
83
|
-
${stepsText}
|
|
84
|
-
|
|
85
|
-
## Execution Workflow
|
|
86
|
-
|
|
87
|
-
For each step:
|
|
88
|
-
1. Emit \`STEP_START|<step-id>|<step-title>\`
|
|
89
|
-
2. Run \`snapshot\` and identify selectors from the ARIA tree
|
|
90
|
-
3. Execute the action using an ALLOWED selector
|
|
91
|
-
4. Emit \`AB_ACTION|...\` for every browser action (see below)
|
|
92
|
-
5. Run \`snapshot\` again to verify the outcome
|
|
93
|
-
6. Confirm at least **two independent signals** (URL change, element appearance, text change, etc.)
|
|
94
|
-
7. For each verified signal, emit \`AB_ACTION|assert|...\` (see Assertion Protocol below)
|
|
95
|
-
8. Emit \`ROUTE_STEP|...\`
|
|
96
|
-
9. Emit \`STEP_DONE\`, \`ASSERTION_FAILED\`, or \`STEP_SKIPPED\`
|
|
97
|
-
|
|
98
|
-
**After form submission or navigation:** take a snapshot before continuing. If an intermediate screen appears (e.g. account selection, role picker), complete it and emit AB_ACTION for each interaction.
|
|
99
|
-
|
|
100
|
-
## Guardrails
|
|
101
|
-
|
|
102
|
-
- **Stop after 3 consecutive failures on the same step** — emit \`ASSERTION_FAILED\` and report the blocker. Failures include: selector not found, element not interactable, command blocked by hook.
|
|
103
|
-
- **Do NOT use workarounds** — if all ALLOWED selectors fail, do NOT fall back to \`mouse move\`, coordinate-based clicks, \`Tab\`+\`Enter\` keyboard navigation, or any other indirect method. These cannot be recorded as reliable test actions. Instead, emit \`ASSERTION_FAILED\` with category \`selector-drift\` and describe which element you could not reach.
|
|
104
|
-
- **Do NOT use bare tag selectors** — never use \`click "button"\`, \`click "td"\`, \`click "main a"\`, or \`click "a"\` alone. These match too many elements and are non-deterministic. Always use a specific ALLOWED selector format.
|
|
105
|
-
- Do NOT retry a selector without taking a fresh snapshot first
|
|
106
|
-
- Do NOT work around blockers (login walls, missing data, captchas) — stop and report
|
|
107
|
-
- **Do NOT suppress errors** — never use \`2>/dev/null\`, \`|| true\`, \`; other-command\`, or any other technique that hides agent-browser failures. Each \`agent-browser\` command must run standalone so failures are properly detected and recorded.
|
|
108
|
-
|
|
109
|
-
## Source Code Reference
|
|
110
|
-
|
|
111
|
-
You have access to **Read**, **Grep**, and **Glob** tools to inspect the application source code. Use them proactively to find correct selectors — do NOT guess \`a[href*='...']\` patterns by trial and error.
|
|
112
|
-
|
|
113
|
-
**When to read source code:**
|
|
114
|
-
- Before clicking a link: Grep for the link text or URL pattern in the codebase to find the exact \`href\` value
|
|
115
|
-
- Before navigating to a new page: Glob for page/route files to understand the URL structure
|
|
116
|
-
- When the ARIA snapshot shows an element but \`text=\` and \`[aria-label=]\` selectors fail: Read the component to find what HTML attributes the element has
|
|
117
|
-
|
|
118
|
-
**How:**
|
|
119
|
-
1. Use \`Grep\` to search for UI text, component names, or URL patterns
|
|
120
|
-
2. Use \`Read\` to inspect the component's JSX/TSX and find \`href\`, \`aria-label\`, \`data-testid\`, or class names
|
|
121
|
-
3. Build a precise ALLOWED selector from the discovered attributes
|
|
122
|
-
|
|
123
|
-
**Rules:**
|
|
124
|
-
- Only READ source files — never modify them
|
|
125
|
-
- Keep source reading focused — search for specific strings, not entire directories
|
|
126
|
-
|
|
127
|
-
## Waiting for Async Operations
|
|
128
|
-
|
|
129
|
-
Prefer the \`wait\` command over polling:
|
|
130
|
-
|
|
131
|
-
\`\`\`bash
|
|
132
|
-
# Best: wait for expected text to appear
|
|
133
|
-
agent-browser --session ${sessionName} wait --text "<completion text>"
|
|
134
|
-
\`\`\`
|
|
135
|
-
|
|
136
|
-
If polling is required (e.g. waiting for a spinner to disappear):
|
|
137
|
-
|
|
138
|
-
\`\`\`bash
|
|
139
|
-
for i in $(seq 1 18); do
|
|
140
|
-
sleep 10
|
|
141
|
-
result=$(agent-browser --session ${sessionName} snapshot 2>&1)
|
|
142
|
-
# Check result for the expected change and break when found
|
|
143
|
-
echo "$result" | grep -q "<done indicator>" && break
|
|
144
|
-
done
|
|
145
|
-
agent-browser --session ${sessionName} snapshot
|
|
146
|
-
\`\`\`
|
|
147
|
-
|
|
148
|
-
After waiting, always take a final snapshot. Emit \`AB_ACTION|wait|text=<text>|<label>\`.
|
|
149
|
-
|
|
150
|
-
## AB_ACTION Protocol
|
|
151
|
-
|
|
152
|
-
After **every** browser action, emit one line (outside any code block):
|
|
153
|
-
|
|
154
|
-
\`\`\`
|
|
155
|
-
AB_ACTION|cookies_clear
|
|
156
|
-
AB_ACTION|open|<url>
|
|
157
|
-
AB_ACTION|click|<selector>|<visible label>
|
|
158
|
-
AB_ACTION|dblclick|<selector>|<visible label>
|
|
159
|
-
AB_ACTION|fill|<selector>|<value>|<aria label>
|
|
160
|
-
AB_ACTION|check|<selector>|<visible label>
|
|
161
|
-
AB_ACTION|uncheck|<selector>|<visible label>
|
|
162
|
-
AB_ACTION|press|<Key>
|
|
163
|
-
AB_ACTION|select|<selector>|<value>|<aria label>
|
|
164
|
-
AB_ACTION|hover|<selector>|<visible label>
|
|
165
|
-
AB_ACTION|scroll|<direction>|<pixels>
|
|
166
|
-
AB_ACTION|drag|<source selector>|<target selector>|<source label>
|
|
167
|
-
AB_ACTION|wait|<selector or text>|<label>
|
|
168
|
-
AB_ACTION|snapshot|<key observation, max 100 chars>
|
|
169
|
-
AB_ACTION|assert|<assertType>|<selector or "">|<value or "">|<observation>
|
|
170
|
-
\`\`\`
|
|
171
|
-
|
|
172
|
-
The selector in AB_ACTION must be one of the ALLOWED formats above.
|
|
173
|
-
|
|
174
|
-
## Assertion Protocol
|
|
175
|
-
|
|
176
|
-
After verifying each step, emit \`AB_ACTION|assert\` lines for each signal you confirmed.
|
|
177
|
-
|
|
178
|
-
**Available assertTypes:**
|
|
179
|
-
|
|
180
|
-
| assertType | Use when | selector | value |
|
|
181
|
-
|------------|----------|----------|-------|
|
|
182
|
-
| \`text_visible\` | Stable text appears on page | (empty) | text to find |
|
|
183
|
-
| \`text_not_visible\` | Text should be gone | (empty) | text that should be absent |
|
|
184
|
-
| \`element_visible\` | Element is visible | CSS selector | (empty) |
|
|
185
|
-
| \`element_not_visible\` | Element is hidden/removed | CSS selector | (empty) |
|
|
186
|
-
| \`url_contains\` | URL contains a pattern | (empty) | URL substring |
|
|
187
|
-
| \`element_enabled\` | Button/input is enabled | CSS selector | (empty) |
|
|
188
|
-
| \`element_disabled\` | Button/input is disabled | CSS selector | (empty) |
|
|
189
|
-
| \`element_checked\` | Checkbox is checked | CSS selector | (empty) |
|
|
190
|
-
| \`element_unchecked\` | Checkbox is unchecked | CSS selector | (empty) |
|
|
191
|
-
|
|
192
|
-
**Stability rules — CRITICAL:**
|
|
193
|
-
- **NEVER** assert on: timestamps (dates, times), session IDs, exact numeric counts that vary between runs
|
|
194
|
-
- For dynamic counts (e.g. "42 results"): assert on the STABLE part only (e.g. "results"), not the number
|
|
195
|
-
- **PREFER** asserting on: status text, button labels, URL patterns, element enabled/disabled state
|
|
196
|
-
|
|
197
|
-
**Page context rules — CRITICAL:**
|
|
198
|
-
- After a page navigation (\`open\` or \`click\` that navigates), take a **fresh snapshot** BEFORE emitting any assertions
|
|
199
|
-
- Only assert on text/elements that are visible on the **current** page — never assert on text from the previous page
|
|
200
|
-
- If you navigated away from a page, its text is gone — do not emit \`text_visible\` for it
|
|
201
|
-
|
|
202
|
-
**Selector rules for assert actions — CRITICAL:**
|
|
203
|
-
- Use the **same ALLOWED formats** as browser actions — never invent aria-label values
|
|
204
|
-
- Only use \`[aria-label='...']\` if that **exact** aria-label string appears in the current ARIA snapshot output
|
|
205
|
-
- When unsure, prefer \`text_visible\`/\`text_not_visible\` (no selector needed) over guessing a selector
|
|
206
|
-
- For \`element_disabled\`/\`element_enabled\`: use a CSS class selector if no aria-label is confirmed in the snapshot
|
|
207
|
-
|
|
208
|
-
**Examples:**
|
|
209
|
-
\`\`\`
|
|
210
|
-
AB_ACTION|assert|url_contains|||/dashboard|Navigated to dashboard
|
|
211
|
-
AB_ACTION|assert|element_disabled|.btn-submit||Submit button disabled before form is valid
|
|
212
|
-
AB_ACTION|assert|element_enabled|.btn-submit||Submit button enabled after form is filled
|
|
213
|
-
AB_ACTION|assert|text_visible|||Loading|Operation started
|
|
214
|
-
AB_ACTION|assert|text_visible|||Done|Operation completed
|
|
215
|
-
AB_ACTION|assert|text_visible|||Success|Confirmation message appeared
|
|
216
|
-
\`\`\`
|
|
217
|
-
|
|
218
|
-
## Status Protocol
|
|
219
|
-
|
|
220
|
-
Emit exactly one status line per step (outside any code block):
|
|
221
|
-
|
|
222
|
-
\`\`\`
|
|
223
|
-
STEP_START|<step-id>|<step-title>
|
|
224
|
-
STEP_DONE|<step-id>|<what was verified>
|
|
225
|
-
ASSERTION_FAILED|<step-id>|<category: app-bug|env-issue|auth-blocked|missing-test-data|selector-drift|agent-misread>: <reason>
|
|
226
|
-
STEP_SKIPPED|<step-id>|<reason>
|
|
227
|
-
RUN_COMPLETED|passed|<summary>
|
|
228
|
-
RUN_COMPLETED|failed|<summary>
|
|
229
|
-
\`\`\`
|
|
230
|
-
|
|
231
|
-
## Route Recording
|
|
232
|
-
|
|
233
|
-
After each step (outside any code block):
|
|
234
|
-
|
|
235
|
-
\`\`\`
|
|
236
|
-
ROUTE_STEP|<step-id>|<step-title>|ACTION:<what you did>|OBSERVATION:<what you verified>|STATUS:<PASSED|FAILED|SKIPPED>
|
|
237
|
-
\`\`\`
|
|
238
|
-
|
|
239
|
-
## Start
|
|
240
|
-
|
|
241
|
-
${skipCookiesClear ? `A setup procedure has already been executed in this session. Do NOT clear cookies — keep the existing session state.
|
|
242
|
-
|
|
243
|
-
\`\`\`bash
|
|
244
|
-
agent-browser --session ${sessionName} open ${spec.baseUrl}
|
|
245
|
-
\`\`\`
|
|
246
|
-
|
|
247
|
-
Emit:
|
|
248
|
-
\`\`\`
|
|
249
|
-
AB_ACTION|open|${spec.baseUrl}
|
|
250
|
-
\`\`\`` : `\`\`\`bash
|
|
251
|
-
agent-browser --session ${sessionName} cookies clear
|
|
252
|
-
agent-browser --session ${sessionName} open ${spec.baseUrl}
|
|
253
|
-
\`\`\`
|
|
254
|
-
|
|
255
|
-
Emit:
|
|
256
|
-
\`\`\`
|
|
257
|
-
AB_ACTION|cookies_clear
|
|
258
|
-
AB_ACTION|open|${spec.baseUrl}
|
|
259
|
-
\`\`\``}
|
|
260
|
-
|
|
261
|
-
Then emit \`STEP_START|step-01|...\` and begin.`;
|
|
262
|
-
}
|
|
263
|
-
|
|
264
|
-
export function buildTracePrompt(spec: TestSpec): string {
|
|
265
|
-
return `Execute the test for "${spec.title}" at ${spec.baseUrl}.`;
|
|
266
|
-
}
|
|
267
|
-
|
|
268
|
-
export function buildSetupTraceSystemPrompt(spec: SetupSpec): string {
|
|
269
|
-
return buildTraceSystemPrompt({
|
|
270
|
-
title: spec.title,
|
|
271
|
-
baseUrl: "about:blank",
|
|
272
|
-
steps: spec.steps,
|
|
273
|
-
});
|
|
274
|
-
}
|
|
275
|
-
|
|
276
|
-
export function buildSetupTracePrompt(spec: SetupSpec): string {
|
|
277
|
-
return `Execute the setup procedure "${spec.title}". Follow each step precisely.`;
|
|
278
|
-
}
|
|
@@ -1,133 +0,0 @@
|
|
|
1
|
-
import { spawnSync } from "node:child_process";
|
|
2
|
-
import { createRequire } from "node:module";
|
|
3
|
-
|
|
4
|
-
// Use createRequire instead of import.meta.resolve so this module works under
|
|
5
|
-
// Vite/Vitest's SSR transform (which replaces import.meta with a shim that
|
|
6
|
-
// lacks .resolve). import.meta.url survives the transform, so createRequire
|
|
7
|
-
// based on it can still locate peer-installed packages.
|
|
8
|
-
const require = createRequire(import.meta.url);
|
|
9
|
-
const AB = require.resolve("agent-browser/bin/agent-browser.js");
|
|
10
|
-
|
|
11
|
-
type Result = { status: number | null; stdout: string; stderr: string };
|
|
12
|
-
|
|
13
|
-
function spawnAB(args: string[]): Result {
|
|
14
|
-
const result = spawnSync(AB, args, { stdio: "pipe" });
|
|
15
|
-
return {
|
|
16
|
-
status: result.status,
|
|
17
|
-
stdout: result.stdout?.toString() ?? "",
|
|
18
|
-
stderr: result.stderr?.toString() ?? "",
|
|
19
|
-
};
|
|
20
|
-
}
|
|
21
|
-
|
|
22
|
-
function logStep(action: string, args: readonly unknown[]): void {
|
|
23
|
-
const pretty = args.map((a) => (typeof a === "string" ? a : JSON.stringify(a))).join(" ");
|
|
24
|
-
process.stdout.write(` ▶ ${action.padEnd(14)} ${pretty}\n`);
|
|
25
|
-
}
|
|
26
|
-
|
|
27
|
-
function fail(summary: string, result: Result): never {
|
|
28
|
-
process.stdout.write(` ✗ ${summary}\n`);
|
|
29
|
-
const details = [result.stdout, result.stderr].map((s) => s.trim()).filter(Boolean).join("\n");
|
|
30
|
-
if (details) {
|
|
31
|
-
for (const line of details.split("\n")) {
|
|
32
|
-
process.stdout.write(` ${line}\n`);
|
|
33
|
-
}
|
|
34
|
-
}
|
|
35
|
-
throw new Error(summary);
|
|
36
|
-
}
|
|
37
|
-
|
|
38
|
-
export function ab(...args: string[]): void {
|
|
39
|
-
const [command = "", ...rest] = args;
|
|
40
|
-
logStep(command, rest);
|
|
41
|
-
const result = spawnAB(args);
|
|
42
|
-
if (result.status !== 0) {
|
|
43
|
-
fail(`agent-browser ${command} failed (exit ${result.status})`, result);
|
|
44
|
-
}
|
|
45
|
-
}
|
|
46
|
-
|
|
47
|
-
/** Wait for element/text with an explicit timeout so long-running async ops don't hang. */
|
|
48
|
-
export function abWait(selector: string, timeoutMs = 180_000): void {
|
|
49
|
-
logStep("wait", [selector]);
|
|
50
|
-
const args = selector.startsWith("text=")
|
|
51
|
-
? ["wait", "--text", selector.slice(5), "--timeout", String(timeoutMs)]
|
|
52
|
-
: ["wait", selector, "--timeout", String(timeoutMs)];
|
|
53
|
-
const result = spawnAB(args);
|
|
54
|
-
if (result.status !== 0) {
|
|
55
|
-
fail(`wait failed: ${selector}`, result);
|
|
56
|
-
}
|
|
57
|
-
}
|
|
58
|
-
|
|
59
|
-
/** Assert stable text is visible on page (via wait --text). */
|
|
60
|
-
export function abAssertTextVisible(text: string, timeoutMs = 30_000): void {
|
|
61
|
-
logStep("assert.text", [text]);
|
|
62
|
-
const result = spawnAB(["wait", "--text", text, "--timeout", String(timeoutMs)]);
|
|
63
|
-
if (result.status !== 0) {
|
|
64
|
-
fail(`Assertion failed: text ${JSON.stringify(text)} not found within ${timeoutMs}ms`, result);
|
|
65
|
-
}
|
|
66
|
-
}
|
|
67
|
-
|
|
68
|
-
/** Assert element is visible (via wait). */
|
|
69
|
-
export function abAssertVisible(selector: string, timeoutMs = 30_000): void {
|
|
70
|
-
logStep("assert.visible", [selector]);
|
|
71
|
-
const result = spawnAB(["wait", selector, "--timeout", String(timeoutMs)]);
|
|
72
|
-
if (result.status !== 0) {
|
|
73
|
-
fail(`Assertion failed: ${JSON.stringify(selector)} not visible within ${timeoutMs}ms`, result);
|
|
74
|
-
}
|
|
75
|
-
}
|
|
76
|
-
|
|
77
|
-
/** Assert element is NOT visible (via wait --state hidden). */
|
|
78
|
-
export function abAssertNotVisible(selector: string, timeoutMs = 30_000): void {
|
|
79
|
-
logStep("assert.hidden", [selector]);
|
|
80
|
-
const args = selector.startsWith("text=")
|
|
81
|
-
? ["wait", "--text", selector.slice(5), "--state", "hidden", "--timeout", String(timeoutMs)]
|
|
82
|
-
: ["wait", selector, "--state", "hidden", "--timeout", String(timeoutMs)];
|
|
83
|
-
const result = spawnAB(args);
|
|
84
|
-
if (result.status !== 0) {
|
|
85
|
-
fail(`Assertion failed: ${JSON.stringify(selector)} still visible after ${timeoutMs}ms`, result);
|
|
86
|
-
}
|
|
87
|
-
}
|
|
88
|
-
|
|
89
|
-
/** Assert URL contains a pattern (via get url). */
|
|
90
|
-
export function abAssertUrl(pattern: string): void {
|
|
91
|
-
logStep("assert.url", [pattern]);
|
|
92
|
-
const result = spawnAB(["get", "url"]);
|
|
93
|
-
const url = result.stdout.trim();
|
|
94
|
-
if (!url.includes(pattern)) {
|
|
95
|
-
fail(`Assertion failed: URL ${JSON.stringify(url)} does not contain ${JSON.stringify(pattern)}`, result);
|
|
96
|
-
}
|
|
97
|
-
}
|
|
98
|
-
|
|
99
|
-
/** Assert element is enabled (via is enabled). */
|
|
100
|
-
export function abAssertEnabled(selector: string): void {
|
|
101
|
-
logStep("assert.enabled", [selector]);
|
|
102
|
-
const result = spawnAB(["is", "enabled", selector]);
|
|
103
|
-
if (result.status !== 0) fail(`Assertion failed: element ${JSON.stringify(selector)} not found`, result);
|
|
104
|
-
const value = result.stdout.trim();
|
|
105
|
-
if (value !== "true") fail(`Assertion failed: ${JSON.stringify(selector)} is not enabled (got: ${value})`, result);
|
|
106
|
-
}
|
|
107
|
-
|
|
108
|
-
/** Assert element is disabled (via is enabled). */
|
|
109
|
-
export function abAssertDisabled(selector: string): void {
|
|
110
|
-
logStep("assert.disabled", [selector]);
|
|
111
|
-
const result = spawnAB(["is", "enabled", selector]);
|
|
112
|
-
if (result.status !== 0) fail(`Assertion failed: element ${JSON.stringify(selector)} not found`, result);
|
|
113
|
-
const value = result.stdout.trim();
|
|
114
|
-
if (value !== "false") fail(`Assertion failed: ${JSON.stringify(selector)} is not disabled (got: ${value})`, result);
|
|
115
|
-
}
|
|
116
|
-
|
|
117
|
-
/** Assert checkbox is checked (via is checked). */
|
|
118
|
-
export function abAssertChecked(selector: string): void {
|
|
119
|
-
logStep("assert.checked", [selector]);
|
|
120
|
-
const result = spawnAB(["is", "checked", selector]);
|
|
121
|
-
if (result.status !== 0) fail(`Assertion failed: element ${JSON.stringify(selector)} not found`, result);
|
|
122
|
-
const value = result.stdout.trim();
|
|
123
|
-
if (value !== "true") fail(`Assertion failed: ${JSON.stringify(selector)} is not checked (got: ${value})`, result);
|
|
124
|
-
}
|
|
125
|
-
|
|
126
|
-
/** Assert checkbox is unchecked (via is checked). */
|
|
127
|
-
export function abAssertUnchecked(selector: string): void {
|
|
128
|
-
logStep("assert.unchecked", [selector]);
|
|
129
|
-
const result = spawnAB(["is", "checked", selector]);
|
|
130
|
-
if (result.status !== 0) fail(`Assertion failed: element ${JSON.stringify(selector)} not found`, result);
|
|
131
|
-
const value = result.stdout.trim();
|
|
132
|
-
if (value !== "false") fail(`Assertion failed: ${JSON.stringify(selector)} is not unchecked (got: ${value})`, result);
|
|
133
|
-
}
|
|
@@ -1,15 +0,0 @@
|
|
|
1
|
-
import { defineConfig } from "vitest/config";
|
|
2
|
-
|
|
3
|
-
// Default vitest config used by `ccqa run`. Passed via `--config` so the host
|
|
4
|
-
// project's vitest.config.ts is not picked up. ccqa specs are Node-side E2E
|
|
5
|
-
// tests driving agent-browser; host configs (setupFiles, jsdom/happy-dom
|
|
6
|
-
// environment, @ aliases, etc.) don't apply and often break the run.
|
|
7
|
-
//
|
|
8
|
-
// Consumers can override by placing .ccqa/vitest.config.ts in their project;
|
|
9
|
-
// `ccqa run` prefers that file when present.
|
|
10
|
-
export default defineConfig({
|
|
11
|
-
test: {
|
|
12
|
-
environment: "node",
|
|
13
|
-
globals: false,
|
|
14
|
-
},
|
|
15
|
-
});
|
package/src/spec/parser.test.ts
DELETED
|
@@ -1,135 +0,0 @@
|
|
|
1
|
-
import { describe, test, expect } from "bun:test";
|
|
2
|
-
import { parseTestSpec } from "./parser.ts";
|
|
3
|
-
|
|
4
|
-
const BASIC_SPEC = `---
|
|
5
|
-
title: My Test
|
|
6
|
-
baseUrl: http://localhost:3000
|
|
7
|
-
---
|
|
8
|
-
|
|
9
|
-
### Step 1: Login
|
|
10
|
-
**Instruction**: Navigate to the login page
|
|
11
|
-
**Expected**: Login form is visible
|
|
12
|
-
`;
|
|
13
|
-
|
|
14
|
-
describe("parseTestSpec", () => {
|
|
15
|
-
test("parses title and baseUrl from frontmatter", () => {
|
|
16
|
-
const result = parseTestSpec(BASIC_SPEC);
|
|
17
|
-
expect(result.title).toBe("My Test");
|
|
18
|
-
expect(result.baseUrl).toBe("http://localhost:3000");
|
|
19
|
-
});
|
|
20
|
-
|
|
21
|
-
test("defaults title to Untitled when missing", () => {
|
|
22
|
-
const result = parseTestSpec("---\nbaseUrl: http://localhost:3000\n---\n");
|
|
23
|
-
expect(result.title).toBe("Untitled");
|
|
24
|
-
});
|
|
25
|
-
|
|
26
|
-
test("defaults baseUrl to http://localhost:3000 when missing", () => {
|
|
27
|
-
const result = parseTestSpec("---\ntitle: My Test\n---\n");
|
|
28
|
-
expect(result.baseUrl).toBe("http://localhost:3000");
|
|
29
|
-
});
|
|
30
|
-
|
|
31
|
-
test("parses a single step correctly", () => {
|
|
32
|
-
const result = parseTestSpec(BASIC_SPEC);
|
|
33
|
-
expect(result.steps).toHaveLength(1);
|
|
34
|
-
expect(result.steps[0]).toEqual({
|
|
35
|
-
id: "step-01",
|
|
36
|
-
title: "Login",
|
|
37
|
-
instruction: "Navigate to the login page",
|
|
38
|
-
expected: "Login form is visible",
|
|
39
|
-
});
|
|
40
|
-
});
|
|
41
|
-
|
|
42
|
-
test("parses multiple steps with correct id generation", () => {
|
|
43
|
-
const spec = `---
|
|
44
|
-
title: Multi
|
|
45
|
-
baseUrl: http://localhost
|
|
46
|
-
---
|
|
47
|
-
|
|
48
|
-
### Step 1: First
|
|
49
|
-
**Instruction**: Do first
|
|
50
|
-
**Expected**: First done
|
|
51
|
-
|
|
52
|
-
### Step 2: Second
|
|
53
|
-
**Instruction**: Do second
|
|
54
|
-
**Expected**: Second done
|
|
55
|
-
`;
|
|
56
|
-
const result = parseTestSpec(spec);
|
|
57
|
-
expect(result.steps).toHaveLength(2);
|
|
58
|
-
expect(result.steps[0]?.id).toBe("step-01");
|
|
59
|
-
expect(result.steps[1]?.id).toBe("step-02");
|
|
60
|
-
});
|
|
61
|
-
|
|
62
|
-
test("returns empty steps array when no steps defined", () => {
|
|
63
|
-
const result = parseTestSpec("---\ntitle: No Steps\nbaseUrl: http://localhost\n---\n");
|
|
64
|
-
expect(result.steps).toHaveLength(0);
|
|
65
|
-
});
|
|
66
|
-
|
|
67
|
-
test("skips steps missing Instruction or Expected", () => {
|
|
68
|
-
const spec = `---
|
|
69
|
-
title: Missing Fields
|
|
70
|
-
baseUrl: http://localhost
|
|
71
|
-
---
|
|
72
|
-
|
|
73
|
-
### Step 1: Complete
|
|
74
|
-
**Instruction**: Do something
|
|
75
|
-
**Expected**: See something
|
|
76
|
-
|
|
77
|
-
### Step 2: Missing Expected
|
|
78
|
-
**Instruction**: Do something
|
|
79
|
-
|
|
80
|
-
### Step 3: Missing Instruction
|
|
81
|
-
**Expected**: See something
|
|
82
|
-
`;
|
|
83
|
-
const result = parseTestSpec(spec);
|
|
84
|
-
expect(result.steps).toHaveLength(1);
|
|
85
|
-
expect(result.steps[0]?.id).toBe("step-01");
|
|
86
|
-
});
|
|
87
|
-
|
|
88
|
-
test("parses prerequisites when present", () => {
|
|
89
|
-
const spec = `---
|
|
90
|
-
title: With Prereqs
|
|
91
|
-
baseUrl: http://localhost
|
|
92
|
-
---
|
|
93
|
-
|
|
94
|
-
## Prerequisites
|
|
95
|
-
User must be logged in as admin.
|
|
96
|
-
|
|
97
|
-
### Step 1: Check
|
|
98
|
-
**Instruction**: Do check
|
|
99
|
-
**Expected**: Check done
|
|
100
|
-
`;
|
|
101
|
-
const result = parseTestSpec(spec);
|
|
102
|
-
expect(result.prerequisites).toBe("User must be logged in as admin.");
|
|
103
|
-
});
|
|
104
|
-
|
|
105
|
-
test("returns undefined for prerequisites when section is absent", () => {
|
|
106
|
-
const result = parseTestSpec(BASIC_SPEC);
|
|
107
|
-
expect(result.prerequisites).toBeUndefined();
|
|
108
|
-
});
|
|
109
|
-
|
|
110
|
-
test("parses setups from frontmatter", () => {
|
|
111
|
-
const spec = `---
|
|
112
|
-
title: Setup Test
|
|
113
|
-
baseUrl: http://localhost:3000
|
|
114
|
-
setups:
|
|
115
|
-
- name: login
|
|
116
|
-
params:
|
|
117
|
-
email: user@example.com
|
|
118
|
-
password: secret
|
|
119
|
-
---
|
|
120
|
-
|
|
121
|
-
### Step 1: Check
|
|
122
|
-
**Instruction**: Do check
|
|
123
|
-
**Expected**: Check done
|
|
124
|
-
`;
|
|
125
|
-
const result = parseTestSpec(spec);
|
|
126
|
-
expect(result.setups).toHaveLength(1);
|
|
127
|
-
expect(result.setups![0]!.name).toBe("login");
|
|
128
|
-
expect(result.setups![0]!.params).toEqual({ email: "user@example.com", password: "secret" });
|
|
129
|
-
});
|
|
130
|
-
|
|
131
|
-
test("returns undefined for setups when not specified", () => {
|
|
132
|
-
const result = parseTestSpec(BASIC_SPEC);
|
|
133
|
-
expect(result.setups).toBeUndefined();
|
|
134
|
-
});
|
|
135
|
-
});
|
package/src/spec/parser.ts
DELETED
|
@@ -1,96 +0,0 @@
|
|
|
1
|
-
import matter from "gray-matter";
|
|
2
|
-
import type { TestSpec, TestStep, SetupSpec, SetupRef } from "../types.ts";
|
|
3
|
-
|
|
4
|
-
export function parseTestSpec(content: string): TestSpec {
|
|
5
|
-
const { data, content: body } = matter(content);
|
|
6
|
-
|
|
7
|
-
const steps = parseSteps(body);
|
|
8
|
-
|
|
9
|
-
const prerequisites = parsePrerequisites(body);
|
|
10
|
-
|
|
11
|
-
return {
|
|
12
|
-
title: String(data["title"] ?? "Untitled"),
|
|
13
|
-
baseUrl: String(data["baseUrl"] ?? "http://localhost:3000"),
|
|
14
|
-
prerequisites: prerequisites || undefined,
|
|
15
|
-
setups: parseSetupRefs(data["setups"]),
|
|
16
|
-
steps,
|
|
17
|
-
};
|
|
18
|
-
}
|
|
19
|
-
|
|
20
|
-
export function parseSetupSpec(content: string): SetupSpec {
|
|
21
|
-
const { data, content: body } = matter(content);
|
|
22
|
-
|
|
23
|
-
const steps = parseSteps(body);
|
|
24
|
-
const placeholders = parsePlaceholders(data["placeholders"]);
|
|
25
|
-
|
|
26
|
-
return {
|
|
27
|
-
title: String(data["title"] ?? "Untitled"),
|
|
28
|
-
placeholders: Object.keys(placeholders).length > 0 ? placeholders : undefined,
|
|
29
|
-
steps,
|
|
30
|
-
};
|
|
31
|
-
}
|
|
32
|
-
|
|
33
|
-
function parsePlaceholders(raw: unknown): Record<string, { dummy: string; description?: string }> {
|
|
34
|
-
if (!raw || typeof raw !== "object") return {};
|
|
35
|
-
const result: Record<string, { dummy: string; description?: string }> = {};
|
|
36
|
-
for (const [key, val] of Object.entries(raw as Record<string, unknown>)) {
|
|
37
|
-
if (val && typeof val === "object" && "dummy" in val) {
|
|
38
|
-
const v = val as Record<string, unknown>;
|
|
39
|
-
result[key] = {
|
|
40
|
-
dummy: String(v["dummy"]),
|
|
41
|
-
description: v["description"] ? String(v["description"]) : undefined,
|
|
42
|
-
};
|
|
43
|
-
}
|
|
44
|
-
}
|
|
45
|
-
return result;
|
|
46
|
-
}
|
|
47
|
-
|
|
48
|
-
function parseSetupRefs(raw: unknown): SetupRef[] | undefined {
|
|
49
|
-
if (!Array.isArray(raw)) return undefined;
|
|
50
|
-
const refs: SetupRef[] = [];
|
|
51
|
-
for (const item of raw) {
|
|
52
|
-
if (typeof item === "object" && item !== null && "name" in item) {
|
|
53
|
-
const i = item as Record<string, unknown>;
|
|
54
|
-
refs.push({
|
|
55
|
-
name: String(i["name"]),
|
|
56
|
-
params: i["params"] && typeof i["params"] === "object"
|
|
57
|
-
? Object.fromEntries(
|
|
58
|
-
Object.entries(i["params"] as Record<string, unknown>).map(([k, v]) => [k, String(v)])
|
|
59
|
-
)
|
|
60
|
-
: undefined,
|
|
61
|
-
});
|
|
62
|
-
}
|
|
63
|
-
}
|
|
64
|
-
return refs.length > 0 ? refs : undefined;
|
|
65
|
-
}
|
|
66
|
-
|
|
67
|
-
function parsePrerequisites(body: string): string | null {
|
|
68
|
-
const match = body.match(/##\s+Prerequisites\s+([\s\S]*?)(?=##|$)/);
|
|
69
|
-
if (!match || !match[1]) return null;
|
|
70
|
-
return match[1].trim();
|
|
71
|
-
}
|
|
72
|
-
|
|
73
|
-
function parseSteps(body: string): TestStep[] {
|
|
74
|
-
const stepBlocks = body.split(/###\s+Step\s+\d+:/);
|
|
75
|
-
const steps: TestStep[] = [];
|
|
76
|
-
|
|
77
|
-
for (let i = 1; i < stepBlocks.length; i++) {
|
|
78
|
-
const block = stepBlocks[i];
|
|
79
|
-
if (!block) continue;
|
|
80
|
-
|
|
81
|
-
const titleMatch = block.match(/^(.+)/);
|
|
82
|
-
const instructionMatch = block.match(/\*\*Instruction\*\*:\s*(.+)/);
|
|
83
|
-
const expectedMatch = block.match(/\*\*Expected\*\*:\s*(.+)/);
|
|
84
|
-
|
|
85
|
-
if (!titleMatch || !instructionMatch || !expectedMatch) continue;
|
|
86
|
-
|
|
87
|
-
steps.push({
|
|
88
|
-
id: `step-${String(i).padStart(2, "0")}`,
|
|
89
|
-
title: titleMatch[1]?.trim() ?? "",
|
|
90
|
-
instruction: instructionMatch[1]?.trim() ?? "",
|
|
91
|
-
expected: expectedMatch[1]?.trim() ?? "",
|
|
92
|
-
});
|
|
93
|
-
}
|
|
94
|
-
|
|
95
|
-
return steps;
|
|
96
|
-
}
|