@hasna/testers 0.0.28 → 0.0.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -154
- package/README.md +60 -0
- package/dist/cli/index.js +944 -76
- package/dist/db/database.d.ts.map +1 -1
- package/dist/db/personas.d.ts.map +1 -1
- package/dist/db/runs.d.ts +29 -0
- package/dist/db/runs.d.ts.map +1 -1
- package/dist/db/scenarios.d.ts +12 -0
- package/dist/db/scenarios.d.ts.map +1 -1
- package/dist/db/sessions.d.ts +36 -0
- package/dist/db/sessions.d.ts.map +1 -0
- package/dist/db/step-results.d.ts +30 -0
- package/dist/db/step-results.d.ts.map +1 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +439 -24
- package/dist/lib/a11y-audit.d.ts +54 -0
- package/dist/lib/a11y-audit.d.ts.map +1 -0
- package/dist/lib/api-discovery.d.ts +46 -0
- package/dist/lib/api-discovery.d.ts.map +1 -0
- package/dist/lib/assertions.d.ts.map +1 -1
- package/dist/lib/auth-profiles.d.ts +16 -0
- package/dist/lib/auth-profiles.d.ts.map +1 -0
- package/dist/lib/auth-session-pool.d.ts +57 -0
- package/dist/lib/auth-session-pool.d.ts.map +1 -0
- package/dist/lib/batch-actions.d.ts +44 -0
- package/dist/lib/batch-actions.d.ts.map +1 -0
- package/dist/lib/browser-compat.d.ts +14 -0
- package/dist/lib/browser-compat.d.ts.map +1 -0
- package/dist/lib/browser.d.ts +7 -8
- package/dist/lib/browser.d.ts.map +1 -1
- package/dist/lib/ci.d.ts +12 -0
- package/dist/lib/ci.d.ts.map +1 -1
- package/dist/lib/discovery.d.ts +23 -0
- package/dist/lib/discovery.d.ts.map +1 -0
- package/dist/lib/dom-mutation.d.ts +53 -0
- package/dist/lib/dom-mutation.d.ts.map +1 -0
- package/dist/lib/environment.d.ts +26 -0
- package/dist/lib/environment.d.ts.map +1 -0
- package/dist/lib/health-scan.d.ts +2 -1
- package/dist/lib/health-scan.d.ts.map +1 -1
- package/dist/lib/junit-export.d.ts +24 -0
- package/dist/lib/junit-export.d.ts.map +1 -0
- package/dist/lib/network-mock.d.ts +38 -0
- package/dist/lib/network-mock.d.ts.map +1 -0
- package/dist/lib/offline-mode.d.ts +31 -0
- package/dist/lib/offline-mode.d.ts.map +1 -0
- package/dist/lib/pdf-export.d.ts +27 -0
- package/dist/lib/pdf-export.d.ts.map +1 -0
- package/dist/lib/performance.d.ts +65 -0
- package/dist/lib/performance.d.ts.map +1 -0
- package/dist/lib/pr-comment.d.ts +27 -0
- package/dist/lib/pr-comment.d.ts.map +1 -0
- package/dist/lib/preview-detect.d.ts +27 -0
- package/dist/lib/preview-detect.d.ts.map +1 -0
- package/dist/lib/recorder.d.ts +42 -0
- package/dist/lib/recorder.d.ts.map +1 -1
- package/dist/lib/repo-discovery.d.ts +102 -0
- package/dist/lib/repo-discovery.d.ts.map +1 -0
- package/dist/lib/repo-executor.d.ts +56 -0
- package/dist/lib/repo-executor.d.ts.map +1 -0
- package/dist/lib/responsive.d.ts +43 -0
- package/dist/lib/responsive.d.ts.map +1 -0
- package/dist/lib/runner.d.ts +1 -0
- package/dist/lib/runner.d.ts.map +1 -1
- package/dist/lib/scenario-chain.d.ts +52 -0
- package/dist/lib/scenario-chain.d.ts.map +1 -0
- package/dist/lib/templates.d.ts.map +1 -1
- package/dist/lib/webhooks.d.ts +3 -0
- package/dist/lib/webhooks.d.ts.map +1 -1
- package/dist/mcp/index.js +491 -38
- package/dist/sdk/index.d.ts +47 -0
- package/dist/sdk/index.d.ts.map +1 -0
- package/dist/server/index.js +274 -28
- package/dist/types/index.d.ts +64 -2
- package/dist/types/index.d.ts.map +1 -1
- package/package.json +1 -1
package/dist/cli/index.js
CHANGED
|
@@ -2148,7 +2148,8 @@ function scenarioFromRow(row) {
|
|
|
2148
2148
|
createdAt: row.created_at,
|
|
2149
2149
|
updatedAt: row.updated_at,
|
|
2150
2150
|
lastPassedAt: row.last_passed_at ?? null,
|
|
2151
|
-
lastPassedUrl: row.last_passed_url ?? null
|
|
2151
|
+
lastPassedUrl: row.last_passed_url ?? null,
|
|
2152
|
+
parameters: row.parameters ? JSON.parse(row.parameters) : null
|
|
2152
2153
|
};
|
|
2153
2154
|
}
|
|
2154
2155
|
function runFromRow(row) {
|
|
@@ -2168,7 +2169,14 @@ function runFromRow(row) {
|
|
|
2168
2169
|
metadata: row.metadata ? JSON.parse(row.metadata) : null,
|
|
2169
2170
|
isBaseline: row.is_baseline === 1,
|
|
2170
2171
|
samples: row.samples ?? 1,
|
|
2171
|
-
flakinessThreshold: row.flakiness_threshold ?? 0.95
|
|
2172
|
+
flakinessThreshold: row.flakiness_threshold ?? 0.95,
|
|
2173
|
+
prNumber: row.pr_number ?? null,
|
|
2174
|
+
prTitle: row.pr_title ?? null,
|
|
2175
|
+
prBranch: row.pr_branch ?? null,
|
|
2176
|
+
prBaseBranch: row.pr_base_branch ?? null,
|
|
2177
|
+
prCommitSha: row.pr_commit_sha ?? null,
|
|
2178
|
+
prUrl: row.pr_url ?? null,
|
|
2179
|
+
ghAppInstallationId: row.gh_app_installation_id ?? null
|
|
2172
2180
|
};
|
|
2173
2181
|
}
|
|
2174
2182
|
function resultFromRow(row) {
|
|
@@ -2189,7 +2197,8 @@ function resultFromRow(row) {
|
|
|
2189
2197
|
createdAt: row.created_at,
|
|
2190
2198
|
personaId: row.persona_id ?? null,
|
|
2191
2199
|
personaName: row.persona_name ?? null,
|
|
2192
|
-
failureAnalysis: row.failure_analysis ? JSON.parse(row.failure_analysis) : null
|
|
2200
|
+
failureAnalysis: row.failure_analysis ? JSON.parse(row.failure_analysis) : null,
|
|
2201
|
+
harPath: row.har_path ?? null
|
|
2193
2202
|
};
|
|
2194
2203
|
}
|
|
2195
2204
|
function screenshotFromRow(row) {
|
|
@@ -2281,7 +2290,10 @@ function personaFromRow(row) {
|
|
|
2281
2290
|
email: row.auth_email,
|
|
2282
2291
|
password: row.auth_password,
|
|
2283
2292
|
loginPath: row.auth_login_path ?? "/login",
|
|
2284
|
-
cookies: row.auth_cookies ? JSON.parse(row.auth_cookies) : null
|
|
2293
|
+
cookies: row.auth_cookies ? JSON.parse(row.auth_cookies) : null,
|
|
2294
|
+
strategy: row.auth_strategy ?? "form-login",
|
|
2295
|
+
headers: row.auth_headers ? JSON.parse(row.auth_headers) : undefined,
|
|
2296
|
+
customScript: row.auth_script ?? undefined
|
|
2285
2297
|
} : null
|
|
2286
2298
|
};
|
|
2287
2299
|
}
|
|
@@ -12200,6 +12212,43 @@ ALTER TABLE scenarios ADD COLUMN required_role TEXT;
|
|
|
12200
12212
|
machine_id TEXT,
|
|
12201
12213
|
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
12202
12214
|
);
|
|
12215
|
+
`,
|
|
12216
|
+
`
|
|
12217
|
+
ALTER TABLE results ADD COLUMN har_path TEXT;
|
|
12218
|
+
`,
|
|
12219
|
+
`
|
|
12220
|
+
ALTER TABLE scenarios ADD COLUMN parameters TEXT;
|
|
12221
|
+
`,
|
|
12222
|
+
`
|
|
12223
|
+
ALTER TABLE personas ADD COLUMN auth_strategy TEXT DEFAULT 'form-login';
|
|
12224
|
+
ALTER TABLE personas ADD COLUMN auth_headers TEXT;
|
|
12225
|
+
ALTER TABLE personas ADD COLUMN auth_script TEXT;
|
|
12226
|
+
`,
|
|
12227
|
+
`
|
|
12228
|
+
CREATE TABLE IF NOT EXISTS step_results (
|
|
12229
|
+
id TEXT PRIMARY KEY,
|
|
12230
|
+
result_id TEXT NOT NULL REFERENCES results(id) ON DELETE CASCADE,
|
|
12231
|
+
step_number INTEGER NOT NULL,
|
|
12232
|
+
action TEXT NOT NULL,
|
|
12233
|
+
status TEXT NOT NULL DEFAULT 'running' CHECK(status IN ('passed','failed','error','running','skipped')),
|
|
12234
|
+
tool_name TEXT,
|
|
12235
|
+
tool_input TEXT,
|
|
12236
|
+
tool_result TEXT,
|
|
12237
|
+
thinking TEXT,
|
|
12238
|
+
error TEXT,
|
|
12239
|
+
duration_ms INTEGER,
|
|
12240
|
+
screenshot_id TEXT REFERENCES screenshots(id),
|
|
12241
|
+
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
12242
|
+
);
|
|
12243
|
+
`,
|
|
12244
|
+
`
|
|
12245
|
+
ALTER TABLE runs ADD COLUMN pr_number INTEGER;
|
|
12246
|
+
ALTER TABLE runs ADD COLUMN pr_title TEXT;
|
|
12247
|
+
ALTER TABLE runs ADD COLUMN pr_branch TEXT;
|
|
12248
|
+
ALTER TABLE runs ADD COLUMN pr_base_branch TEXT;
|
|
12249
|
+
ALTER TABLE runs ADD COLUMN pr_commit_sha TEXT;
|
|
12250
|
+
ALTER TABLE runs ADD COLUMN pr_url TEXT;
|
|
12251
|
+
ALTER TABLE runs ADD COLUMN gh_app_installation_id TEXT;
|
|
12203
12252
|
`
|
|
12204
12253
|
];
|
|
12205
12254
|
});
|
|
@@ -12207,6 +12256,7 @@ ALTER TABLE scenarios ADD COLUMN required_role TEXT;
|
|
|
12207
12256
|
// src/db/scenarios.ts
|
|
12208
12257
|
var exports_scenarios = {};
|
|
12209
12258
|
__export(exports_scenarios, {
|
|
12259
|
+
upsertScenario: () => upsertScenario,
|
|
12210
12260
|
updateScenarioPassedCache: () => updateScenarioPassedCache,
|
|
12211
12261
|
updateScenario: () => updateScenario,
|
|
12212
12262
|
listScenarios: () => listScenarios,
|
|
@@ -12235,9 +12285,9 @@ function createScenario(input) {
|
|
|
12235
12285
|
const short_id = nextShortId(input.projectId);
|
|
12236
12286
|
const timestamp = now();
|
|
12237
12287
|
db2.query(`
|
|
12238
|
-
INSERT INTO scenarios (id, short_id, project_id, name, description, steps, tags, priority, model, timeout_ms, target_path, requires_auth, auth_config, metadata, assertions, version, created_at, updated_at)
|
|
12239
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1, ?, ?)
|
|
12240
|
-
`).run(id, short_id, input.projectId ?? null, input.name, input.description, JSON.stringify(input.steps ?? []), JSON.stringify(input.tags ?? []), input.priority ?? "medium", input.model ?? null, input.timeoutMs ?? null, input.targetPath ?? null, input.requiresAuth ? 1 : 0, input.authConfig ? JSON.stringify(input.authConfig) : null, input.metadata ? JSON.stringify(input.metadata) : null, JSON.stringify(input.assertions ?? []), timestamp, timestamp);
|
|
12288
|
+
INSERT INTO scenarios (id, short_id, project_id, name, description, steps, tags, priority, model, timeout_ms, target_path, requires_auth, auth_config, metadata, assertions, parameters, version, created_at, updated_at)
|
|
12289
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1, ?, ?)
|
|
12290
|
+
`).run(id, short_id, input.projectId ?? null, input.name, input.description, JSON.stringify(input.steps ?? []), JSON.stringify(input.tags ?? []), input.priority ?? "medium", input.model ?? null, input.timeoutMs ?? null, input.targetPath ?? null, input.requiresAuth ? 1 : 0, input.authConfig ? JSON.stringify(input.authConfig) : null, input.metadata ? JSON.stringify(input.metadata) : null, JSON.stringify(input.assertions ?? []), input.parameters ? JSON.stringify(input.parameters) : null, timestamp, timestamp);
|
|
12241
12291
|
return getScenario(id);
|
|
12242
12292
|
}
|
|
12243
12293
|
function getScenario(id) {
|
|
@@ -12387,6 +12437,10 @@ function updateScenario(id, input, version) {
|
|
|
12387
12437
|
sets.push("assertions = ?");
|
|
12388
12438
|
params.push(JSON.stringify(input.assertions));
|
|
12389
12439
|
}
|
|
12440
|
+
if (input.parameters !== undefined) {
|
|
12441
|
+
sets.push("parameters = ?");
|
|
12442
|
+
params.push(JSON.stringify(input.parameters));
|
|
12443
|
+
}
|
|
12390
12444
|
if (sets.length === 0) {
|
|
12391
12445
|
return existing;
|
|
12392
12446
|
}
|
|
@@ -12459,6 +12513,75 @@ function deleteScenario(id) {
|
|
|
12459
12513
|
const result = db2.query("DELETE FROM scenarios WHERE id = ?").run(scenario.id);
|
|
12460
12514
|
return result.changes > 0;
|
|
12461
12515
|
}
|
|
12516
|
+
function upsertScenario(input) {
|
|
12517
|
+
const db2 = getDatabase();
|
|
12518
|
+
const existing = db2.query("SELECT * FROM scenarios WHERE name = ? AND project_id IS NOT DISTINCT FROM ?").get(input.name, input.projectId ?? null);
|
|
12519
|
+
if (!existing) {
|
|
12520
|
+
return { scenario: createScenario(input), action: "created" };
|
|
12521
|
+
}
|
|
12522
|
+
const existingSteps = JSON.parse(existing.steps);
|
|
12523
|
+
const existingTags = JSON.parse(existing.tags);
|
|
12524
|
+
const newSteps = input.steps ?? [];
|
|
12525
|
+
const newTags = input.tags ?? [];
|
|
12526
|
+
const isIdentical = existing.description === (input.description ?? "") && existingSteps.length === newSteps.length && existingSteps.every((s, i) => s === newSteps[i]) && existingTags.length === newTags.length && existingTags.every((t, i) => t === newTags[i]) && existing.priority === (input.priority ?? "medium");
|
|
12527
|
+
if (isIdentical) {
|
|
12528
|
+
return { scenario: scenarioFromRow(existing), action: "deduped" };
|
|
12529
|
+
}
|
|
12530
|
+
const sets = [];
|
|
12531
|
+
const params = [];
|
|
12532
|
+
if (input.description !== undefined) {
|
|
12533
|
+
sets.push("description = ?");
|
|
12534
|
+
params.push(input.description);
|
|
12535
|
+
}
|
|
12536
|
+
if (input.steps !== undefined) {
|
|
12537
|
+
sets.push("steps = ?");
|
|
12538
|
+
params.push(JSON.stringify(input.steps));
|
|
12539
|
+
}
|
|
12540
|
+
if (input.tags !== undefined) {
|
|
12541
|
+
sets.push("tags = ?");
|
|
12542
|
+
params.push(JSON.stringify(input.tags));
|
|
12543
|
+
}
|
|
12544
|
+
if (input.priority !== undefined) {
|
|
12545
|
+
sets.push("priority = ?");
|
|
12546
|
+
params.push(input.priority);
|
|
12547
|
+
}
|
|
12548
|
+
if (input.model !== undefined) {
|
|
12549
|
+
sets.push("model = ?");
|
|
12550
|
+
params.push(input.model);
|
|
12551
|
+
}
|
|
12552
|
+
if (input.timeoutMs !== undefined) {
|
|
12553
|
+
sets.push("timeout_ms = ?");
|
|
12554
|
+
params.push(input.timeoutMs);
|
|
12555
|
+
}
|
|
12556
|
+
if (input.targetPath !== undefined) {
|
|
12557
|
+
sets.push("target_path = ?");
|
|
12558
|
+
params.push(input.targetPath);
|
|
12559
|
+
}
|
|
12560
|
+
if (input.requiresAuth !== undefined) {
|
|
12561
|
+
sets.push("requires_auth = ?");
|
|
12562
|
+
params.push(input.requiresAuth ? 1 : 0);
|
|
12563
|
+
}
|
|
12564
|
+
if (input.authConfig !== undefined) {
|
|
12565
|
+
sets.push("auth_config = ?");
|
|
12566
|
+
params.push(JSON.stringify(input.authConfig));
|
|
12567
|
+
}
|
|
12568
|
+
if (input.metadata !== undefined) {
|
|
12569
|
+
sets.push("metadata = ?");
|
|
12570
|
+
params.push(JSON.stringify(input.metadata));
|
|
12571
|
+
}
|
|
12572
|
+
if (input.assertions !== undefined) {
|
|
12573
|
+
sets.push("assertions = ?");
|
|
12574
|
+
params.push(JSON.stringify(input.assertions));
|
|
12575
|
+
}
|
|
12576
|
+
sets.push("version = ?", "updated_at = ?");
|
|
12577
|
+
params.push(existing.version + 1, now());
|
|
12578
|
+
params.push(existing.id);
|
|
12579
|
+
const result = db2.query(`UPDATE scenarios SET ${sets.join(", ")} WHERE id = ?`).run(...params);
|
|
12580
|
+
if (result.changes === 0) {
|
|
12581
|
+
return { scenario: scenarioFromRow(existing), action: "deduped" };
|
|
12582
|
+
}
|
|
12583
|
+
return { scenario: getScenario(existing.id), action: "updated" };
|
|
12584
|
+
}
|
|
12462
12585
|
var init_scenarios = __esm(() => {
|
|
12463
12586
|
init_types();
|
|
12464
12587
|
init_database();
|
|
@@ -12468,8 +12591,12 @@ var init_scenarios = __esm(() => {
|
|
|
12468
12591
|
var exports_runs = {};
|
|
12469
12592
|
__export(exports_runs, {
|
|
12470
12593
|
updateRun: () => updateRun,
|
|
12594
|
+
updatePrRunMetadata: () => updatePrRunMetadata,
|
|
12471
12595
|
listRuns: () => listRuns,
|
|
12596
|
+
listPrRuns: () => listPrRuns,
|
|
12597
|
+
getRunsByPr: () => getRunsByPr,
|
|
12472
12598
|
getRun: () => getRun,
|
|
12599
|
+
getLatestPrRun: () => getLatestPrRun,
|
|
12473
12600
|
deleteRun: () => deleteRun,
|
|
12474
12601
|
createRun: () => createRun,
|
|
12475
12602
|
countRuns: () => countRuns
|
|
@@ -12479,9 +12606,9 @@ function createRun(input) {
|
|
|
12479
12606
|
const id = uuid();
|
|
12480
12607
|
const timestamp = now();
|
|
12481
12608
|
db2.query(`
|
|
12482
|
-
INSERT INTO runs (id, project_id, status, url, model, headed, parallel, total, passed, failed, started_at, finished_at, metadata, samples, flakiness_threshold)
|
|
12483
|
-
VALUES (?, ?, 'pending', ?, ?, ?, ?, 0, 0, 0, ?, NULL, ?, ?, ?)
|
|
12484
|
-
`).run(id, input.projectId ?? null, input.url, input.model, input.headed ? 1 : 0, input.parallel ?? 1, timestamp,
|
|
12609
|
+
INSERT INTO runs (id, project_id, status, url, model, headed, parallel, total, passed, failed, started_at, finished_at, metadata, samples, flakiness_threshold, pr_number, pr_title, pr_branch, pr_base_branch, pr_commit_sha, pr_url, gh_app_installation_id)
|
|
12610
|
+
VALUES (?, ?, 'pending', ?, ?, ?, ?, 0, 0, 0, ?, NULL, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
12611
|
+
`).run(id, input.projectId ?? null, input.url, input.model, input.headed ? 1 : 0, input.parallel ?? 1, timestamp, JSON.stringify({}), input.samples ?? 1, input.flakinessThreshold ?? 0.95, input.prNumber ?? null, input.prTitle ?? null, input.prBranch ?? null, input.prBaseBranch ?? null, input.prCommitSha ?? null, input.prUrl ?? null, input.ghAppInstallationId ?? null);
|
|
12485
12612
|
return getRun(id);
|
|
12486
12613
|
}
|
|
12487
12614
|
function getRun(id) {
|
|
@@ -12509,6 +12636,14 @@ function listRuns(filter) {
|
|
|
12509
12636
|
conditions.push("status = ?");
|
|
12510
12637
|
params.push(filter.status);
|
|
12511
12638
|
}
|
|
12639
|
+
if (filter?.since) {
|
|
12640
|
+
conditions.push("started_at >= ?");
|
|
12641
|
+
params.push(filter.since);
|
|
12642
|
+
}
|
|
12643
|
+
if (filter?.until) {
|
|
12644
|
+
conditions.push("started_at <= ?");
|
|
12645
|
+
params.push(filter.until);
|
|
12646
|
+
}
|
|
12512
12647
|
let sql = "SELECT * FROM runs";
|
|
12513
12648
|
if (conditions.length > 0) {
|
|
12514
12649
|
sql += " WHERE " + conditions.join(" AND ");
|
|
@@ -12540,6 +12675,14 @@ function countRuns(filter) {
|
|
|
12540
12675
|
conditions.push("status = ?");
|
|
12541
12676
|
params.push(filter.status);
|
|
12542
12677
|
}
|
|
12678
|
+
if (filter?.since) {
|
|
12679
|
+
conditions.push("started_at >= ?");
|
|
12680
|
+
params.push(filter.since);
|
|
12681
|
+
}
|
|
12682
|
+
if (filter?.until) {
|
|
12683
|
+
conditions.push("started_at <= ?");
|
|
12684
|
+
params.push(filter.until);
|
|
12685
|
+
}
|
|
12543
12686
|
let sql = "SELECT COUNT(*) as count FROM runs";
|
|
12544
12687
|
if (conditions.length > 0)
|
|
12545
12688
|
sql += " WHERE " + conditions.join(" AND ");
|
|
@@ -12617,6 +12760,52 @@ function deleteRun(id) {
|
|
|
12617
12760
|
const result = db2.query("DELETE FROM runs WHERE id = ?").run(run.id);
|
|
12618
12761
|
return result.changes > 0;
|
|
12619
12762
|
}
|
|
12763
|
+
function getRunsByPr(prNumber) {
|
|
12764
|
+
const db2 = getDatabase();
|
|
12765
|
+
const rows = db2.query("SELECT * FROM runs WHERE pr_number = ? ORDER BY started_at DESC").all(prNumber);
|
|
12766
|
+
return rows.map(runFromRow);
|
|
12767
|
+
}
|
|
12768
|
+
function getLatestPrRun(prNumber) {
|
|
12769
|
+
const db2 = getDatabase();
|
|
12770
|
+
const row = db2.query("SELECT * FROM runs WHERE pr_number = ? ORDER BY started_at DESC, rowid DESC LIMIT 1").get(prNumber);
|
|
12771
|
+
return row ? runFromRow(row) : null;
|
|
12772
|
+
}
|
|
12773
|
+
function listPrRuns(filter) {
|
|
12774
|
+
const db2 = getDatabase();
|
|
12775
|
+
const conditions = ["pr_number IS NOT NULL"];
|
|
12776
|
+
const params = [];
|
|
12777
|
+
if (filter?.branch) {
|
|
12778
|
+
conditions.push("pr_branch = ?");
|
|
12779
|
+
params.push(filter.branch);
|
|
12780
|
+
}
|
|
12781
|
+
if (filter?.baseBranch) {
|
|
12782
|
+
conditions.push("pr_base_branch = ?");
|
|
12783
|
+
params.push(filter.baseBranch);
|
|
12784
|
+
}
|
|
12785
|
+
let sql = "SELECT * FROM runs WHERE " + conditions.join(" AND ") + " ORDER BY started_at DESC";
|
|
12786
|
+
if (filter?.limit) {
|
|
12787
|
+
sql += " LIMIT ?";
|
|
12788
|
+
params.push(filter.limit);
|
|
12789
|
+
}
|
|
12790
|
+
if (filter?.offset) {
|
|
12791
|
+
sql += " OFFSET ?";
|
|
12792
|
+
params.push(filter.offset);
|
|
12793
|
+
}
|
|
12794
|
+
const rows = db2.query(sql).all(...params);
|
|
12795
|
+
return rows.map(runFromRow);
|
|
12796
|
+
}
|
|
12797
|
+
function updatePrRunMetadata(runId, prData) {
|
|
12798
|
+
const db2 = getDatabase();
|
|
12799
|
+
const existing = getRun(runId);
|
|
12800
|
+
if (!existing) {
|
|
12801
|
+
throw new Error(`Run not found: ${runId}`);
|
|
12802
|
+
}
|
|
12803
|
+
db2.query(`
|
|
12804
|
+
UPDATE runs SET pr_number = ?, pr_title = ?, pr_branch = ?, pr_base_branch = ?, pr_commit_sha = ?, pr_url = ?, gh_app_installation_id = ?
|
|
12805
|
+
WHERE id = ?
|
|
12806
|
+
`).run(prData.prNumber, prData.prTitle ?? null, prData.prBranch ?? null, prData.prBaseBranch ?? null, prData.prCommitSha ?? null, prData.prUrl ?? null, prData.ghAppInstallationId ?? null, runId);
|
|
12807
|
+
return getRun(runId);
|
|
12808
|
+
}
|
|
12620
12809
|
var init_runs = __esm(() => {
|
|
12621
12810
|
init_types();
|
|
12622
12811
|
init_database();
|
|
@@ -13382,6 +13571,16 @@ async function launchBrowser(options) {
|
|
|
13382
13571
|
const headless = options?.headless ?? true;
|
|
13383
13572
|
const viewport = options?.viewport ?? DEFAULT_VIEWPORT;
|
|
13384
13573
|
try {
|
|
13574
|
+
if (engine === "playwright-firefox") {
|
|
13575
|
+
const { firefox } = await import("playwright");
|
|
13576
|
+
const browser = await firefox.launch({ headless });
|
|
13577
|
+
return browser;
|
|
13578
|
+
}
|
|
13579
|
+
if (engine === "playwright-webkit") {
|
|
13580
|
+
const { webkit } = await import("playwright");
|
|
13581
|
+
const browser = await webkit.launch({ headless });
|
|
13582
|
+
return browser;
|
|
13583
|
+
}
|
|
13385
13584
|
return await launchPlaywright({ headless, viewport });
|
|
13386
13585
|
} catch (error) {
|
|
13387
13586
|
const message = error instanceof Error ? error.message : String(error);
|
|
@@ -13498,8 +13697,9 @@ async function installBrowser(engine) {
|
|
|
13498
13697
|
const { installLightpanda: installLightpanda2 } = await Promise.resolve().then(() => (init_browser_lightpanda(), exports_browser_lightpanda));
|
|
13499
13698
|
return installLightpanda2();
|
|
13500
13699
|
}
|
|
13700
|
+
const browserName = engine === "playwright-firefox" ? "firefox" : engine === "playwright-webkit" ? "webkit" : "chromium";
|
|
13501
13701
|
try {
|
|
13502
|
-
execSync(
|
|
13702
|
+
execSync(`bunx playwright install ${browserName}`, {
|
|
13503
13703
|
stdio: "inherit"
|
|
13504
13704
|
});
|
|
13505
13705
|
} catch (error) {
|
|
@@ -13631,7 +13831,7 @@ function getDefaultConfig() {
|
|
|
13631
13831
|
browser: {
|
|
13632
13832
|
headless: true,
|
|
13633
13833
|
viewport: { width: 1280, height: 720 },
|
|
13634
|
-
timeout:
|
|
13834
|
+
timeout: 120000
|
|
13635
13835
|
},
|
|
13636
13836
|
screenshots: {
|
|
13637
13837
|
dir: join9(getTestersDir(), "screenshots"),
|
|
@@ -15657,6 +15857,76 @@ var init_costs = __esm(() => {
|
|
|
15657
15857
|
};
|
|
15658
15858
|
});
|
|
15659
15859
|
|
|
15860
|
+
// src/db/step-results.ts
|
|
15861
|
+
function createStepResult(input) {
|
|
15862
|
+
const db2 = getDatabase();
|
|
15863
|
+
const id = uuid();
|
|
15864
|
+
const timestamp = now();
|
|
15865
|
+
db2.query(`
|
|
15866
|
+
INSERT INTO step_results (id, result_id, step_number, action, status, tool_name, tool_input, thinking, created_at)
|
|
15867
|
+
VALUES (?, ?, ?, ?, 'running', ?, ?, ?, ?)
|
|
15868
|
+
`).run(id, input.resultId, input.stepNumber, input.action, input.toolName ?? null, input.toolInput ? JSON.stringify(input.toolInput) : null, input.thinking ?? null, timestamp);
|
|
15869
|
+
return getStepResult(id);
|
|
15870
|
+
}
|
|
15871
|
+
function getStepResult(id) {
|
|
15872
|
+
const db2 = getDatabase();
|
|
15873
|
+
const row = db2.query("SELECT * FROM step_results WHERE id = ?").get(id);
|
|
15874
|
+
return row ? stepResultFromRow(row) : null;
|
|
15875
|
+
}
|
|
15876
|
+
function updateStepResult(id, updates) {
|
|
15877
|
+
const db2 = getDatabase();
|
|
15878
|
+
const existing = getStepResult(id);
|
|
15879
|
+
if (!existing)
|
|
15880
|
+
return null;
|
|
15881
|
+
const sets = [];
|
|
15882
|
+
const params = [];
|
|
15883
|
+
if (updates.status !== undefined) {
|
|
15884
|
+
sets.push("status = ?");
|
|
15885
|
+
params.push(updates.status);
|
|
15886
|
+
}
|
|
15887
|
+
if (updates.toolResult !== undefined) {
|
|
15888
|
+
sets.push("tool_result = ?");
|
|
15889
|
+
params.push(updates.toolResult);
|
|
15890
|
+
}
|
|
15891
|
+
if (updates.error !== undefined) {
|
|
15892
|
+
sets.push("error = ?");
|
|
15893
|
+
params.push(updates.error);
|
|
15894
|
+
}
|
|
15895
|
+
if (updates.durationMs !== undefined) {
|
|
15896
|
+
sets.push("duration_ms = ?");
|
|
15897
|
+
params.push(updates.durationMs);
|
|
15898
|
+
}
|
|
15899
|
+
if (updates.screenshotId !== undefined) {
|
|
15900
|
+
sets.push("screenshot_id = ?");
|
|
15901
|
+
params.push(updates.screenshotId);
|
|
15902
|
+
}
|
|
15903
|
+
if (sets.length === 0)
|
|
15904
|
+
return existing;
|
|
15905
|
+
params.push(id);
|
|
15906
|
+
db2.query(`UPDATE step_results SET ${sets.join(", ")} WHERE id = ?`).run(...params);
|
|
15907
|
+
return getStepResult(id);
|
|
15908
|
+
}
|
|
15909
|
+
function stepResultFromRow(row) {
|
|
15910
|
+
return {
|
|
15911
|
+
id: row.id,
|
|
15912
|
+
resultId: row.result_id,
|
|
15913
|
+
stepNumber: row.step_number,
|
|
15914
|
+
action: row.action,
|
|
15915
|
+
status: row.status,
|
|
15916
|
+
toolName: row.tool_name,
|
|
15917
|
+
toolInput: row.tool_input ? JSON.parse(row.tool_input) : null,
|
|
15918
|
+
toolResult: row.tool_result,
|
|
15919
|
+
thinking: row.thinking,
|
|
15920
|
+
error: row.error,
|
|
15921
|
+
durationMs: row.duration_ms,
|
|
15922
|
+
screenshotId: row.screenshot_id,
|
|
15923
|
+
createdAt: row.created_at
|
|
15924
|
+
};
|
|
15925
|
+
}
|
|
15926
|
+
var init_step_results = __esm(() => {
|
|
15927
|
+
init_database();
|
|
15928
|
+
});
|
|
15929
|
+
|
|
15660
15930
|
// src/db/personas.ts
|
|
15661
15931
|
function createPersona(input) {
|
|
15662
15932
|
const db2 = getDatabase();
|
|
@@ -15664,9 +15934,9 @@ function createPersona(input) {
|
|
|
15664
15934
|
const short_id = shortUuid();
|
|
15665
15935
|
const timestamp = now();
|
|
15666
15936
|
db2.query(`
|
|
15667
|
-
INSERT INTO personas (id, short_id, project_id, name, description, role, instructions, traits, goals, behaviors, expertise_level, demographics, pain_points, metadata, enabled, auth_email, auth_password, auth_login_path, version, created_at, updated_at)
|
|
15668
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1, ?, ?)
|
|
15669
|
-
`).run(id, short_id, input.projectId ?? null, input.name, input.description ?? "", input.role, input.instructions ?? "", JSON.stringify(input.traits ?? []), JSON.stringify(input.goals ?? []), JSON.stringify(input.behaviors ?? []), input.expertiseLevel ?? "intermediate", JSON.stringify(input.demographics ?? {}), JSON.stringify(input.painPoints ?? []), input.metadata ? JSON.stringify(input.metadata) : "{}", input.enabled === false ? 0 : 1, input.authEmail ?? null, input.authPassword ?? null, input.authLoginPath ?? null, timestamp, timestamp);
|
|
15937
|
+
INSERT INTO personas (id, short_id, project_id, name, description, role, instructions, traits, goals, behaviors, expertise_level, demographics, pain_points, metadata, enabled, auth_email, auth_password, auth_login_path, auth_cookies, auth_strategy, auth_headers, auth_script, version, created_at, updated_at)
|
|
15938
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1, ?, ?)
|
|
15939
|
+
`).run(id, short_id, input.projectId ?? null, input.name, input.description ?? "", input.role, input.instructions ?? "", JSON.stringify(input.traits ?? []), JSON.stringify(input.goals ?? []), JSON.stringify(input.behaviors ?? []), input.expertiseLevel ?? "intermediate", JSON.stringify(input.demographics ?? {}), JSON.stringify(input.painPoints ?? []), input.metadata ? JSON.stringify(input.metadata) : "{}", input.enabled === false ? 0 : 1, input.authEmail ?? null, input.authPassword ?? null, input.authLoginPath ?? null, null, input.authStrategy ?? "form-login", input.authHeaders ? JSON.stringify(input.authHeaders) : null, input.authCustomScript ?? null, timestamp, timestamp);
|
|
15670
15940
|
return getPersona(id);
|
|
15671
15941
|
}
|
|
15672
15942
|
function getPersona(id) {
|
|
@@ -16215,6 +16485,24 @@ function signPayload(body, secret) {
|
|
|
16215
16485
|
}
|
|
16216
16486
|
return `sha256=${Math.abs(hash).toString(16).padStart(16, "0")}`;
|
|
16217
16487
|
}
|
|
16488
|
+
function formatDiscordPayload(payload) {
|
|
16489
|
+
const isPassed = payload.run.status === "passed";
|
|
16490
|
+
const color = isPassed ? 2278750 : 15680580;
|
|
16491
|
+
return {
|
|
16492
|
+
username: "open-testers",
|
|
16493
|
+
embeds: [
|
|
16494
|
+
{
|
|
16495
|
+
title: `Test Run ${payload.run.status.toUpperCase()}`,
|
|
16496
|
+
color,
|
|
16497
|
+
description: `URL: ${payload.run.url}
|
|
16498
|
+
` + `Results: ${payload.run.passed}/${payload.run.total} passed` + (payload.run.failed > 0 ? ` (${payload.run.failed} failed)` : "") + (payload.schedule ? `
|
|
16499
|
+
Schedule: ${payload.schedule.name}` : ""),
|
|
16500
|
+
timestamp: payload.timestamp,
|
|
16501
|
+
footer: { text: "open-testers" }
|
|
16502
|
+
}
|
|
16503
|
+
]
|
|
16504
|
+
};
|
|
16505
|
+
}
|
|
16218
16506
|
function formatSlackPayload(payload) {
|
|
16219
16507
|
const status = payload.run.status === "passed" ? ":white_check_mark:" : ":x:";
|
|
16220
16508
|
const color = payload.run.status === "passed" ? "#22c55e" : "#ef4444";
|
|
@@ -16257,7 +16545,8 @@ async function dispatchWebhooks(event, run, schedule) {
|
|
|
16257
16545
|
if (!webhook.events.includes(event) && !webhook.events.includes("*"))
|
|
16258
16546
|
continue;
|
|
16259
16547
|
const isSlack = webhook.url.includes("hooks.slack.com");
|
|
16260
|
-
const
|
|
16548
|
+
const isDiscord = webhook.url.includes("discord.com/api/webhooks") || webhook.url.includes("discordapp.com/api/webhooks");
|
|
16549
|
+
const body = isSlack ? JSON.stringify(formatSlackPayload(payload)) : isDiscord ? JSON.stringify(formatDiscordPayload(payload)) : JSON.stringify(payload);
|
|
16261
16550
|
const headers = {
|
|
16262
16551
|
"Content-Type": "application/json"
|
|
16263
16552
|
};
|
|
@@ -16744,6 +17033,8 @@ __export(exports_runner, {
|
|
|
16744
17033
|
runBatch: () => runBatch,
|
|
16745
17034
|
onRunEvent: () => onRunEvent
|
|
16746
17035
|
});
|
|
17036
|
+
import { mkdirSync as mkdirSync8 } from "fs";
|
|
17037
|
+
import { join as join13 } from "path";
|
|
16747
17038
|
import { enableNetworkLogging } from "@hasna/browser";
|
|
16748
17039
|
function onRunEvent(handler) {
|
|
16749
17040
|
eventHandler = handler;
|
|
@@ -16829,13 +17120,35 @@ async function runSingleScenario(scenario, runId, options) {
|
|
|
16829
17120
|
emit({ type: "scenario:start", scenarioId: scenario.id, scenarioName: scenario.name, resultId: result.id, runId });
|
|
16830
17121
|
let browser = null;
|
|
16831
17122
|
let page = null;
|
|
17123
|
+
let context = null;
|
|
17124
|
+
let harPath = null;
|
|
16832
17125
|
let stopNetworkLogging = null;
|
|
16833
17126
|
const networkErrors = [];
|
|
16834
17127
|
try {
|
|
16835
17128
|
browser = await launchBrowser({ headless: !(effectiveOptions.headed ?? false), engine: effectiveOptions.engine });
|
|
16836
|
-
|
|
16837
|
-
|
|
16838
|
-
|
|
17129
|
+
const useHar = effectiveOptions.engine !== "lightpanda" && effectiveOptions.engine !== "bun";
|
|
17130
|
+
if (useHar) {
|
|
17131
|
+
const testersDir = process.env["HASNA_TESTERS_DIR"] || join13(process.env["HOME"] || "", ".hasna", "testers");
|
|
17132
|
+
const harDir = join13(testersDir, "hars");
|
|
17133
|
+
mkdirSync8(harDir, { recursive: true });
|
|
17134
|
+
harPath = join13(harDir, `${result.id}.har`);
|
|
17135
|
+
const contextOptions = {
|
|
17136
|
+
viewport: config.browser.viewport,
|
|
17137
|
+
recordHar: { path: harPath, mode: "full" }
|
|
17138
|
+
};
|
|
17139
|
+
if (effectiveOptions.recordVideo) {
|
|
17140
|
+
const videoDir = join13(testersDir, "videos");
|
|
17141
|
+
mkdirSync8(videoDir, { recursive: true });
|
|
17142
|
+
contextOptions.recordVideo = { dir: videoDir, size: config.browser.viewport };
|
|
17143
|
+
}
|
|
17144
|
+
context = await browser.newContext(contextOptions);
|
|
17145
|
+
page = await context.newPage();
|
|
17146
|
+
} else {
|
|
17147
|
+
page = await getPage(browser, {
|
|
17148
|
+
viewport: config.browser.viewport,
|
|
17149
|
+
engine: effectiveOptions.engine
|
|
17150
|
+
});
|
|
17151
|
+
}
|
|
16839
17152
|
const targetUrl = scenario.targetPath ? `${options.url.replace(/\/$/, "")}${scenario.targetPath}` : options.url;
|
|
16840
17153
|
const scenarioTimeout = scenario.timeoutMs ?? options.timeout ?? config.browser.timeout ?? 60000;
|
|
16841
17154
|
registerSession({
|
|
@@ -16855,7 +17168,11 @@ async function runSingleScenario(scenario, runId, options) {
|
|
|
16855
17168
|
}
|
|
16856
17169
|
});
|
|
16857
17170
|
const consoleErrors = [];
|
|
17171
|
+
const consoleLogs = [];
|
|
17172
|
+
let currentStep = 0;
|
|
16858
17173
|
page.on("console", (msg) => {
|
|
17174
|
+
const logEntry = { step: currentStep > 0 ? currentStep : null, type: msg.type(), text: msg.text(), timestamp: Date.now() };
|
|
17175
|
+
consoleLogs.push(logEntry);
|
|
16859
17176
|
if (msg.type() === "error")
|
|
16860
17177
|
consoleErrors.push(msg.text());
|
|
16861
17178
|
});
|
|
@@ -16887,6 +17204,7 @@ async function runSingleScenario(scenario, runId, options) {
|
|
|
16887
17204
|
}
|
|
16888
17205
|
await page.goto(targetUrl, { timeout: Math.min(scenarioTimeout, 30000) });
|
|
16889
17206
|
const stepStartTimes = new Map;
|
|
17207
|
+
const stepResultIds = new Map;
|
|
16890
17208
|
const agentResult = await withTimeout(runAgentLoop({
|
|
16891
17209
|
client,
|
|
16892
17210
|
page,
|
|
@@ -16909,13 +17227,32 @@ async function runSingleScenario(scenario, runId, options) {
|
|
|
16909
17227
|
onStep: (stepEvent) => {
|
|
16910
17228
|
let stepDurationMs;
|
|
16911
17229
|
if (stepEvent.type === "tool_call") {
|
|
17230
|
+
currentStep = stepEvent.stepNumber;
|
|
16912
17231
|
stepStartTimes.set(stepEvent.stepNumber, Date.now());
|
|
17232
|
+
const stepResult = createStepResult({
|
|
17233
|
+
resultId: result.id,
|
|
17234
|
+
stepNumber: stepEvent.stepNumber,
|
|
17235
|
+
action: stepEvent.toolName ?? `step-${stepEvent.stepNumber}`,
|
|
17236
|
+
toolName: stepEvent.toolName,
|
|
17237
|
+
toolInput: stepEvent.toolInput,
|
|
17238
|
+
thinking: stepEvent.thinking
|
|
17239
|
+
});
|
|
17240
|
+
stepResultIds.set(stepEvent.stepNumber, stepResult.id);
|
|
16913
17241
|
} else if (stepEvent.type === "tool_result") {
|
|
16914
17242
|
const startTime = stepStartTimes.get(stepEvent.stepNumber);
|
|
16915
17243
|
if (startTime !== undefined) {
|
|
16916
17244
|
stepDurationMs = Date.now() - startTime;
|
|
16917
17245
|
stepStartTimes.delete(stepEvent.stepNumber);
|
|
16918
17246
|
}
|
|
17247
|
+
const stepResultId = stepResultIds.get(stepEvent.stepNumber);
|
|
17248
|
+
if (stepResultId) {
|
|
17249
|
+
const isSuccess = !stepEvent.toolResult?.toLowerCase().includes("error") && !stepEvent.toolResult?.toLowerCase().includes("failed");
|
|
17250
|
+
updateStepResult(stepResultId, {
|
|
17251
|
+
status: isSuccess ? "passed" : "failed",
|
|
17252
|
+
toolResult: stepEvent.toolResult,
|
|
17253
|
+
durationMs: stepDurationMs
|
|
17254
|
+
});
|
|
17255
|
+
}
|
|
16919
17256
|
}
|
|
16920
17257
|
emit({
|
|
16921
17258
|
type: `step:${stepEvent.type}`,
|
|
@@ -16964,7 +17301,7 @@ async function runSingleScenario(scenario, runId, options) {
|
|
|
16964
17301
|
durationMs: Date.now() - new Date(result.createdAt).getTime(),
|
|
16965
17302
|
tokensUsed: agentResult.tokensUsed,
|
|
16966
17303
|
costCents: estimateCost(model, agentResult.tokensUsed),
|
|
16967
|
-
metadata: networkErrors.length > 0 ? networkMeta :
|
|
17304
|
+
metadata: { consoleLogs, ...networkErrors.length > 0 ? networkMeta : {} }
|
|
16968
17305
|
});
|
|
16969
17306
|
if (agentResult.status === "failed" || agentResult.status === "error") {
|
|
16970
17307
|
const failureAnalysis = analyzeFailure(null, agentResult.reasoning ?? null);
|
|
@@ -17000,8 +17337,16 @@ async function runSingleScenario(scenario, runId, options) {
|
|
|
17000
17337
|
emit({ type: "scenario:error", scenarioId: scenario.id, scenarioName: scenario.name, error: errorMsg, runId });
|
|
17001
17338
|
return updatedResult;
|
|
17002
17339
|
} finally {
|
|
17003
|
-
if (
|
|
17004
|
-
|
|
17340
|
+
if (harPath) {
|
|
17341
|
+
try {
|
|
17342
|
+
updateResult(result.id, { metadata: { harPath } });
|
|
17343
|
+
} catch {}
|
|
17344
|
+
}
|
|
17345
|
+
if (browser) {
|
|
17346
|
+
try {
|
|
17347
|
+
await closeBrowser(browser, effectiveOptions.engine);
|
|
17348
|
+
} catch {}
|
|
17349
|
+
}
|
|
17005
17350
|
}
|
|
17006
17351
|
}
|
|
17007
17352
|
async function runBatch(scenarios, options) {
|
|
@@ -17297,6 +17642,7 @@ var init_runner = __esm(() => {
|
|
|
17297
17642
|
init_results();
|
|
17298
17643
|
init_costs();
|
|
17299
17644
|
init_screenshots();
|
|
17645
|
+
init_step_results();
|
|
17300
17646
|
init_scenarios();
|
|
17301
17647
|
init_personas();
|
|
17302
17648
|
init_browser();
|
|
@@ -24387,8 +24733,10 @@ var init_pii = __esm(() => {
|
|
|
24387
24733
|
// src/lib/ci.ts
|
|
24388
24734
|
var exports_ci = {};
|
|
24389
24735
|
__export(exports_ci, {
|
|
24736
|
+
resolvePullRequestNumber: () => resolvePullRequestNumber,
|
|
24390
24737
|
postGitHubComment: () => postGitHubComment,
|
|
24391
|
-
generateGitHubActionsWorkflow: () => generateGitHubActionsWorkflow
|
|
24738
|
+
generateGitHubActionsWorkflow: () => generateGitHubActionsWorkflow,
|
|
24739
|
+
formatPRComment: () => formatPRComment
|
|
24392
24740
|
});
|
|
24393
24741
|
function generateGitHubActionsWorkflow() {
|
|
24394
24742
|
return `name: AI QA Tests
|
|
@@ -24397,6 +24745,10 @@ on:
|
|
|
24397
24745
|
push:
|
|
24398
24746
|
branches: [main]
|
|
24399
24747
|
|
|
24748
|
+
permissions:
|
|
24749
|
+
contents: read
|
|
24750
|
+
pull-requests: write
|
|
24751
|
+
|
|
24400
24752
|
jobs:
|
|
24401
24753
|
test:
|
|
24402
24754
|
runs-on: ubuntu-latest
|
|
@@ -24411,6 +24763,7 @@ jobs:
|
|
|
24411
24763
|
TEST_URL: \${{ vars.TEST_URL || 'http://localhost:3000' }}
|
|
24412
24764
|
GITHUB_TOKEN: \${{ secrets.GITHUB_TOKEN }}
|
|
24413
24765
|
- run: testers report --latest --output report.html
|
|
24766
|
+
if: always()
|
|
24414
24767
|
- uses: actions/upload-artifact@v4
|
|
24415
24768
|
if: always()
|
|
24416
24769
|
with:
|
|
@@ -24421,46 +24774,73 @@ jobs:
|
|
|
24421
24774
|
`;
|
|
24422
24775
|
}
|
|
24423
24776
|
function formatPRComment(run, results, dashboardUrl) {
|
|
24424
|
-
const icon = run.status === "passed" ? "\u2705" : "\u274C";
|
|
24777
|
+
const icon = run.status === "passed" ? "\u2705" : run.status === "failed" ? "\u274C" : "\u26A0\uFE0F";
|
|
24425
24778
|
const passRate = run.total > 0 ? Math.round(run.passed / run.total * 100) : 0;
|
|
24426
|
-
const
|
|
24427
|
-
const
|
|
24779
|
+
const ordered = [...results].sort((a, b) => {
|
|
24780
|
+
const rank = (s) => s === "failed" ? 0 : s === "error" ? 1 : s === "flaky" ? 2 : s === "skipped" ? 3 : 4;
|
|
24781
|
+
return rank(a.status) - rank(b.status);
|
|
24782
|
+
});
|
|
24783
|
+
const MAX_ROWS = 20;
|
|
24784
|
+
const rows = ordered.slice(0, MAX_ROWS).map((r) => {
|
|
24785
|
+
const rowIcon = r.status === "passed" ? "\u2705" : r.status === "failed" ? "\u274C" : r.status === "error" ? "\u26A0\uFE0F" : r.status === "flaky" ? "\uD83D\uDFE1" : "\u23ED\uFE0F";
|
|
24428
24786
|
const dur = r.durationMs > 0 ? `${(r.durationMs / 1000).toFixed(1)}s` : "\u2014";
|
|
24429
|
-
const
|
|
24430
|
-
|
|
24787
|
+
const scenario = (() => {
|
|
24788
|
+
try {
|
|
24789
|
+
return getScenario(r.scenarioId);
|
|
24790
|
+
} catch {
|
|
24791
|
+
return null;
|
|
24792
|
+
}
|
|
24793
|
+
})();
|
|
24794
|
+
const name = scenario ? scenario.name : r.scenarioId.slice(0, 8);
|
|
24795
|
+
const safeName = name.replace(/\|/g, "\\|");
|
|
24796
|
+
const errSource = r.error ?? r.reasoning ?? "";
|
|
24797
|
+
const err = errSource ? ` ${errSource.replace(/\s+/g, " ").slice(0, 140).replace(/\|/g, "\\|")}` : "";
|
|
24798
|
+
return `| ${rowIcon} | ${safeName} | ${r.status} | ${dur} |${err} |`;
|
|
24431
24799
|
}).join(`
|
|
24432
24800
|
`);
|
|
24433
|
-
const truncated = results.length >
|
|
24434
|
-
_...and ${results.length -
|
|
24801
|
+
const truncated = results.length > MAX_ROWS ? `
|
|
24802
|
+
_...and ${results.length - MAX_ROWS} more_` : "";
|
|
24435
24803
|
const dashLink = dashboardUrl ? `
|
|
24436
24804
|
|
|
24437
24805
|
[View full report \u2192](${dashboardUrl}/runs/${run.id})` : "";
|
|
24806
|
+
const totalCostCents = results.reduce((sum, r) => sum + (r.costCents ?? 0), 0);
|
|
24807
|
+
const costStr = totalCostCents > 0 ? ` \xB7 $${(totalCostCents / 100).toFixed(4)}` : "";
|
|
24808
|
+
const headerRow = `| | Scenario | Status | Duration | Details |
|
|
24809
|
+
|---|---|---|---|---|`;
|
|
24810
|
+
const body = results.length > 0 ? `${headerRow}
|
|
24811
|
+
${rows}${truncated}` : `_No scenarios ran. Use \`testers add\` to create scenarios or run with a URL to auto-generate them._`;
|
|
24438
24812
|
return `## ${icon} AI QA Tests \u2014 ${run.status.toUpperCase()}
|
|
24439
24813
|
|
|
24440
|
-
**${run.passed}/${run.total} passed** (${passRate}%) \xB7 URL: \`${run.url}
|
|
24814
|
+
**${run.passed}/${run.total} passed** (${passRate}%) \xB7 URL: \`${run.url}\`${costStr}
|
|
24441
24815
|
|
|
24442
|
-
|
|
24443
|
-
|---|---|---|
|
|
24444
|
-
${rows}
|
|
24445
|
-
${truncated}${dashLink}
|
|
24816
|
+
${body}${dashLink}
|
|
24446
24817
|
|
|
24447
24818
|
_Generated by [@hasna/testers](https://www.npmjs.com/package/@hasna/testers)_`;
|
|
24448
24819
|
}
|
|
24820
|
+
function resolvePullRequestNumber(explicit) {
|
|
24821
|
+
if (explicit && Number.isFinite(explicit))
|
|
24822
|
+
return explicit;
|
|
24823
|
+
const fromEnv = process.env["GITHUB_PR_NUMBER"];
|
|
24824
|
+
if (fromEnv) {
|
|
24825
|
+
const parsed = parseInt(fromEnv, 10);
|
|
24826
|
+
if (Number.isFinite(parsed))
|
|
24827
|
+
return parsed;
|
|
24828
|
+
}
|
|
24829
|
+
const ref = process.env["GITHUB_REF"] ?? "";
|
|
24830
|
+
const match = ref.match(/refs\/pull\/(\d+)\//);
|
|
24831
|
+
if (match && match[1]) {
|
|
24832
|
+
const parsed = parseInt(match[1], 10);
|
|
24833
|
+
if (Number.isFinite(parsed))
|
|
24834
|
+
return parsed;
|
|
24835
|
+
}
|
|
24836
|
+
return null;
|
|
24837
|
+
}
|
|
24449
24838
|
async function postGitHubComment(run, results, options) {
|
|
24450
24839
|
const token = process.env["GITHUB_TOKEN"];
|
|
24451
24840
|
if (!token)
|
|
24452
24841
|
return false;
|
|
24453
|
-
|
|
24454
|
-
if (
|
|
24455
|
-
prNumber = parseInt(process.env["GITHUB_PR_NUMBER"], 10);
|
|
24456
|
-
}
|
|
24457
|
-
if (!prNumber) {
|
|
24458
|
-
const ref = process.env["GITHUB_REF"] ?? "";
|
|
24459
|
-
const match = ref.match(/refs\/pull\/(\d+)\//);
|
|
24460
|
-
if (match)
|
|
24461
|
-
prNumber = parseInt(match[1], 10);
|
|
24462
|
-
}
|
|
24463
|
-
if (!prNumber)
|
|
24842
|
+
const prNumber = resolvePullRequestNumber(options?.prNumber);
|
|
24843
|
+
if (prNumber === null)
|
|
24464
24844
|
return false;
|
|
24465
24845
|
const repo = process.env["GITHUB_REPOSITORY"];
|
|
24466
24846
|
if (!repo)
|
|
@@ -24483,6 +24863,235 @@ async function postGitHubComment(run, results, options) {
|
|
|
24483
24863
|
return false;
|
|
24484
24864
|
}
|
|
24485
24865
|
}
|
|
24866
|
+
var init_ci = __esm(() => {
|
|
24867
|
+
init_scenarios();
|
|
24868
|
+
});
|
|
24869
|
+
|
|
24870
|
+
// src/lib/crawl-and-generate.ts
|
|
24871
|
+
var exports_crawl_and_generate = {};
|
|
24872
|
+
__export(exports_crawl_and_generate, {
|
|
24873
|
+
crawlAndGenerate: () => crawlAndGenerate
|
|
24874
|
+
});
|
|
24875
|
+
function shouldSkip(href, rootOrigin, skipPaths) {
|
|
24876
|
+
try {
|
|
24877
|
+
const u = new URL(href);
|
|
24878
|
+
if (u.origin !== rootOrigin)
|
|
24879
|
+
return true;
|
|
24880
|
+
const path = u.pathname;
|
|
24881
|
+
const allSkip = [...DEFAULT_SKIP_PATTERNS, ...skipPaths];
|
|
24882
|
+
return allSkip.some((p) => path.startsWith(p) || path.includes(p));
|
|
24883
|
+
} catch {
|
|
24884
|
+
return true;
|
|
24885
|
+
}
|
|
24886
|
+
}
|
|
24887
|
+
function normaliseUrl(href) {
|
|
24888
|
+
try {
|
|
24889
|
+
const u = new URL(href);
|
|
24890
|
+
return `${u.origin}${u.pathname}`;
|
|
24891
|
+
} catch {
|
|
24892
|
+
return href;
|
|
24893
|
+
}
|
|
24894
|
+
}
|
|
24895
|
+
async function getPageContext(browser, pageUrl, timeoutMs) {
|
|
24896
|
+
const page = await getPage(browser, {});
|
|
24897
|
+
try {
|
|
24898
|
+
await page.goto(pageUrl, { timeout: timeoutMs, waitUntil: "domcontentloaded" });
|
|
24899
|
+
await page.waitForTimeout(800).catch(() => {});
|
|
24900
|
+
const [title, html, links, screenshot] = await Promise.all([
|
|
24901
|
+
page.title().catch(() => ""),
|
|
24902
|
+
page.evaluate(() => {
|
|
24903
|
+
const body = document.body;
|
|
24904
|
+
if (!body)
|
|
24905
|
+
return "";
|
|
24906
|
+
const clone = body.cloneNode(true);
|
|
24907
|
+
clone.querySelectorAll("script,style,svg,noscript,iframe").forEach((el) => el.remove());
|
|
24908
|
+
return clone.innerText?.slice(0, 3000) ?? clone.textContent?.slice(0, 3000) ?? "";
|
|
24909
|
+
}).catch(() => ""),
|
|
24910
|
+
page.evaluate((origin) => Array.from(document.querySelectorAll("a[href]")).map((a) => a.href).filter((h) => {
|
|
24911
|
+
try {
|
|
24912
|
+
return new URL(h).origin === origin;
|
|
24913
|
+
} catch {
|
|
24914
|
+
return false;
|
|
24915
|
+
}
|
|
24916
|
+
}), new URL(pageUrl).origin).catch(() => []),
|
|
24917
|
+
page.screenshot({ fullPage: false }).catch(() => null)
|
|
24918
|
+
]);
|
|
24919
|
+
return { title, path: new URL(pageUrl).pathname, html, screenshot, links };
|
|
24920
|
+
} finally {
|
|
24921
|
+
await page.close().catch(() => {});
|
|
24922
|
+
}
|
|
24923
|
+
}
|
|
24924
|
+
async function generateScenariosForPage(client, model, pageContext, baseUrl, count) {
|
|
24925
|
+
const Anthropic4 = (await import("@anthropic-ai/sdk")).default;
|
|
24926
|
+
const anthropicClient = client;
|
|
24927
|
+
const pageDesc = [
|
|
24928
|
+
`URL: ${baseUrl.replace(/\/$/, "")}${pageContext.path}`,
|
|
24929
|
+
`Title: ${pageContext.title || pageContext.path}`,
|
|
24930
|
+
pageContext.html ? `
|
|
24931
|
+
Page content (text):
|
|
24932
|
+
${pageContext.html.slice(0, 2000)}` : ""
|
|
24933
|
+
].filter(Boolean).join(`
|
|
24934
|
+
`);
|
|
24935
|
+
const prompt = `You are a QA engineer. Analyze this web page and write ${count} practical test scenarios.
|
|
24936
|
+
|
|
24937
|
+
${pageDesc}
|
|
24938
|
+
|
|
24939
|
+
Return ONLY a JSON array (no markdown, no explanation). Each scenario:
|
|
24940
|
+
{
|
|
24941
|
+
"name": "short action-oriented name (e.g. 'User can log in with valid credentials')",
|
|
24942
|
+
"description": "what this test verifies",
|
|
24943
|
+
"steps": ["step 1", "step 2", "step 3"],
|
|
24944
|
+
"tags": ["tag1"],
|
|
24945
|
+
"priority": "low|medium|high|critical"
|
|
24946
|
+
}
|
|
24947
|
+
|
|
24948
|
+
Rules:
|
|
24949
|
+
- Focus on user flows, not implementation details
|
|
24950
|
+
- Steps should be plain English instructions the browser agent can follow
|
|
24951
|
+
- Vary priorities: 1 critical/high per page for the main flow, rest medium/low
|
|
24952
|
+
- Keep steps concise (max 8 per scenario)
|
|
24953
|
+
- Tags should reflect the page area (e.g. "auth", "dashboard", "settings", "checkout")`;
|
|
24954
|
+
const contentParts = [
|
|
24955
|
+
...pageContext.screenshot ? [{
|
|
24956
|
+
type: "image",
|
|
24957
|
+
source: {
|
|
24958
|
+
type: "base64",
|
|
24959
|
+
media_type: "image/png",
|
|
24960
|
+
data: pageContext.screenshot.toString("base64")
|
|
24961
|
+
}
|
|
24962
|
+
}] : [],
|
|
24963
|
+
{ type: "text", text: prompt }
|
|
24964
|
+
];
|
|
24965
|
+
const messages = [{ role: "user", content: contentParts }];
|
|
24966
|
+
try {
|
|
24967
|
+
const response = await anthropicClient.messages.create({
|
|
24968
|
+
model,
|
|
24969
|
+
max_tokens: 2048,
|
|
24970
|
+
messages
|
|
24971
|
+
});
|
|
24972
|
+
const text = response.content.filter((b) => b.type === "text").map((b) => b.text).join("");
|
|
24973
|
+
const match = text.match(/\[[\s\S]*\]/);
|
|
24974
|
+
if (!match)
|
|
24975
|
+
return [];
|
|
24976
|
+
const parsed = JSON.parse(match[0]);
|
|
24977
|
+
return parsed.map((s) => ({
|
|
24978
|
+
name: s.name ?? "Untitled scenario",
|
|
24979
|
+
description: s.description ?? "",
|
|
24980
|
+
steps: s.steps ?? [],
|
|
24981
|
+
tags: s.tags ?? [],
|
|
24982
|
+
priority: s.priority ?? "medium"
|
|
24983
|
+
}));
|
|
24984
|
+
} catch {
|
|
24985
|
+
return [];
|
|
24986
|
+
}
|
|
24987
|
+
}
|
|
24988
|
+
async function crawlAndGenerate(options) {
|
|
24989
|
+
const {
|
|
24990
|
+
url,
|
|
24991
|
+
projectId,
|
|
24992
|
+
maxPages = 20,
|
|
24993
|
+
scenariosPerPage = 3,
|
|
24994
|
+
headed = false,
|
|
24995
|
+
skipPaths = [],
|
|
24996
|
+
tags: extraTags = []
|
|
24997
|
+
} = options;
|
|
24998
|
+
const config = loadConfig();
|
|
24999
|
+
const model = resolveModel(options.model ?? config.defaultModel ?? "thorough");
|
|
25000
|
+
const client = createClient(options.apiKey ?? config.anthropicApiKey);
|
|
25001
|
+
const rootOrigin = new URL(url).origin;
|
|
25002
|
+
const visited = new Set;
|
|
25003
|
+
const queue = [url];
|
|
25004
|
+
const pageContexts = [];
|
|
25005
|
+
const skipped = [];
|
|
25006
|
+
const browser = await launchBrowser({ headless: !headed });
|
|
25007
|
+
try {
|
|
25008
|
+
while (queue.length > 0 && visited.size < maxPages) {
|
|
25009
|
+
const pageUrl = queue.shift();
|
|
25010
|
+
const norm = normaliseUrl(pageUrl);
|
|
25011
|
+
if (visited.has(norm))
|
|
25012
|
+
continue;
|
|
25013
|
+
if (shouldSkip(pageUrl, rootOrigin, skipPaths)) {
|
|
25014
|
+
skipped.push(pageUrl);
|
|
25015
|
+
continue;
|
|
25016
|
+
}
|
|
25017
|
+
visited.add(norm);
|
|
25018
|
+
try {
|
|
25019
|
+
const ctx = await getPageContext(browser, pageUrl, 15000);
|
|
25020
|
+
pageContexts.push(ctx);
|
|
25021
|
+
for (const link of ctx.links) {
|
|
25022
|
+
const normLink = normaliseUrl(link);
|
|
25023
|
+
if (!visited.has(normLink) && !shouldSkip(link, rootOrigin, skipPaths)) {
|
|
25024
|
+
queue.push(link);
|
|
25025
|
+
}
|
|
25026
|
+
}
|
|
25027
|
+
} catch {
|
|
25028
|
+
skipped.push(pageUrl);
|
|
25029
|
+
}
|
|
25030
|
+
}
|
|
25031
|
+
} finally {
|
|
25032
|
+
await closeBrowser(browser).catch(() => {});
|
|
25033
|
+
}
|
|
25034
|
+
const pages = [];
|
|
25035
|
+
let totalCreated = 0;
|
|
25036
|
+
for (const ctx of pageContexts) {
|
|
25037
|
+
const generated = await generateScenariosForPage(client, model, ctx, url, scenariosPerPage);
|
|
25038
|
+
const createdScenarios = [];
|
|
25039
|
+
for (const s of generated) {
|
|
25040
|
+
try {
|
|
25041
|
+
const priority = ["low", "medium", "high", "critical"].includes(s.priority) ? s.priority : "medium";
|
|
25042
|
+
const scenario = createScenario({
|
|
25043
|
+
name: s.name,
|
|
25044
|
+
description: s.description,
|
|
25045
|
+
steps: s.steps,
|
|
25046
|
+
tags: [...s.tags ?? [], ...extraTags, "generated"],
|
|
25047
|
+
priority,
|
|
25048
|
+
targetPath: ctx.path,
|
|
25049
|
+
projectId
|
|
25050
|
+
});
|
|
25051
|
+
createdScenarios.push({ id: scenario.id, shortId: scenario.shortId, name: scenario.name });
|
|
25052
|
+
totalCreated++;
|
|
25053
|
+
} catch {}
|
|
25054
|
+
}
|
|
25055
|
+
if (createdScenarios.length > 0) {
|
|
25056
|
+
pages.push({
|
|
25057
|
+
path: ctx.path,
|
|
25058
|
+
title: ctx.title,
|
|
25059
|
+
scenariosCreated: createdScenarios.length,
|
|
25060
|
+
scenarios: createdScenarios
|
|
25061
|
+
});
|
|
25062
|
+
}
|
|
25063
|
+
}
|
|
25064
|
+
return {
|
|
25065
|
+
projectId: projectId ?? null,
|
|
25066
|
+
url,
|
|
25067
|
+
pagesDiscovered: pageContexts.length,
|
|
25068
|
+
pagesGenerated: pages.length,
|
|
25069
|
+
totalScenariosCreated: totalCreated,
|
|
25070
|
+
pages,
|
|
25071
|
+
skipped
|
|
25072
|
+
};
|
|
25073
|
+
}
|
|
25074
|
+
var DEFAULT_SKIP_PATTERNS;
|
|
25075
|
+
var init_crawl_and_generate = __esm(() => {
|
|
25076
|
+
init_browser();
|
|
25077
|
+
init_scenarios();
|
|
25078
|
+
init_ai_client();
|
|
25079
|
+
init_config2();
|
|
25080
|
+
init_ai_client();
|
|
25081
|
+
DEFAULT_SKIP_PATTERNS = [
|
|
25082
|
+
"/logout",
|
|
25083
|
+
"/sign-out",
|
|
25084
|
+
"/signout",
|
|
25085
|
+
"/static/",
|
|
25086
|
+
"/assets/",
|
|
25087
|
+
"/_next/",
|
|
25088
|
+
"/__/",
|
|
25089
|
+
"/favicon",
|
|
25090
|
+
"/robots.txt",
|
|
25091
|
+
"/sitemap",
|
|
25092
|
+
"#"
|
|
25093
|
+
];
|
|
25094
|
+
});
|
|
24486
25095
|
|
|
24487
25096
|
// src/lib/affected.ts
|
|
24488
25097
|
var exports_affected = {};
|
|
@@ -25044,10 +25653,14 @@ var init_generator = __esm(() => {
|
|
|
25044
25653
|
// src/lib/recorder.ts
|
|
25045
25654
|
var exports_recorder = {};
|
|
25046
25655
|
__export(exports_recorder, {
|
|
25656
|
+
replayAuthState: () => replayAuthState,
|
|
25047
25657
|
recordSession: () => recordSession,
|
|
25658
|
+
recordAuthFlow: () => recordAuthFlow,
|
|
25048
25659
|
recordAndSave: () => recordAndSave,
|
|
25660
|
+
authStateToScenarioMetadata: () => authStateToScenarioMetadata,
|
|
25049
25661
|
actionsToScenarioInput: () => actionsToScenarioInput
|
|
25050
25662
|
});
|
|
25663
|
+
import { chromium as chromium2 } from "playwright";
|
|
25051
25664
|
import { startRecording, stopRecording } from "@hasna/browser";
|
|
25052
25665
|
import { launchPlaywright as launchPlaywright2 } from "@hasna/browser";
|
|
25053
25666
|
async function recordSession(url, options) {
|
|
@@ -25212,6 +25825,99 @@ async function recordAndSave(url, name, projectId) {
|
|
|
25212
25825
|
const scenario = createScenario(input);
|
|
25213
25826
|
return { recording, scenario };
|
|
25214
25827
|
}
|
|
25828
|
+
async function recordAuthFlow(loginUrl, options) {
|
|
25829
|
+
const browser = await chromium2.launch({ headless: false });
|
|
25830
|
+
const context = await browser.newContext({ viewport: { width: 1280, height: 720 } });
|
|
25831
|
+
const page = await context.newPage();
|
|
25832
|
+
const emailSelector = options.emailSelector ?? 'input[name="email"], input[type="email"], #email';
|
|
25833
|
+
const passwordSelector = options.passwordSelector ?? 'input[name="password"], input[type="password"], #password';
|
|
25834
|
+
const submitSelector = options.submitSelector ?? 'button[type="submit"], input[type="submit"]';
|
|
25835
|
+
try {
|
|
25836
|
+
await page.goto(loginUrl, { waitUntil: "domcontentloaded", timeout: options.timeoutMs ?? 30000 });
|
|
25837
|
+
await page.fill(emailSelector, options.email);
|
|
25838
|
+
await page.fill(passwordSelector, options.password);
|
|
25839
|
+
await page.click(submitSelector);
|
|
25840
|
+
if (options.waitForUrl) {
|
|
25841
|
+
await page.waitForURL(options.waitForUrl, { timeout: options.timeoutMs ?? 30000 });
|
|
25842
|
+
} else {
|
|
25843
|
+
await page.waitForLoadState("networkidle", { timeout: options.timeoutMs ?? 30000 });
|
|
25844
|
+
}
|
|
25845
|
+
const cookies = await context.cookies();
|
|
25846
|
+
const formattedCookies = cookies.map((c) => ({
|
|
25847
|
+
name: c.name,
|
|
25848
|
+
value: c.value,
|
|
25849
|
+
domain: c.domain || "",
|
|
25850
|
+
path: c.path || "/"
|
|
25851
|
+
}));
|
|
25852
|
+
const frames = page.frames();
|
|
25853
|
+
const localStorageEntries = [];
|
|
25854
|
+
for (const frame of frames) {
|
|
25855
|
+
try {
|
|
25856
|
+
const origin = frame.url();
|
|
25857
|
+
if (origin && origin !== "about:blank") {
|
|
25858
|
+
const entries = await frame.evaluate(() => {
|
|
25859
|
+
const items = [];
|
|
25860
|
+
for (let i = 0;i < localStorage.length; i++) {
|
|
25861
|
+
const key = localStorage.key(i);
|
|
25862
|
+
if (key)
|
|
25863
|
+
items.push({ name: key, value: localStorage.getItem(key) || "" });
|
|
25864
|
+
}
|
|
25865
|
+
return items;
|
|
25866
|
+
});
|
|
25867
|
+
if (entries.length > 0) {
|
|
25868
|
+
localStorageEntries.push({ origin, entries });
|
|
25869
|
+
}
|
|
25870
|
+
}
|
|
25871
|
+
} catch {}
|
|
25872
|
+
}
|
|
25873
|
+
return {
|
|
25874
|
+
cookies: formattedCookies,
|
|
25875
|
+
localStorage: localStorageEntries,
|
|
25876
|
+
loginUrl,
|
|
25877
|
+
recordedAt: new Date().toISOString()
|
|
25878
|
+
};
|
|
25879
|
+
} finally {
|
|
25880
|
+
await browser.close();
|
|
25881
|
+
}
|
|
25882
|
+
}
|
|
25883
|
+
async function replayAuthState(context, authState) {
|
|
25884
|
+
for (const cookie of authState.cookies) {
|
|
25885
|
+
try {
|
|
25886
|
+
await context.addCookies([{
|
|
25887
|
+
name: cookie.name,
|
|
25888
|
+
value: cookie.value,
|
|
25889
|
+
domain: cookie.domain,
|
|
25890
|
+
path: cookie.path,
|
|
25891
|
+
expires: -1
|
|
25892
|
+
}]);
|
|
25893
|
+
} catch {}
|
|
25894
|
+
}
|
|
25895
|
+
const page = await context.newPage();
|
|
25896
|
+
const origin = new URL(authState.loginUrl).origin;
|
|
25897
|
+
await page.goto(`${origin}/about:blank`, { waitUntil: "domcontentloaded" }).catch(() => {});
|
|
25898
|
+
for (const entry of authState.localStorage) {
|
|
25899
|
+
try {
|
|
25900
|
+
await page.evaluate((items) => {
|
|
25901
|
+
for (const item of items) {
|
|
25902
|
+
localStorage.setItem(item.name, item.value);
|
|
25903
|
+
}
|
|
25904
|
+
}, entry.entries);
|
|
25905
|
+
} catch {}
|
|
25906
|
+
}
|
|
25907
|
+
await page.close();
|
|
25908
|
+
}
|
|
25909
|
+
function authStateToScenarioMetadata(authState, name, projectId) {
|
|
25910
|
+
return createScenario({
|
|
25911
|
+
name,
|
|
25912
|
+
description: `Authenticated test scenario from recorded auth state at ${authState.loginUrl}`,
|
|
25913
|
+
steps: [`Navigate to authenticated session`],
|
|
25914
|
+
tags: ["auth", "recorded"],
|
|
25915
|
+
requiresAuth: true,
|
|
25916
|
+
authConfig: { loginPath: new URL(authState.loginUrl).pathname },
|
|
25917
|
+
metadata: { authState: JSON.parse(JSON.stringify(authState)) },
|
|
25918
|
+
projectId
|
|
25919
|
+
});
|
|
25920
|
+
}
|
|
25215
25921
|
var init_recorder = __esm(() => {
|
|
25216
25922
|
init_scenarios();
|
|
25217
25923
|
});
|
|
@@ -25683,7 +26389,7 @@ async function scanBrokenLinks(options) {
|
|
|
25683
26389
|
try {
|
|
25684
26390
|
while (queue.length > 0 && visited.size < maxPages) {
|
|
25685
26391
|
const { pageUrl, sourceUrl } = queue.shift();
|
|
25686
|
-
const normalised =
|
|
26392
|
+
const normalised = normaliseUrl2(pageUrl);
|
|
25687
26393
|
if (visited.has(normalised))
|
|
25688
26394
|
continue;
|
|
25689
26395
|
visited.add(normalised);
|
|
@@ -25748,7 +26454,7 @@ async function scanBrokenLinks(options) {
|
|
|
25748
26454
|
issues
|
|
25749
26455
|
};
|
|
25750
26456
|
}
|
|
25751
|
-
function
|
|
26457
|
+
function normaliseUrl2(rawUrl) {
|
|
25752
26458
|
try {
|
|
25753
26459
|
const u = new URL(rawUrl);
|
|
25754
26460
|
return `${u.origin}${u.pathname}`;
|
|
@@ -26175,6 +26881,16 @@ async function runHealthScan(options) {
|
|
|
26175
26881
|
});
|
|
26176
26882
|
results.push(piiResult);
|
|
26177
26883
|
}
|
|
26884
|
+
if (scanners.includes("a11y")) {
|
|
26885
|
+
const a11yResult = await scanA11y({
|
|
26886
|
+
url,
|
|
26887
|
+
pages,
|
|
26888
|
+
wcagLevel: options.wcagLevel ?? "AA",
|
|
26889
|
+
headed,
|
|
26890
|
+
timeoutMs
|
|
26891
|
+
});
|
|
26892
|
+
results.push(a11yResult);
|
|
26893
|
+
}
|
|
26178
26894
|
const allIssues = results.flatMap((r) => r.issues);
|
|
26179
26895
|
let newCount = 0;
|
|
26180
26896
|
let regressedCount = 0;
|
|
@@ -27119,7 +27835,7 @@ import chalk6 from "chalk";
|
|
|
27119
27835
|
// package.json
|
|
27120
27836
|
var package_default = {
|
|
27121
27837
|
name: "@hasna/testers",
|
|
27122
|
-
version: "0.0.
|
|
27838
|
+
version: "0.0.29",
|
|
27123
27839
|
description: "AI-powered QA testing CLI \u2014 spawns cheap AI agents to test web apps with headless browsers",
|
|
27124
27840
|
type: "module",
|
|
27125
27841
|
main: "dist/index.js",
|
|
@@ -27214,13 +27930,13 @@ import { render, Box, Text, useInput, useApp } from "ink";
|
|
|
27214
27930
|
import React, { useState } from "react";
|
|
27215
27931
|
import { readFileSync as readFileSync8, readdirSync as readdirSync3, writeFileSync as writeFileSync4 } from "fs";
|
|
27216
27932
|
import { createInterface } from "readline";
|
|
27217
|
-
import { join as
|
|
27933
|
+
import { join as join16, resolve } from "path";
|
|
27218
27934
|
|
|
27219
27935
|
// src/lib/init.ts
|
|
27220
27936
|
init_paths();
|
|
27221
27937
|
init_scenarios();
|
|
27222
|
-
import { existsSync as existsSync11, readFileSync as readFileSync3, writeFileSync as writeFileSync3, mkdirSync as
|
|
27223
|
-
import { join as
|
|
27938
|
+
import { existsSync as existsSync11, readFileSync as readFileSync3, writeFileSync as writeFileSync3, mkdirSync as mkdirSync9 } from "fs";
|
|
27939
|
+
import { join as join14, basename } from "path";
|
|
27224
27940
|
|
|
27225
27941
|
// src/db/projects.ts
|
|
27226
27942
|
init_types();
|
|
@@ -27258,7 +27974,7 @@ function ensureProject(name, path) {
|
|
|
27258
27974
|
|
|
27259
27975
|
// src/lib/init.ts
|
|
27260
27976
|
function detectFramework(dir) {
|
|
27261
|
-
const pkgPath =
|
|
27977
|
+
const pkgPath = join14(dir, "package.json");
|
|
27262
27978
|
if (!existsSync11(pkgPath))
|
|
27263
27979
|
return null;
|
|
27264
27980
|
let pkg;
|
|
@@ -27486,9 +28202,9 @@ function initProject(options) {
|
|
|
27486
28202
|
}
|
|
27487
28203
|
}).filter((s) => s !== null);
|
|
27488
28204
|
const configDir = getTestersDir();
|
|
27489
|
-
const configPath =
|
|
28205
|
+
const configPath = join14(configDir, "config.json");
|
|
27490
28206
|
if (!existsSync11(configDir)) {
|
|
27491
|
-
|
|
28207
|
+
mkdirSync9(configDir, { recursive: true });
|
|
27492
28208
|
}
|
|
27493
28209
|
let config = {};
|
|
27494
28210
|
if (existsSync11(configPath)) {
|
|
@@ -27880,8 +28596,8 @@ init_results();
|
|
|
27880
28596
|
init_runs();
|
|
27881
28597
|
init_scenarios();
|
|
27882
28598
|
init_database();
|
|
27883
|
-
import { readFileSync as readFileSync4, existsSync as existsSync12, mkdirSync as
|
|
27884
|
-
import { join as
|
|
28599
|
+
import { readFileSync as readFileSync4, existsSync as existsSync12, mkdirSync as mkdirSync10 } from "fs";
|
|
28600
|
+
import { join as join15, dirname as dirname3 } from "path";
|
|
27885
28601
|
import chalk4 from "chalk";
|
|
27886
28602
|
var DEFAULT_THRESHOLD = 0.1;
|
|
27887
28603
|
function setBaseline(runId) {
|
|
@@ -27936,8 +28652,8 @@ async function compareImages(image1Path, image2Path, options) {
|
|
|
27936
28652
|
let diffImagePath;
|
|
27937
28653
|
if (options?.saveDiff) {
|
|
27938
28654
|
const dir = options.diffDir ?? dirname3(image2Path);
|
|
27939
|
-
|
|
27940
|
-
diffImagePath =
|
|
28655
|
+
mkdirSync10(dir, { recursive: true });
|
|
28656
|
+
diffImagePath = join15(dir, `diff-${Date.now()}.png`);
|
|
27941
28657
|
await sharp.default(diffBuffer, { raw: { width: w, height: h, channels } }).png().toFile(diffImagePath);
|
|
27942
28658
|
}
|
|
27943
28659
|
return { diffPercent, diffPixels: changedPixels, totalPixels, diffImagePath };
|
|
@@ -28535,6 +29251,18 @@ var SCENARIO_TEMPLATES = {
|
|
|
28535
29251
|
a11y: [
|
|
28536
29252
|
{ name: "Keyboard navigation", description: "Navigate the page using only keyboard (Tab, Enter, Escape). Verify all interactive elements are reachable and focusable.", tags: ["a11y", "keyboard"], priority: "high", steps: ["Press Tab to move through elements", "Verify focus indicators are visible", "Press Enter on buttons/links", "Verify actions trigger correctly", "Press Escape to close modals/dropdowns"] },
|
|
28537
29253
|
{ name: "Image alt text", description: "Check that all images have meaningful alt text attributes.", tags: ["a11y"], priority: "medium", steps: ["Find all images on the page", "Check each image has an alt attribute", "Verify alt text is descriptive, not empty or generic"] }
|
|
29254
|
+
],
|
|
29255
|
+
checkout: [
|
|
29256
|
+
{ name: "Add item to cart", description: "Navigate to a product page, add an item to cart, and verify cart count increments.", tags: ["checkout", "cart"], priority: "critical", steps: ["Navigate to product detail page", "Click add to cart button", "Verify cart count badge updates", "Verify success notification appears"] },
|
|
29257
|
+
{ name: "Cart page shows added items", description: "Navigate to the cart page and verify previously added items are listed with correct prices and quantities.", tags: ["checkout", "cart"], priority: "high", steps: ["Navigate to cart page", "Verify all added items are listed", "Verify prices and quantities are correct", "Verify subtotal is calculated"] },
|
|
29258
|
+
{ name: "Checkout flow completion", description: "Complete the full checkout flow: cart -> shipping -> payment -> confirmation. Verify order is placed successfully.", tags: ["checkout", "smoke"], priority: "critical", steps: ["Navigate to cart", "Click proceed to checkout", "Fill shipping address", "Select shipping method", "Fill payment details", "Review order summary", "Place order", "Verify order confirmation page"] },
|
|
29259
|
+
{ name: "Apply coupon/discount code", description: "Enter a valid coupon code on the cart or checkout page and verify discount is applied to the total.", tags: ["checkout", "discount"], priority: "high", steps: ["Navigate to cart or checkout", "Enter coupon code", "Click apply", "Verify discount amount is shown", "Verify total is updated"] }
|
|
29260
|
+
],
|
|
29261
|
+
search: [
|
|
29262
|
+
{ name: "Search returns relevant results", description: "Enter a known search term and verify relevant results appear. Check that results match the query intent.", tags: ["search"], priority: "high", steps: ["Navigate to search page or focus search bar", "Enter known search term", "Submit search", "Verify results appear", "Verify result titles contain search term"] },
|
|
29263
|
+
{ name: "Empty search handling", description: "Submit an empty search and verify the system handles it gracefully (show all results or a prompt, no errors).", tags: ["search", "validation"], priority: "medium", steps: ["Focus search bar", "Submit empty search", "Verify no crash or error state", "Verify fallback behavior (all results or prompt)"] },
|
|
29264
|
+
{ name: "No results handling", description: "Search for a term that returns no results and verify appropriate 'no results found' messaging is displayed.", tags: ["search"], priority: "medium", steps: ["Navigate to search", "Enter nonsense term like 'xyzqwrtp'", "Submit search", "Verify no results found message", "Verify no error states"] },
|
|
29265
|
+
{ name: "Search filters work", description: "Perform a search, apply a filter (category, date, price range), and verify results are narrowed down.", tags: ["search", "filter"], priority: "high", steps: ["Perform initial search", "Note initial result count", "Apply a filter", "Verify result count decreased", "Verify results match filter criteria"] }
|
|
28538
29266
|
]
|
|
28539
29267
|
};
|
|
28540
29268
|
function getTemplate(name) {
|
|
@@ -28653,6 +29381,10 @@ function getDefaultEnvironment() {
|
|
|
28653
29381
|
const row = db2.query("SELECT * FROM environments WHERE is_default = 1 LIMIT 1").get();
|
|
28654
29382
|
return row ? fromRow3(row) : null;
|
|
28655
29383
|
}
|
|
29384
|
+
|
|
29385
|
+
// src/cli/index.tsx
|
|
29386
|
+
init_ci();
|
|
29387
|
+
|
|
28656
29388
|
// src/lib/assertions.ts
|
|
28657
29389
|
function parseAssertionString(str) {
|
|
28658
29390
|
const trimmed = str.trim();
|
|
@@ -28709,12 +29441,44 @@ function parseAssertionString(str) {
|
|
|
28709
29441
|
}
|
|
28710
29442
|
throw new Error(`Unknown selector action: "${action}". Expected "visible" or "not-visible"`);
|
|
28711
29443
|
}
|
|
29444
|
+
if (trimmed.startsWith("cookie:exists:")) {
|
|
29445
|
+
const name = trimmed.slice("cookie:exists:".length);
|
|
29446
|
+
return { type: "cookie_exists", expected: name, description: `Cookie "${name}" exists` };
|
|
29447
|
+
}
|
|
29448
|
+
if (trimmed.startsWith("cookie:not-exists:")) {
|
|
29449
|
+
const name = trimmed.slice("cookie:not-exists:".length);
|
|
29450
|
+
return { type: "cookie_not_exists", expected: name, description: `Cookie "${name}" does not exist` };
|
|
29451
|
+
}
|
|
29452
|
+
if (trimmed.startsWith("cookie:value:")) {
|
|
29453
|
+
const valueStr = trimmed.slice("cookie:value:".length);
|
|
29454
|
+
return { type: "cookie_value", expected: valueStr, description: `Cookie value is "${valueStr}"` };
|
|
29455
|
+
}
|
|
29456
|
+
if (trimmed.startsWith("local:exists:")) {
|
|
29457
|
+
const key = trimmed.slice("local:exists:".length);
|
|
29458
|
+
return { type: "local_storage_exists", expected: key, description: `LocalStorage key "${key}" exists` };
|
|
29459
|
+
}
|
|
29460
|
+
if (trimmed.startsWith("local:not-exists:")) {
|
|
29461
|
+
const key = trimmed.slice("local:not-exists:".length);
|
|
29462
|
+
return { type: "local_storage_not_exists", expected: key, description: `LocalStorage key "${key}" does not exist` };
|
|
29463
|
+
}
|
|
29464
|
+
if (trimmed.startsWith("local:value:")) {
|
|
29465
|
+
const valueStr = trimmed.slice("local:value:".length);
|
|
29466
|
+
return { type: "local_storage_value", expected: valueStr, description: `LocalStorage value is "${valueStr}"` };
|
|
29467
|
+
}
|
|
29468
|
+
if (trimmed.startsWith("session:value:")) {
|
|
29469
|
+
const valueStr = trimmed.slice("session:value:".length);
|
|
29470
|
+
return { type: "session_storage_value", expected: valueStr, description: `SessionStorage value is "${valueStr}"` };
|
|
29471
|
+
}
|
|
29472
|
+
if (trimmed.startsWith("session:not-exists:")) {
|
|
29473
|
+
const key = trimmed.slice("session:not-exists:".length);
|
|
29474
|
+
return { type: "session_storage_not_exists", expected: key, description: `SessionStorage key "${key}" does not exist` };
|
|
29475
|
+
}
|
|
28712
29476
|
throw new Error(`Cannot parse assertion: "${str}". See --help for assertion formats.`);
|
|
28713
29477
|
}
|
|
28714
29478
|
|
|
28715
29479
|
// src/cli/index.tsx
|
|
28716
29480
|
init_paths();
|
|
28717
|
-
import { existsSync as existsSync14, mkdirSync as
|
|
29481
|
+
import { existsSync as existsSync14, mkdirSync as mkdirSync11 } from "fs";
|
|
28718
29482
|
import { jsxDEV } from "react/jsx-dev-runtime";
|
|
28719
29483
|
var PRIORITIES = ["low", "medium", "high", "critical"];
|
|
28720
29484
|
function AddForm({ onComplete }) {
|
|
@@ -28974,7 +29738,7 @@ function logError(...args) {
|
|
|
28974
29738
|
}
|
|
28975
29739
|
program2.name("testers").version(package_default.version).description("AI-powered browser testing CLI").option("-q, --quiet", "Suppress all output", false).option("--no-color", "Disable color output");
|
|
28976
29740
|
var CONFIG_DIR5 = getTestersDir();
|
|
28977
|
-
var CONFIG_PATH3 =
|
|
29741
|
+
var CONFIG_PATH3 = join16(CONFIG_DIR5, "config.json");
|
|
28978
29742
|
function getActiveProject() {
|
|
28979
29743
|
try {
|
|
28980
29744
|
if (existsSync14(CONFIG_PATH3)) {
|
|
@@ -29242,7 +30006,7 @@ program2.command("remove <id>").alias("uninstall").description("Remove a scenari
|
|
|
29242
30006
|
program2.command("run [url] [description]").alias("test").description("Run test scenarios against a URL").option("-t, --tag <tag>", "Filter by tag (repeatable)", (val, acc) => {
|
|
29243
30007
|
acc.push(val);
|
|
29244
30008
|
return acc;
|
|
29245
|
-
}, []).option("-s, --scenario <id>", "Run specific scenario ID").option("-p, --priority <level>", "Filter by priority").option("--headed", "Run browser in headed mode", false).option("-m, --model <model>", "AI model to use").option("--parallel <n>", "Number of parallel browsers", "1").option("--json", "Output results as JSON", false).option("-o, --output <filepath>", "Write JSON results to file").option("--timeout <ms>", "Timeout in milliseconds").option("--from-todos", "Import scenarios from todos before running", false).option("--project <id>", "Project ID").option("-b, --background", "Start run in background and return immediately", false).option("--browser <engine>", "Browser engine: playwright (default), lightpanda (9x faster, no screenshots), or bun (native WKWebView, 11x faster, Bun canary required)", "playwright").option("--env <name>", "Use a named environment for the URL").option("--dry-run", "Print what would run without launching browser", false).option("--retry <n>", "Retry failed scenarios up to n times", "0").option("--samples <n>", "Run each scenario N times and report flakiness (pass rate)", "1").option("--flakiness-threshold <n>", "Pass rate threshold below which a scenario is marked flaky (0-1)", "0.95").option("--a11y [level]", "Run axe-core WCAG accessibility scan after each navigation (level: A, AA, AAA \u2014 default AA)").option("--self-heal", "Enable AI-powered selector repair when elements can't be found (requires judgeModel or ANTHROPIC_API_KEY)", false).option("--verbose", "Show per-step timing and full tool results", false).option("--watch-results", "When used with --background, poll and display live results table until run completes", false).option("--failed-only", "Only show failed/error scenarios in output (passed count shown as summary)", false).option("--smoke", "Run only smoke-tagged scenarios (fast validation suite, <2 min)", false).option("--minimal", "Fastest possible run: cheapest model, max parallelism, min turns (ideal for CI)", false).option("--github-comment", "Post pass/fail summary as a GitHub PR comment (requires GITHUB_TOKEN env var)", false).option("--pr <number>", "GitHub PR number (auto-detected from GITHUB_REF if not provided)").option("--persona <id>", "Override persona for this run (comma-separated IDs for divergence testing)").option("--max-cost <dollars>", "Hard budget cap in dollars \u2014 abort if estimated cost exceeds this (e.g. 0.50 for 50 cents)").option("--cache-max-age <seconds>", "Skip scenarios that passed at the same URL within this many seconds (0 = disabled)", "0").option("--diff", "Auto-detect changed files from git diff and run only relevant scenarios", false).action(async (urlArg, description, opts) => {
|
|
30009
|
+
}, []).option("-s, --scenario <id>", "Run specific scenario ID").option("-p, --priority <level>", "Filter by priority").option("--headed", "Run browser in headed mode", false).option("-m, --model <model>", "AI model to use").option("--parallel <n>", "Number of parallel browsers", "1").option("--json", "Output results as JSON", false).option("-o, --output <filepath>", "Write JSON results to file").option("--timeout <ms>", "Timeout in milliseconds").option("--from-todos", "Import scenarios from todos before running", false).option("--project <id>", "Project ID").option("-b, --background", "Start run in background and return immediately", false).option("--browser <engine>", "Browser engine: playwright (default), lightpanda (9x faster, no screenshots), or bun (native WKWebView, 11x faster, Bun canary required)", "playwright").option("--env <name>", "Use a named environment for the URL").option("--dry-run", "Print what would run without launching browser", false).option("--retry <n>", "Retry failed scenarios up to n times", "0").option("--samples <n>", "Run each scenario N times and report flakiness (pass rate)", "1").option("--flakiness-threshold <n>", "Pass rate threshold below which a scenario is marked flaky (0-1)", "0.95").option("--a11y [level]", "Run axe-core WCAG accessibility scan after each navigation (level: A, AA, AAA \u2014 default AA)").option("--self-heal", "Enable AI-powered selector repair when elements can't be found (requires judgeModel or ANTHROPIC_API_KEY)", false).option("--verbose", "Show per-step timing and full tool results", false).option("--watch-results", "When used with --background, poll and display live results table until run completes", false).option("--failed-only", "Only show failed/error scenarios in output (passed count shown as summary)", false).option("--smoke", "Run only smoke-tagged scenarios (fast validation suite, <2 min)", false).option("--minimal", "Fastest possible run: cheapest model, max parallelism, min turns (ideal for CI)", false).option("--github-comment", "Post pass/fail summary as a GitHub PR comment (requires GITHUB_TOKEN env var)", false).option("--pr <number>", "GitHub PR number (auto-detected from GITHUB_REF if not provided)").option("--persona <id>", "Override persona for this run (comma-separated IDs for divergence testing)").option("--max-cost <dollars>", "Hard budget cap in dollars \u2014 abort if estimated cost exceeds this (e.g. 0.50 for 50 cents)").option("--cache-max-age <seconds>", "Skip scenarios that passed at the same URL within this many seconds (0 = disabled)", "0").option("--diff", "Auto-detect changed files from git diff and run only relevant scenarios", false).option("--auto-generate", "If no scenarios exist, crawl the URL and generate scenarios automatically (enabled by default when a URL is given as the first arg)").option("--no-auto-generate", "Disable automatic scenario generation when no scenarios exist").option("--overall-timeout <ms>", "Hard overall timeout for the whole run in milliseconds (default 10 minutes)").option("-y, --yes", "Skip confirmation prompts (e.g. proceed past budget warnings)", false).action(async (urlArg, description, opts) => {
|
|
29246
30010
|
try {
|
|
29247
30011
|
const projectId = resolveProject(opts.project);
|
|
29248
30012
|
let url = urlArg;
|
|
@@ -29263,7 +30027,51 @@ program2.command("run [url] [description]").alias("test").description("Run test
|
|
|
29263
30027
|
}
|
|
29264
30028
|
if (!url) {
|
|
29265
30029
|
logError(chalk6.red("No URL provided. Pass a URL argument, use --env <name>, or set a default environment with 'testers env use <name>'."));
|
|
29266
|
-
process.exit(
|
|
30030
|
+
process.exit(2);
|
|
30031
|
+
}
|
|
30032
|
+
if (!opts.dryRun) {
|
|
30033
|
+
const hasAnthropic = Boolean(process.env["ANTHROPIC_API_KEY"]);
|
|
30034
|
+
const hasOpenAI = Boolean(process.env["OPENAI_API_KEY"]);
|
|
30035
|
+
const hasGoogle = Boolean(process.env["GOOGLE_API_KEY"]);
|
|
30036
|
+
const hasCerebras = Boolean(process.env["CEREBRAS_API_KEY"]);
|
|
30037
|
+
if (!hasAnthropic && !hasOpenAI && !hasGoogle && !hasCerebras) {
|
|
30038
|
+
logError(chalk6.red("No AI API key found. Set ANTHROPIC_API_KEY (recommended), or OPENAI_API_KEY / GOOGLE_API_KEY / CEREBRAS_API_KEY."));
|
|
30039
|
+
logError(chalk6.red("For GitHub Actions, add ANTHROPIC_API_KEY to your repo secrets and pass it via: env: ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}"));
|
|
30040
|
+
process.exit(2);
|
|
30041
|
+
}
|
|
30042
|
+
}
|
|
30043
|
+
if (!opts.dryRun && !opts.background) {
|
|
30044
|
+
const reachable = await (async () => {
|
|
30045
|
+
try {
|
|
30046
|
+
const ctrl = new AbortController;
|
|
30047
|
+
const t = setTimeout(() => ctrl.abort(), 1e4);
|
|
30048
|
+
let res;
|
|
30049
|
+
try {
|
|
30050
|
+
res = await fetch(url, { method: "HEAD", signal: ctrl.signal, redirect: "follow" });
|
|
30051
|
+
} catch {
|
|
30052
|
+
res = await fetch(url, { method: "GET", signal: ctrl.signal, redirect: "follow" });
|
|
30053
|
+
}
|
|
30054
|
+
clearTimeout(t);
|
|
30055
|
+
if (res.status >= 500)
|
|
30056
|
+
return { ok: false, reason: `HTTP ${res.status}` };
|
|
30057
|
+
return { ok: true };
|
|
30058
|
+
} catch (err) {
|
|
30059
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
30060
|
+
return { ok: false, reason: msg };
|
|
30061
|
+
}
|
|
30062
|
+
})();
|
|
30063
|
+
if (!reachable.ok) {
|
|
30064
|
+
logError(chalk6.red(`URL unreachable: ${url}${reachable.reason ? ` (${reachable.reason})` : ""}`));
|
|
30065
|
+
logError(chalk6.red("Check that your preview deployment is up and the URL is correct."));
|
|
30066
|
+
process.exit(2);
|
|
30067
|
+
}
|
|
30068
|
+
}
|
|
30069
|
+
const overallTimeoutMs = opts.overallTimeout ? parseInt(opts.overallTimeout, 10) : 10 * 60 * 1000;
|
|
30070
|
+
if (!opts.dryRun && !opts.background && overallTimeoutMs > 0) {
|
|
30071
|
+
setTimeout(() => {
|
|
30072
|
+
logError(chalk6.red(`Overall timeout reached (${Math.round(overallTimeoutMs / 1000)}s). Aborting.`));
|
|
30073
|
+
process.exit(2);
|
|
30074
|
+
}, overallTimeoutMs).unref();
|
|
29267
30075
|
}
|
|
29268
30076
|
if (!opts.dryRun && !opts.background) {
|
|
29269
30077
|
const budgetResult = checkBudget(0);
|
|
@@ -29485,13 +30293,15 @@ program2.command("run [url] [description]").alias("test").description("Run test
|
|
|
29485
30293
|
log(formatTerminal(run2, results2, { failedOnly: opts.failedOnly }));
|
|
29486
30294
|
}
|
|
29487
30295
|
if (opts.githubComment) {
|
|
29488
|
-
const { postGitHubComment: postGitHubComment2 } = await Promise.resolve().then(() => exports_ci);
|
|
30296
|
+
const { postGitHubComment: postGitHubComment2 } = await Promise.resolve().then(() => (init_ci(), exports_ci));
|
|
29489
30297
|
const prNumber = opts.pr ? parseInt(opts.pr, 10) : undefined;
|
|
29490
30298
|
const posted = await postGitHubComment2(run2, results2, { prNumber });
|
|
29491
30299
|
if (posted) {
|
|
29492
30300
|
log(chalk6.green(" GitHub PR comment posted."));
|
|
29493
30301
|
} else if (!process.env["GITHUB_TOKEN"]) {
|
|
29494
30302
|
log(chalk6.yellow(" --github-comment: GITHUB_TOKEN not set, skipping PR comment."));
|
|
30303
|
+
} else {
|
|
30304
|
+
log(chalk6.yellow(" --github-comment: could not post PR comment (check GITHUB_PR_NUMBER / GITHUB_REF / GITHUB_REPOSITORY env)."));
|
|
29495
30305
|
}
|
|
29496
30306
|
}
|
|
29497
30307
|
process.exit(getExitCode(run2));
|
|
@@ -29502,6 +30312,33 @@ program2.command("run [url] [description]").alias("test").description("Run test
|
|
|
29502
30312
|
log(chalk6.bold(` Running all ${allScenarios.length} scenarios...`));
|
|
29503
30313
|
log("");
|
|
29504
30314
|
}
|
|
30315
|
+
const shouldAutoGenerate = urlArg !== undefined && opts.autoGenerate !== false && noFilters;
|
|
30316
|
+
if (shouldAutoGenerate) {
|
|
30317
|
+
const existingScenarios = listScenarios({ projectId });
|
|
30318
|
+
if (existingScenarios.length === 0) {
|
|
30319
|
+
log(chalk6.blue(" No scenarios found \u2014 crawling URL to auto-generate scenarios..."));
|
|
30320
|
+
log(chalk6.dim(` (disable with --no-auto-generate)`));
|
|
30321
|
+
try {
|
|
30322
|
+
const { crawlAndGenerate: crawlAndGenerate2 } = await Promise.resolve().then(() => (init_crawl_and_generate(), exports_crawl_and_generate));
|
|
30323
|
+
const crawlResult = await crawlAndGenerate2({
|
|
30324
|
+
url,
|
|
30325
|
+
projectId,
|
|
30326
|
+
maxPages: 5,
|
|
30327
|
+
scenariosPerPage: 2,
|
|
30328
|
+
model: opts.model,
|
|
30329
|
+
apiKey: process.env["ANTHROPIC_API_KEY"],
|
|
30330
|
+
headed: opts.headed,
|
|
30331
|
+
tags: ["auto-generated"]
|
|
30332
|
+
});
|
|
30333
|
+
log(chalk6.green(` Generated ${crawlResult.totalScenariosCreated} scenarios from ${crawlResult.pagesGenerated} page(s).`));
|
|
30334
|
+
log("");
|
|
30335
|
+
} catch (err) {
|
|
30336
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
30337
|
+
log(chalk6.yellow(` Auto-generate failed: ${msg}`));
|
|
30338
|
+
log(chalk6.yellow(` Continuing with 0 scenarios \u2014 the run will exit cleanly (0 passed, 0 failed).`));
|
|
30339
|
+
}
|
|
30340
|
+
}
|
|
30341
|
+
}
|
|
29505
30342
|
let diffScenarioIds;
|
|
29506
30343
|
if (opts.diff) {
|
|
29507
30344
|
try {
|
|
@@ -29564,6 +30401,18 @@ program2.command("run [url] [description]").alias("test").description("Run test
|
|
|
29564
30401
|
} else {
|
|
29565
30402
|
log(formatTerminal(run, results, { failedOnly: opts.failedOnly }));
|
|
29566
30403
|
}
|
|
30404
|
+
if (opts.githubComment) {
|
|
30405
|
+
const { postGitHubComment: postGitHubComment2 } = await Promise.resolve().then(() => (init_ci(), exports_ci));
|
|
30406
|
+
const prNumber = opts.pr ? parseInt(opts.pr, 10) : undefined;
|
|
30407
|
+
const posted = await postGitHubComment2(run, results, { prNumber });
|
|
30408
|
+
if (posted) {
|
|
30409
|
+
log(chalk6.green(" GitHub PR comment posted."));
|
|
30410
|
+
} else if (!process.env["GITHUB_TOKEN"]) {
|
|
30411
|
+
log(chalk6.yellow(" --github-comment: GITHUB_TOKEN not set, skipping PR comment."));
|
|
30412
|
+
} else {
|
|
30413
|
+
log(chalk6.yellow(" --github-comment: could not post PR comment (check GITHUB_PR_NUMBER / GITHUB_REF / GITHUB_REPOSITORY env)."));
|
|
30414
|
+
}
|
|
30415
|
+
}
|
|
29567
30416
|
process.exit(getExitCode(run));
|
|
29568
30417
|
} catch (error) {
|
|
29569
30418
|
logError(chalk6.red(`Error: ${error instanceof Error ? error.message : String(error)}`));
|
|
@@ -29741,7 +30590,7 @@ program2.command("import <dir>").description("Import markdown test files as scen
|
|
|
29741
30590
|
}
|
|
29742
30591
|
let imported = 0;
|
|
29743
30592
|
for (const file of files) {
|
|
29744
|
-
const content = readFileSync8(
|
|
30593
|
+
const content = readFileSync8(join16(absDir, file), "utf-8");
|
|
29745
30594
|
const lines = content.split(`
|
|
29746
30595
|
`);
|
|
29747
30596
|
let name = file.replace(/\.md$/, "");
|
|
@@ -29802,7 +30651,7 @@ program2.command("export [format]").description("Export scenarios as JSON (defau
|
|
|
29802
30651
|
}
|
|
29803
30652
|
const outputDir = opts.output ?? ".";
|
|
29804
30653
|
if (!existsSync14(outputDir)) {
|
|
29805
|
-
|
|
30654
|
+
mkdirSync11(outputDir, { recursive: true });
|
|
29806
30655
|
}
|
|
29807
30656
|
for (const s of scenarios) {
|
|
29808
30657
|
const lines = [];
|
|
@@ -29829,7 +30678,7 @@ program2.command("export [format]").description("Export scenarios as JSON (defau
|
|
|
29829
30678
|
lines.push("");
|
|
29830
30679
|
}
|
|
29831
30680
|
const safeFilename = s.name.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "").slice(0, 80);
|
|
29832
|
-
const filePath =
|
|
30681
|
+
const filePath = join16(outputDir, `${s.shortId}-${safeFilename}.md`);
|
|
29833
30682
|
writeFileSync4(filePath, lines.join(`
|
|
29834
30683
|
`), "utf-8");
|
|
29835
30684
|
log(chalk6.dim(` ${s.shortId}: ${s.name} \u2192 ${filePath}`));
|
|
@@ -29854,7 +30703,7 @@ program2.command("status").description("Show database and auth status").action((
|
|
|
29854
30703
|
try {
|
|
29855
30704
|
const config = loadConfig();
|
|
29856
30705
|
const hasApiKey = !!config.anthropicApiKey || !!process.env["ANTHROPIC_API_KEY"];
|
|
29857
|
-
const dbPath =
|
|
30706
|
+
const dbPath = join16(getTestersDir(), "testers.db");
|
|
29858
30707
|
log("");
|
|
29859
30708
|
log(chalk6.bold(" Open Testers Status"));
|
|
29860
30709
|
log("");
|
|
@@ -29974,7 +30823,7 @@ projectCmd.command("use <name>").description("Set active project (find or create
|
|
|
29974
30823
|
try {
|
|
29975
30824
|
const project = ensureProject(name, process.cwd());
|
|
29976
30825
|
if (!existsSync14(CONFIG_DIR5)) {
|
|
29977
|
-
|
|
30826
|
+
mkdirSync11(CONFIG_DIR5, { recursive: true });
|
|
29978
30827
|
}
|
|
29979
30828
|
let config = {};
|
|
29980
30829
|
if (existsSync14(CONFIG_PATH3)) {
|
|
@@ -30207,6 +31056,25 @@ Shutting down scheduler daemon...`));
|
|
|
30207
31056
|
process.exit(1);
|
|
30208
31057
|
}
|
|
30209
31058
|
});
|
|
31059
|
+
program2.command("ci [provider]").description("Print or write a CI workflow (default provider: github)").option("-o, --output <path>", "Write the workflow to a file (e.g. .github/workflows/qa.yml)").action(async (providerArg, opts) => {
|
|
31060
|
+
const provider = (providerArg ?? "github").toLowerCase();
|
|
31061
|
+
if (provider !== "github") {
|
|
31062
|
+
logError(chalk6.red(`Unknown CI provider: ${provider}. Supported: github`));
|
|
31063
|
+
process.exit(2);
|
|
31064
|
+
}
|
|
31065
|
+
const workflow = generateGitHubActionsWorkflow();
|
|
31066
|
+
if (opts.output) {
|
|
31067
|
+
const outPath = resolve(opts.output);
|
|
31068
|
+
const outDir = outPath.replace(/\/[^/]*$/, "");
|
|
31069
|
+
if (outDir && !existsSync14(outDir)) {
|
|
31070
|
+
mkdirSync11(outDir, { recursive: true });
|
|
31071
|
+
}
|
|
31072
|
+
writeFileSync4(outPath, workflow, "utf-8");
|
|
31073
|
+
log(chalk6.green(`Workflow written to ${outPath}`));
|
|
31074
|
+
return;
|
|
31075
|
+
}
|
|
31076
|
+
process.stdout.write(workflow);
|
|
31077
|
+
});
|
|
30210
31078
|
program2.command("init").description("Initialize a new testing project").option("-n, --name <name>", "Project name").option("-u, --url <url>", "Base URL").option("-p, --path <path>", "Project path").option("--ci <provider>", "Generate CI workflow (github)").option("-y, --yes", "Skip interactive prompts (non-interactive mode)", false).action(async (opts) => {
|
|
30211
31079
|
try {
|
|
30212
31080
|
const { project, scenarios, framework, url } = initProject({
|
|
@@ -30232,11 +31100,11 @@ program2.command("init").description("Initialize a new testing project").option(
|
|
|
30232
31100
|
log(` ${chalk6.dim(s.shortId)} ${s.name} ${chalk6.dim(`[${s.tags.join(", ")}]`)}`);
|
|
30233
31101
|
}
|
|
30234
31102
|
if (opts.ci === "github") {
|
|
30235
|
-
const workflowDir =
|
|
31103
|
+
const workflowDir = join16(process.cwd(), ".github", "workflows");
|
|
30236
31104
|
if (!existsSync14(workflowDir)) {
|
|
30237
|
-
|
|
31105
|
+
mkdirSync11(workflowDir, { recursive: true });
|
|
30238
31106
|
}
|
|
30239
|
-
const workflowPath =
|
|
31107
|
+
const workflowPath = join16(workflowDir, "testers.yml");
|
|
30240
31108
|
writeFileSync4(workflowPath, generateGitHubActionsWorkflow(), "utf-8");
|
|
30241
31109
|
log(` CI: ${chalk6.green("GitHub Actions workflow written to .github/workflows/testers.yml")}`);
|
|
30242
31110
|
} else if (opts.ci) {
|
|
@@ -31255,7 +32123,7 @@ program2.command("doctor").description("Check system setup and configuration").a
|
|
|
31255
32123
|
log(chalk6.red("\u2717") + " ANTHROPIC_API_KEY is not set (required for AI-powered tests)");
|
|
31256
32124
|
allPassed = false;
|
|
31257
32125
|
}
|
|
31258
|
-
const dbPath =
|
|
32126
|
+
const dbPath = join16(getTestersDir(), "testers.db");
|
|
31259
32127
|
try {
|
|
31260
32128
|
const { Database: Database4 } = await import("bun:sqlite");
|
|
31261
32129
|
const db2 = new Database4(dbPath, { create: true });
|
|
@@ -31266,8 +32134,8 @@ program2.command("doctor").description("Check system setup and configuration").a
|
|
|
31266
32134
|
allPassed = false;
|
|
31267
32135
|
}
|
|
31268
32136
|
try {
|
|
31269
|
-
const { chromium:
|
|
31270
|
-
const execPath =
|
|
32137
|
+
const { chromium: chromium3 } = await import("playwright");
|
|
32138
|
+
const execPath = chromium3.executablePath();
|
|
31271
32139
|
const { existsSync: fsExists } = await import("fs");
|
|
31272
32140
|
if (fsExists(execPath)) {
|
|
31273
32141
|
log(chalk6.green("\u2713") + " Playwright chromium is installed");
|
|
@@ -31307,7 +32175,7 @@ program2.command("serve").description("Start the Open Testers web dashboard").op
|
|
|
31307
32175
|
try {
|
|
31308
32176
|
const port = parseInt(opts.port, 10);
|
|
31309
32177
|
const url = `http://localhost:${port}`;
|
|
31310
|
-
const serverBin =
|
|
32178
|
+
const serverBin = join16(resolve(process.execPath, ".."), "..", "dist", "server", "index.js");
|
|
31311
32179
|
const { join: pathJoin, resolve: pathResolve, dirname: dirname4 } = await import("path");
|
|
31312
32180
|
const { fileURLToPath } = await import("url");
|
|
31313
32181
|
const serverPath = pathJoin(dirname4(fileURLToPath(import.meta.url)), "..", "server", "index.js");
|