@hasna/testers 0.0.28 → 0.0.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -154
- package/README.md +92 -0
- package/dist/cli/index.js +1354 -79
- package/dist/db/database.d.ts.map +1 -1
- package/dist/db/personas.d.ts.map +1 -1
- package/dist/db/runs.d.ts +29 -0
- package/dist/db/runs.d.ts.map +1 -1
- package/dist/db/scenarios.d.ts +12 -0
- package/dist/db/scenarios.d.ts.map +1 -1
- package/dist/db/sessions.d.ts +36 -0
- package/dist/db/sessions.d.ts.map +1 -0
- package/dist/db/step-results.d.ts +30 -0
- package/dist/db/step-results.d.ts.map +1 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +818 -25
- package/dist/lib/a11y-audit.d.ts +54 -0
- package/dist/lib/a11y-audit.d.ts.map +1 -0
- package/dist/lib/api-discovery.d.ts +46 -0
- package/dist/lib/api-discovery.d.ts.map +1 -0
- package/dist/lib/assertions.d.ts.map +1 -1
- package/dist/lib/auth-profiles.d.ts +16 -0
- package/dist/lib/auth-profiles.d.ts.map +1 -0
- package/dist/lib/auth-session-pool.d.ts +57 -0
- package/dist/lib/auth-session-pool.d.ts.map +1 -0
- package/dist/lib/batch-actions.d.ts +44 -0
- package/dist/lib/batch-actions.d.ts.map +1 -0
- package/dist/lib/browser-compat.d.ts +14 -0
- package/dist/lib/browser-compat.d.ts.map +1 -0
- package/dist/lib/browser.d.ts +6 -7
- package/dist/lib/browser.d.ts.map +1 -1
- package/dist/lib/ci.d.ts +12 -0
- package/dist/lib/ci.d.ts.map +1 -1
- package/dist/lib/config.d.ts.map +1 -1
- package/dist/lib/discovery.d.ts +33 -0
- package/dist/lib/discovery.d.ts.map +1 -0
- package/dist/lib/dom-mutation.d.ts +53 -0
- package/dist/lib/dom-mutation.d.ts.map +1 -0
- package/dist/lib/environment.d.ts +26 -0
- package/dist/lib/environment.d.ts.map +1 -0
- package/dist/lib/health-scan.d.ts +2 -1
- package/dist/lib/health-scan.d.ts.map +1 -1
- package/dist/lib/hybrid-runner.d.ts.map +1 -1
- package/dist/lib/junit-export.d.ts +24 -0
- package/dist/lib/junit-export.d.ts.map +1 -0
- package/dist/lib/network-mock.d.ts +38 -0
- package/dist/lib/network-mock.d.ts.map +1 -0
- package/dist/lib/offline-mode.d.ts +31 -0
- package/dist/lib/offline-mode.d.ts.map +1 -0
- package/dist/lib/pdf-export.d.ts +27 -0
- package/dist/lib/pdf-export.d.ts.map +1 -0
- package/dist/lib/performance.d.ts +65 -0
- package/dist/lib/performance.d.ts.map +1 -0
- package/dist/lib/pr-comment.d.ts +27 -0
- package/dist/lib/pr-comment.d.ts.map +1 -0
- package/dist/lib/preview-detect.d.ts +27 -0
- package/dist/lib/preview-detect.d.ts.map +1 -0
- package/dist/lib/prod-debug.d.ts +77 -0
- package/dist/lib/prod-debug.d.ts.map +1 -0
- package/dist/lib/recorder.d.ts +42 -0
- package/dist/lib/recorder.d.ts.map +1 -1
- package/dist/lib/repo-discovery.d.ts +102 -0
- package/dist/lib/repo-discovery.d.ts.map +1 -0
- package/dist/lib/repo-executor.d.ts +56 -0
- package/dist/lib/repo-executor.d.ts.map +1 -0
- package/dist/lib/responsive.d.ts +43 -0
- package/dist/lib/responsive.d.ts.map +1 -0
- package/dist/lib/runner.d.ts +1 -0
- package/dist/lib/runner.d.ts.map +1 -1
- package/dist/lib/scenario-chain.d.ts +52 -0
- package/dist/lib/scenario-chain.d.ts.map +1 -0
- package/dist/lib/templates.d.ts.map +1 -1
- package/dist/lib/webhooks.d.ts +3 -0
- package/dist/lib/webhooks.d.ts.map +1 -1
- package/dist/mcp/index.js +852 -74
- package/dist/sdk/index.d.ts +48 -0
- package/dist/sdk/index.d.ts.map +1 -0
- package/dist/server/index.js +276 -29
- package/dist/types/index.d.ts +66 -2
- package/dist/types/index.d.ts.map +1 -1
- package/package.json +2 -2
package/dist/cli/index.js
CHANGED
|
@@ -2148,7 +2148,8 @@ function scenarioFromRow(row) {
|
|
|
2148
2148
|
createdAt: row.created_at,
|
|
2149
2149
|
updatedAt: row.updated_at,
|
|
2150
2150
|
lastPassedAt: row.last_passed_at ?? null,
|
|
2151
|
-
lastPassedUrl: row.last_passed_url ?? null
|
|
2151
|
+
lastPassedUrl: row.last_passed_url ?? null,
|
|
2152
|
+
parameters: row.parameters ? JSON.parse(row.parameters) : null
|
|
2152
2153
|
};
|
|
2153
2154
|
}
|
|
2154
2155
|
function runFromRow(row) {
|
|
@@ -2168,7 +2169,14 @@ function runFromRow(row) {
|
|
|
2168
2169
|
metadata: row.metadata ? JSON.parse(row.metadata) : null,
|
|
2169
2170
|
isBaseline: row.is_baseline === 1,
|
|
2170
2171
|
samples: row.samples ?? 1,
|
|
2171
|
-
flakinessThreshold: row.flakiness_threshold ?? 0.95
|
|
2172
|
+
flakinessThreshold: row.flakiness_threshold ?? 0.95,
|
|
2173
|
+
prNumber: row.pr_number ?? null,
|
|
2174
|
+
prTitle: row.pr_title ?? null,
|
|
2175
|
+
prBranch: row.pr_branch ?? null,
|
|
2176
|
+
prBaseBranch: row.pr_base_branch ?? null,
|
|
2177
|
+
prCommitSha: row.pr_commit_sha ?? null,
|
|
2178
|
+
prUrl: row.pr_url ?? null,
|
|
2179
|
+
ghAppInstallationId: row.gh_app_installation_id ?? null
|
|
2172
2180
|
};
|
|
2173
2181
|
}
|
|
2174
2182
|
function resultFromRow(row) {
|
|
@@ -2189,7 +2197,8 @@ function resultFromRow(row) {
|
|
|
2189
2197
|
createdAt: row.created_at,
|
|
2190
2198
|
personaId: row.persona_id ?? null,
|
|
2191
2199
|
personaName: row.persona_name ?? null,
|
|
2192
|
-
failureAnalysis: row.failure_analysis ? JSON.parse(row.failure_analysis) : null
|
|
2200
|
+
failureAnalysis: row.failure_analysis ? JSON.parse(row.failure_analysis) : null,
|
|
2201
|
+
harPath: row.har_path ?? null
|
|
2193
2202
|
};
|
|
2194
2203
|
}
|
|
2195
2204
|
function screenshotFromRow(row) {
|
|
@@ -2281,7 +2290,10 @@ function personaFromRow(row) {
|
|
|
2281
2290
|
email: row.auth_email,
|
|
2282
2291
|
password: row.auth_password,
|
|
2283
2292
|
loginPath: row.auth_login_path ?? "/login",
|
|
2284
|
-
cookies: row.auth_cookies ? JSON.parse(row.auth_cookies) : null
|
|
2293
|
+
cookies: row.auth_cookies ? JSON.parse(row.auth_cookies) : null,
|
|
2294
|
+
strategy: row.auth_strategy ?? "form-login",
|
|
2295
|
+
headers: row.auth_headers ? JSON.parse(row.auth_headers) : undefined,
|
|
2296
|
+
customScript: row.auth_script ?? undefined
|
|
2285
2297
|
} : null
|
|
2286
2298
|
};
|
|
2287
2299
|
}
|
|
@@ -12200,6 +12212,43 @@ ALTER TABLE scenarios ADD COLUMN required_role TEXT;
|
|
|
12200
12212
|
machine_id TEXT,
|
|
12201
12213
|
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
12202
12214
|
);
|
|
12215
|
+
`,
|
|
12216
|
+
`
|
|
12217
|
+
ALTER TABLE results ADD COLUMN har_path TEXT;
|
|
12218
|
+
`,
|
|
12219
|
+
`
|
|
12220
|
+
ALTER TABLE scenarios ADD COLUMN parameters TEXT;
|
|
12221
|
+
`,
|
|
12222
|
+
`
|
|
12223
|
+
ALTER TABLE personas ADD COLUMN auth_strategy TEXT DEFAULT 'form-login';
|
|
12224
|
+
ALTER TABLE personas ADD COLUMN auth_headers TEXT;
|
|
12225
|
+
ALTER TABLE personas ADD COLUMN auth_script TEXT;
|
|
12226
|
+
`,
|
|
12227
|
+
`
|
|
12228
|
+
CREATE TABLE IF NOT EXISTS step_results (
|
|
12229
|
+
id TEXT PRIMARY KEY,
|
|
12230
|
+
result_id TEXT NOT NULL REFERENCES results(id) ON DELETE CASCADE,
|
|
12231
|
+
step_number INTEGER NOT NULL,
|
|
12232
|
+
action TEXT NOT NULL,
|
|
12233
|
+
status TEXT NOT NULL DEFAULT 'running' CHECK(status IN ('passed','failed','error','running','skipped')),
|
|
12234
|
+
tool_name TEXT,
|
|
12235
|
+
tool_input TEXT,
|
|
12236
|
+
tool_result TEXT,
|
|
12237
|
+
thinking TEXT,
|
|
12238
|
+
error TEXT,
|
|
12239
|
+
duration_ms INTEGER,
|
|
12240
|
+
screenshot_id TEXT REFERENCES screenshots(id),
|
|
12241
|
+
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
12242
|
+
);
|
|
12243
|
+
`,
|
|
12244
|
+
`
|
|
12245
|
+
ALTER TABLE runs ADD COLUMN pr_number INTEGER;
|
|
12246
|
+
ALTER TABLE runs ADD COLUMN pr_title TEXT;
|
|
12247
|
+
ALTER TABLE runs ADD COLUMN pr_branch TEXT;
|
|
12248
|
+
ALTER TABLE runs ADD COLUMN pr_base_branch TEXT;
|
|
12249
|
+
ALTER TABLE runs ADD COLUMN pr_commit_sha TEXT;
|
|
12250
|
+
ALTER TABLE runs ADD COLUMN pr_url TEXT;
|
|
12251
|
+
ALTER TABLE runs ADD COLUMN gh_app_installation_id TEXT;
|
|
12203
12252
|
`
|
|
12204
12253
|
];
|
|
12205
12254
|
});
|
|
@@ -12207,6 +12256,7 @@ ALTER TABLE scenarios ADD COLUMN required_role TEXT;
|
|
|
12207
12256
|
// src/db/scenarios.ts
|
|
12208
12257
|
var exports_scenarios = {};
|
|
12209
12258
|
__export(exports_scenarios, {
|
|
12259
|
+
upsertScenario: () => upsertScenario,
|
|
12210
12260
|
updateScenarioPassedCache: () => updateScenarioPassedCache,
|
|
12211
12261
|
updateScenario: () => updateScenario,
|
|
12212
12262
|
listScenarios: () => listScenarios,
|
|
@@ -12235,9 +12285,9 @@ function createScenario(input) {
|
|
|
12235
12285
|
const short_id = nextShortId(input.projectId);
|
|
12236
12286
|
const timestamp = now();
|
|
12237
12287
|
db2.query(`
|
|
12238
|
-
INSERT INTO scenarios (id, short_id, project_id, name, description, steps, tags, priority, model, timeout_ms, target_path, requires_auth, auth_config, metadata, assertions, version, created_at, updated_at)
|
|
12239
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1, ?, ?)
|
|
12240
|
-
`).run(id, short_id, input.projectId ?? null, input.name, input.description, JSON.stringify(input.steps ?? []), JSON.stringify(input.tags ?? []), input.priority ?? "medium", input.model ?? null, input.timeoutMs ?? null, input.targetPath ?? null, input.requiresAuth ? 1 : 0, input.authConfig ? JSON.stringify(input.authConfig) : null, input.metadata ? JSON.stringify(input.metadata) : null, JSON.stringify(input.assertions ?? []), timestamp, timestamp);
|
|
12288
|
+
INSERT INTO scenarios (id, short_id, project_id, name, description, steps, tags, priority, model, timeout_ms, target_path, requires_auth, auth_config, metadata, assertions, parameters, version, created_at, updated_at)
|
|
12289
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1, ?, ?)
|
|
12290
|
+
`).run(id, short_id, input.projectId ?? null, input.name, input.description, JSON.stringify(input.steps ?? []), JSON.stringify(input.tags ?? []), input.priority ?? "medium", input.model ?? null, input.timeoutMs ?? null, input.targetPath ?? null, input.requiresAuth ? 1 : 0, input.authConfig ? JSON.stringify(input.authConfig) : null, input.metadata ? JSON.stringify(input.metadata) : null, JSON.stringify(input.assertions ?? []), input.parameters ? JSON.stringify(input.parameters) : null, timestamp, timestamp);
|
|
12241
12291
|
return getScenario(id);
|
|
12242
12292
|
}
|
|
12243
12293
|
function getScenario(id) {
|
|
@@ -12387,6 +12437,10 @@ function updateScenario(id, input, version) {
|
|
|
12387
12437
|
sets.push("assertions = ?");
|
|
12388
12438
|
params.push(JSON.stringify(input.assertions));
|
|
12389
12439
|
}
|
|
12440
|
+
if (input.parameters !== undefined) {
|
|
12441
|
+
sets.push("parameters = ?");
|
|
12442
|
+
params.push(JSON.stringify(input.parameters));
|
|
12443
|
+
}
|
|
12390
12444
|
if (sets.length === 0) {
|
|
12391
12445
|
return existing;
|
|
12392
12446
|
}
|
|
@@ -12459,6 +12513,75 @@ function deleteScenario(id) {
|
|
|
12459
12513
|
const result = db2.query("DELETE FROM scenarios WHERE id = ?").run(scenario.id);
|
|
12460
12514
|
return result.changes > 0;
|
|
12461
12515
|
}
|
|
12516
|
+
function upsertScenario(input) {
|
|
12517
|
+
const db2 = getDatabase();
|
|
12518
|
+
const existing = db2.query("SELECT * FROM scenarios WHERE name = ? AND project_id IS NOT DISTINCT FROM ?").get(input.name, input.projectId ?? null);
|
|
12519
|
+
if (!existing) {
|
|
12520
|
+
return { scenario: createScenario(input), action: "created" };
|
|
12521
|
+
}
|
|
12522
|
+
const existingSteps = JSON.parse(existing.steps);
|
|
12523
|
+
const existingTags = JSON.parse(existing.tags);
|
|
12524
|
+
const newSteps = input.steps ?? [];
|
|
12525
|
+
const newTags = input.tags ?? [];
|
|
12526
|
+
const isIdentical = existing.description === (input.description ?? "") && existingSteps.length === newSteps.length && existingSteps.every((s, i) => s === newSteps[i]) && existingTags.length === newTags.length && existingTags.every((t, i) => t === newTags[i]) && existing.priority === (input.priority ?? "medium");
|
|
12527
|
+
if (isIdentical) {
|
|
12528
|
+
return { scenario: scenarioFromRow(existing), action: "deduped" };
|
|
12529
|
+
}
|
|
12530
|
+
const sets = [];
|
|
12531
|
+
const params = [];
|
|
12532
|
+
if (input.description !== undefined) {
|
|
12533
|
+
sets.push("description = ?");
|
|
12534
|
+
params.push(input.description);
|
|
12535
|
+
}
|
|
12536
|
+
if (input.steps !== undefined) {
|
|
12537
|
+
sets.push("steps = ?");
|
|
12538
|
+
params.push(JSON.stringify(input.steps));
|
|
12539
|
+
}
|
|
12540
|
+
if (input.tags !== undefined) {
|
|
12541
|
+
sets.push("tags = ?");
|
|
12542
|
+
params.push(JSON.stringify(input.tags));
|
|
12543
|
+
}
|
|
12544
|
+
if (input.priority !== undefined) {
|
|
12545
|
+
sets.push("priority = ?");
|
|
12546
|
+
params.push(input.priority);
|
|
12547
|
+
}
|
|
12548
|
+
if (input.model !== undefined) {
|
|
12549
|
+
sets.push("model = ?");
|
|
12550
|
+
params.push(input.model);
|
|
12551
|
+
}
|
|
12552
|
+
if (input.timeoutMs !== undefined) {
|
|
12553
|
+
sets.push("timeout_ms = ?");
|
|
12554
|
+
params.push(input.timeoutMs);
|
|
12555
|
+
}
|
|
12556
|
+
if (input.targetPath !== undefined) {
|
|
12557
|
+
sets.push("target_path = ?");
|
|
12558
|
+
params.push(input.targetPath);
|
|
12559
|
+
}
|
|
12560
|
+
if (input.requiresAuth !== undefined) {
|
|
12561
|
+
sets.push("requires_auth = ?");
|
|
12562
|
+
params.push(input.requiresAuth ? 1 : 0);
|
|
12563
|
+
}
|
|
12564
|
+
if (input.authConfig !== undefined) {
|
|
12565
|
+
sets.push("auth_config = ?");
|
|
12566
|
+
params.push(JSON.stringify(input.authConfig));
|
|
12567
|
+
}
|
|
12568
|
+
if (input.metadata !== undefined) {
|
|
12569
|
+
sets.push("metadata = ?");
|
|
12570
|
+
params.push(JSON.stringify(input.metadata));
|
|
12571
|
+
}
|
|
12572
|
+
if (input.assertions !== undefined) {
|
|
12573
|
+
sets.push("assertions = ?");
|
|
12574
|
+
params.push(JSON.stringify(input.assertions));
|
|
12575
|
+
}
|
|
12576
|
+
sets.push("version = ?", "updated_at = ?");
|
|
12577
|
+
params.push(existing.version + 1, now());
|
|
12578
|
+
params.push(existing.id);
|
|
12579
|
+
const result = db2.query(`UPDATE scenarios SET ${sets.join(", ")} WHERE id = ?`).run(...params);
|
|
12580
|
+
if (result.changes === 0) {
|
|
12581
|
+
return { scenario: scenarioFromRow(existing), action: "deduped" };
|
|
12582
|
+
}
|
|
12583
|
+
return { scenario: getScenario(existing.id), action: "updated" };
|
|
12584
|
+
}
|
|
12462
12585
|
var init_scenarios = __esm(() => {
|
|
12463
12586
|
init_types();
|
|
12464
12587
|
init_database();
|
|
@@ -12468,8 +12591,12 @@ var init_scenarios = __esm(() => {
|
|
|
12468
12591
|
var exports_runs = {};
|
|
12469
12592
|
__export(exports_runs, {
|
|
12470
12593
|
updateRun: () => updateRun,
|
|
12594
|
+
updatePrRunMetadata: () => updatePrRunMetadata,
|
|
12471
12595
|
listRuns: () => listRuns,
|
|
12596
|
+
listPrRuns: () => listPrRuns,
|
|
12597
|
+
getRunsByPr: () => getRunsByPr,
|
|
12472
12598
|
getRun: () => getRun,
|
|
12599
|
+
getLatestPrRun: () => getLatestPrRun,
|
|
12473
12600
|
deleteRun: () => deleteRun,
|
|
12474
12601
|
createRun: () => createRun,
|
|
12475
12602
|
countRuns: () => countRuns
|
|
@@ -12479,9 +12606,9 @@ function createRun(input) {
|
|
|
12479
12606
|
const id = uuid();
|
|
12480
12607
|
const timestamp = now();
|
|
12481
12608
|
db2.query(`
|
|
12482
|
-
INSERT INTO runs (id, project_id, status, url, model, headed, parallel, total, passed, failed, started_at, finished_at, metadata, samples, flakiness_threshold)
|
|
12483
|
-
VALUES (?, ?, 'pending', ?, ?, ?, ?, 0, 0, 0, ?, NULL, ?, ?, ?)
|
|
12484
|
-
`).run(id, input.projectId ?? null, input.url, input.model, input.headed ? 1 : 0, input.parallel ?? 1, timestamp,
|
|
12609
|
+
INSERT INTO runs (id, project_id, status, url, model, headed, parallel, total, passed, failed, started_at, finished_at, metadata, samples, flakiness_threshold, pr_number, pr_title, pr_branch, pr_base_branch, pr_commit_sha, pr_url, gh_app_installation_id)
|
|
12610
|
+
VALUES (?, ?, 'pending', ?, ?, ?, ?, 0, 0, 0, ?, NULL, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
12611
|
+
`).run(id, input.projectId ?? null, input.url, input.model, input.headed ? 1 : 0, input.parallel ?? 1, timestamp, JSON.stringify({}), input.samples ?? 1, input.flakinessThreshold ?? 0.95, input.prNumber ?? null, input.prTitle ?? null, input.prBranch ?? null, input.prBaseBranch ?? null, input.prCommitSha ?? null, input.prUrl ?? null, input.ghAppInstallationId ?? null);
|
|
12485
12612
|
return getRun(id);
|
|
12486
12613
|
}
|
|
12487
12614
|
function getRun(id) {
|
|
@@ -12509,6 +12636,14 @@ function listRuns(filter) {
|
|
|
12509
12636
|
conditions.push("status = ?");
|
|
12510
12637
|
params.push(filter.status);
|
|
12511
12638
|
}
|
|
12639
|
+
if (filter?.since) {
|
|
12640
|
+
conditions.push("started_at >= ?");
|
|
12641
|
+
params.push(filter.since);
|
|
12642
|
+
}
|
|
12643
|
+
if (filter?.until) {
|
|
12644
|
+
conditions.push("started_at <= ?");
|
|
12645
|
+
params.push(filter.until);
|
|
12646
|
+
}
|
|
12512
12647
|
let sql = "SELECT * FROM runs";
|
|
12513
12648
|
if (conditions.length > 0) {
|
|
12514
12649
|
sql += " WHERE " + conditions.join(" AND ");
|
|
@@ -12540,6 +12675,14 @@ function countRuns(filter) {
|
|
|
12540
12675
|
conditions.push("status = ?");
|
|
12541
12676
|
params.push(filter.status);
|
|
12542
12677
|
}
|
|
12678
|
+
if (filter?.since) {
|
|
12679
|
+
conditions.push("started_at >= ?");
|
|
12680
|
+
params.push(filter.since);
|
|
12681
|
+
}
|
|
12682
|
+
if (filter?.until) {
|
|
12683
|
+
conditions.push("started_at <= ?");
|
|
12684
|
+
params.push(filter.until);
|
|
12685
|
+
}
|
|
12543
12686
|
let sql = "SELECT COUNT(*) as count FROM runs";
|
|
12544
12687
|
if (conditions.length > 0)
|
|
12545
12688
|
sql += " WHERE " + conditions.join(" AND ");
|
|
@@ -12617,6 +12760,52 @@ function deleteRun(id) {
|
|
|
12617
12760
|
const result = db2.query("DELETE FROM runs WHERE id = ?").run(run.id);
|
|
12618
12761
|
return result.changes > 0;
|
|
12619
12762
|
}
|
|
12763
|
+
function getRunsByPr(prNumber) {
|
|
12764
|
+
const db2 = getDatabase();
|
|
12765
|
+
const rows = db2.query("SELECT * FROM runs WHERE pr_number = ? ORDER BY started_at DESC").all(prNumber);
|
|
12766
|
+
return rows.map(runFromRow);
|
|
12767
|
+
}
|
|
12768
|
+
function getLatestPrRun(prNumber) {
|
|
12769
|
+
const db2 = getDatabase();
|
|
12770
|
+
const row = db2.query("SELECT * FROM runs WHERE pr_number = ? ORDER BY started_at DESC, rowid DESC LIMIT 1").get(prNumber);
|
|
12771
|
+
return row ? runFromRow(row) : null;
|
|
12772
|
+
}
|
|
12773
|
+
function listPrRuns(filter) {
|
|
12774
|
+
const db2 = getDatabase();
|
|
12775
|
+
const conditions = ["pr_number IS NOT NULL"];
|
|
12776
|
+
const params = [];
|
|
12777
|
+
if (filter?.branch) {
|
|
12778
|
+
conditions.push("pr_branch = ?");
|
|
12779
|
+
params.push(filter.branch);
|
|
12780
|
+
}
|
|
12781
|
+
if (filter?.baseBranch) {
|
|
12782
|
+
conditions.push("pr_base_branch = ?");
|
|
12783
|
+
params.push(filter.baseBranch);
|
|
12784
|
+
}
|
|
12785
|
+
let sql = "SELECT * FROM runs WHERE " + conditions.join(" AND ") + " ORDER BY started_at DESC";
|
|
12786
|
+
if (filter?.limit) {
|
|
12787
|
+
sql += " LIMIT ?";
|
|
12788
|
+
params.push(filter.limit);
|
|
12789
|
+
}
|
|
12790
|
+
if (filter?.offset) {
|
|
12791
|
+
sql += " OFFSET ?";
|
|
12792
|
+
params.push(filter.offset);
|
|
12793
|
+
}
|
|
12794
|
+
const rows = db2.query(sql).all(...params);
|
|
12795
|
+
return rows.map(runFromRow);
|
|
12796
|
+
}
|
|
12797
|
+
function updatePrRunMetadata(runId, prData) {
|
|
12798
|
+
const db2 = getDatabase();
|
|
12799
|
+
const existing = getRun(runId);
|
|
12800
|
+
if (!existing) {
|
|
12801
|
+
throw new Error(`Run not found: ${runId}`);
|
|
12802
|
+
}
|
|
12803
|
+
db2.query(`
|
|
12804
|
+
UPDATE runs SET pr_number = ?, pr_title = ?, pr_branch = ?, pr_base_branch = ?, pr_commit_sha = ?, pr_url = ?, gh_app_installation_id = ?
|
|
12805
|
+
WHERE id = ?
|
|
12806
|
+
`).run(prData.prNumber, prData.prTitle ?? null, prData.prBranch ?? null, prData.prBaseBranch ?? null, prData.prCommitSha ?? null, prData.prUrl ?? null, prData.ghAppInstallationId ?? null, runId);
|
|
12807
|
+
return getRun(runId);
|
|
12808
|
+
}
|
|
12620
12809
|
var init_runs = __esm(() => {
|
|
12621
12810
|
init_types();
|
|
12622
12811
|
init_database();
|
|
@@ -13382,6 +13571,16 @@ async function launchBrowser(options) {
|
|
|
13382
13571
|
const headless = options?.headless ?? true;
|
|
13383
13572
|
const viewport = options?.viewport ?? DEFAULT_VIEWPORT;
|
|
13384
13573
|
try {
|
|
13574
|
+
if (engine === "playwright-firefox") {
|
|
13575
|
+
const { firefox } = await import("playwright");
|
|
13576
|
+
const browser = await firefox.launch({ headless });
|
|
13577
|
+
return browser;
|
|
13578
|
+
}
|
|
13579
|
+
if (engine === "playwright-webkit") {
|
|
13580
|
+
const { webkit } = await import("playwright");
|
|
13581
|
+
const browser = await webkit.launch({ headless });
|
|
13582
|
+
return browser;
|
|
13583
|
+
}
|
|
13385
13584
|
return await launchPlaywright({ headless, viewport });
|
|
13386
13585
|
} catch (error) {
|
|
13387
13586
|
const message = error instanceof Error ? error.message : String(error);
|
|
@@ -13498,8 +13697,9 @@ async function installBrowser(engine) {
|
|
|
13498
13697
|
const { installLightpanda: installLightpanda2 } = await Promise.resolve().then(() => (init_browser_lightpanda(), exports_browser_lightpanda));
|
|
13499
13698
|
return installLightpanda2();
|
|
13500
13699
|
}
|
|
13700
|
+
const browserName = engine === "playwright-firefox" ? "firefox" : engine === "playwright-webkit" ? "webkit" : "chromium";
|
|
13501
13701
|
try {
|
|
13502
|
-
execSync(
|
|
13702
|
+
execSync(`bunx playwright install ${browserName}`, {
|
|
13503
13703
|
stdio: "inherit"
|
|
13504
13704
|
});
|
|
13505
13705
|
} catch (error) {
|
|
@@ -13631,7 +13831,7 @@ function getDefaultConfig() {
|
|
|
13631
13831
|
browser: {
|
|
13632
13832
|
headless: true,
|
|
13633
13833
|
viewport: { width: 1280, height: 720 },
|
|
13634
|
-
timeout:
|
|
13834
|
+
timeout: 120000
|
|
13635
13835
|
},
|
|
13636
13836
|
screenshots: {
|
|
13637
13837
|
dir: join9(getTestersDir(), "screenshots"),
|
|
@@ -13661,7 +13861,8 @@ function loadConfig() {
|
|
|
13661
13861
|
judgeModel: fileConfig.judgeModel,
|
|
13662
13862
|
judgeProvider: fileConfig.judgeProvider,
|
|
13663
13863
|
selfHeal: fileConfig.selfHeal ?? false,
|
|
13664
|
-
conversationsSpace: fileConfig.conversationsSpace
|
|
13864
|
+
conversationsSpace: fileConfig.conversationsSpace,
|
|
13865
|
+
prodDebug: fileConfig.prodDebug
|
|
13665
13866
|
};
|
|
13666
13867
|
const envModel = process.env["TESTERS_MODEL"];
|
|
13667
13868
|
if (envModel) {
|
|
@@ -15657,6 +15858,76 @@ var init_costs = __esm(() => {
|
|
|
15657
15858
|
};
|
|
15658
15859
|
});
|
|
15659
15860
|
|
|
15861
|
+
// src/db/step-results.ts
|
|
15862
|
+
function createStepResult(input) {
|
|
15863
|
+
const db2 = getDatabase();
|
|
15864
|
+
const id = uuid();
|
|
15865
|
+
const timestamp = now();
|
|
15866
|
+
db2.query(`
|
|
15867
|
+
INSERT INTO step_results (id, result_id, step_number, action, status, tool_name, tool_input, thinking, created_at)
|
|
15868
|
+
VALUES (?, ?, ?, ?, 'running', ?, ?, ?, ?)
|
|
15869
|
+
`).run(id, input.resultId, input.stepNumber, input.action, input.toolName ?? null, input.toolInput ? JSON.stringify(input.toolInput) : null, input.thinking ?? null, timestamp);
|
|
15870
|
+
return getStepResult(id);
|
|
15871
|
+
}
|
|
15872
|
+
function getStepResult(id) {
|
|
15873
|
+
const db2 = getDatabase();
|
|
15874
|
+
const row = db2.query("SELECT * FROM step_results WHERE id = ?").get(id);
|
|
15875
|
+
return row ? stepResultFromRow(row) : null;
|
|
15876
|
+
}
|
|
15877
|
+
function updateStepResult(id, updates) {
|
|
15878
|
+
const db2 = getDatabase();
|
|
15879
|
+
const existing = getStepResult(id);
|
|
15880
|
+
if (!existing)
|
|
15881
|
+
return null;
|
|
15882
|
+
const sets = [];
|
|
15883
|
+
const params = [];
|
|
15884
|
+
if (updates.status !== undefined) {
|
|
15885
|
+
sets.push("status = ?");
|
|
15886
|
+
params.push(updates.status);
|
|
15887
|
+
}
|
|
15888
|
+
if (updates.toolResult !== undefined) {
|
|
15889
|
+
sets.push("tool_result = ?");
|
|
15890
|
+
params.push(updates.toolResult);
|
|
15891
|
+
}
|
|
15892
|
+
if (updates.error !== undefined) {
|
|
15893
|
+
sets.push("error = ?");
|
|
15894
|
+
params.push(updates.error);
|
|
15895
|
+
}
|
|
15896
|
+
if (updates.durationMs !== undefined) {
|
|
15897
|
+
sets.push("duration_ms = ?");
|
|
15898
|
+
params.push(updates.durationMs);
|
|
15899
|
+
}
|
|
15900
|
+
if (updates.screenshotId !== undefined) {
|
|
15901
|
+
sets.push("screenshot_id = ?");
|
|
15902
|
+
params.push(updates.screenshotId);
|
|
15903
|
+
}
|
|
15904
|
+
if (sets.length === 0)
|
|
15905
|
+
return existing;
|
|
15906
|
+
params.push(id);
|
|
15907
|
+
db2.query(`UPDATE step_results SET ${sets.join(", ")} WHERE id = ?`).run(...params);
|
|
15908
|
+
return getStepResult(id);
|
|
15909
|
+
}
|
|
15910
|
+
function stepResultFromRow(row) {
|
|
15911
|
+
return {
|
|
15912
|
+
id: row.id,
|
|
15913
|
+
resultId: row.result_id,
|
|
15914
|
+
stepNumber: row.step_number,
|
|
15915
|
+
action: row.action,
|
|
15916
|
+
status: row.status,
|
|
15917
|
+
toolName: row.tool_name,
|
|
15918
|
+
toolInput: row.tool_input ? JSON.parse(row.tool_input) : null,
|
|
15919
|
+
toolResult: row.tool_result,
|
|
15920
|
+
thinking: row.thinking,
|
|
15921
|
+
error: row.error,
|
|
15922
|
+
durationMs: row.duration_ms,
|
|
15923
|
+
screenshotId: row.screenshot_id,
|
|
15924
|
+
createdAt: row.created_at
|
|
15925
|
+
};
|
|
15926
|
+
}
|
|
15927
|
+
var init_step_results = __esm(() => {
|
|
15928
|
+
init_database();
|
|
15929
|
+
});
|
|
15930
|
+
|
|
15660
15931
|
// src/db/personas.ts
|
|
15661
15932
|
function createPersona(input) {
|
|
15662
15933
|
const db2 = getDatabase();
|
|
@@ -15664,9 +15935,9 @@ function createPersona(input) {
|
|
|
15664
15935
|
const short_id = shortUuid();
|
|
15665
15936
|
const timestamp = now();
|
|
15666
15937
|
db2.query(`
|
|
15667
|
-
INSERT INTO personas (id, short_id, project_id, name, description, role, instructions, traits, goals, behaviors, expertise_level, demographics, pain_points, metadata, enabled, auth_email, auth_password, auth_login_path, version, created_at, updated_at)
|
|
15668
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1, ?, ?)
|
|
15669
|
-
`).run(id, short_id, input.projectId ?? null, input.name, input.description ?? "", input.role, input.instructions ?? "", JSON.stringify(input.traits ?? []), JSON.stringify(input.goals ?? []), JSON.stringify(input.behaviors ?? []), input.expertiseLevel ?? "intermediate", JSON.stringify(input.demographics ?? {}), JSON.stringify(input.painPoints ?? []), input.metadata ? JSON.stringify(input.metadata) : "{}", input.enabled === false ? 0 : 1, input.authEmail ?? null, input.authPassword ?? null, input.authLoginPath ?? null, timestamp, timestamp);
|
|
15938
|
+
INSERT INTO personas (id, short_id, project_id, name, description, role, instructions, traits, goals, behaviors, expertise_level, demographics, pain_points, metadata, enabled, auth_email, auth_password, auth_login_path, auth_cookies, auth_strategy, auth_headers, auth_script, version, created_at, updated_at)
|
|
15939
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1, ?, ?)
|
|
15940
|
+
`).run(id, short_id, input.projectId ?? null, input.name, input.description ?? "", input.role, input.instructions ?? "", JSON.stringify(input.traits ?? []), JSON.stringify(input.goals ?? []), JSON.stringify(input.behaviors ?? []), input.expertiseLevel ?? "intermediate", JSON.stringify(input.demographics ?? {}), JSON.stringify(input.painPoints ?? []), input.metadata ? JSON.stringify(input.metadata) : "{}", input.enabled === false ? 0 : 1, input.authEmail ?? null, input.authPassword ?? null, input.authLoginPath ?? null, null, input.authStrategy ?? "form-login", input.authHeaders ? JSON.stringify(input.authHeaders) : null, input.authCustomScript ?? null, timestamp, timestamp);
|
|
15670
15941
|
return getPersona(id);
|
|
15671
15942
|
}
|
|
15672
15943
|
function getPersona(id) {
|
|
@@ -15953,6 +16224,11 @@ function resolveCredential(value) {
|
|
|
15953
16224
|
}
|
|
15954
16225
|
return value;
|
|
15955
16226
|
}
|
|
16227
|
+
function isCredentialReference(value) {
|
|
16228
|
+
if (!value)
|
|
16229
|
+
return false;
|
|
16230
|
+
return value.startsWith("@secrets:") || value.startsWith("$");
|
|
16231
|
+
}
|
|
15956
16232
|
var init_secrets_resolver = () => {};
|
|
15957
16233
|
|
|
15958
16234
|
// src/lib/persona-auth.ts
|
|
@@ -16215,6 +16491,24 @@ function signPayload(body, secret) {
|
|
|
16215
16491
|
}
|
|
16216
16492
|
return `sha256=${Math.abs(hash).toString(16).padStart(16, "0")}`;
|
|
16217
16493
|
}
|
|
16494
|
+
function formatDiscordPayload(payload) {
|
|
16495
|
+
const isPassed = payload.run.status === "passed";
|
|
16496
|
+
const color = isPassed ? 2278750 : 15680580;
|
|
16497
|
+
return {
|
|
16498
|
+
username: "open-testers",
|
|
16499
|
+
embeds: [
|
|
16500
|
+
{
|
|
16501
|
+
title: `Test Run ${payload.run.status.toUpperCase()}`,
|
|
16502
|
+
color,
|
|
16503
|
+
description: `URL: ${payload.run.url}
|
|
16504
|
+
` + `Results: ${payload.run.passed}/${payload.run.total} passed` + (payload.run.failed > 0 ? ` (${payload.run.failed} failed)` : "") + (payload.schedule ? `
|
|
16505
|
+
Schedule: ${payload.schedule.name}` : ""),
|
|
16506
|
+
timestamp: payload.timestamp,
|
|
16507
|
+
footer: { text: "open-testers" }
|
|
16508
|
+
}
|
|
16509
|
+
]
|
|
16510
|
+
};
|
|
16511
|
+
}
|
|
16218
16512
|
function formatSlackPayload(payload) {
|
|
16219
16513
|
const status = payload.run.status === "passed" ? ":white_check_mark:" : ":x:";
|
|
16220
16514
|
const color = payload.run.status === "passed" ? "#22c55e" : "#ef4444";
|
|
@@ -16257,7 +16551,8 @@ async function dispatchWebhooks(event, run, schedule) {
|
|
|
16257
16551
|
if (!webhook.events.includes(event) && !webhook.events.includes("*"))
|
|
16258
16552
|
continue;
|
|
16259
16553
|
const isSlack = webhook.url.includes("hooks.slack.com");
|
|
16260
|
-
const
|
|
16554
|
+
const isDiscord = webhook.url.includes("discord.com/api/webhooks") || webhook.url.includes("discordapp.com/api/webhooks");
|
|
16555
|
+
const body = isSlack ? JSON.stringify(formatSlackPayload(payload)) : isDiscord ? JSON.stringify(formatDiscordPayload(payload)) : JSON.stringify(payload);
|
|
16261
16556
|
const headers = {
|
|
16262
16557
|
"Content-Type": "application/json"
|
|
16263
16558
|
};
|
|
@@ -16744,6 +17039,8 @@ __export(exports_runner, {
|
|
|
16744
17039
|
runBatch: () => runBatch,
|
|
16745
17040
|
onRunEvent: () => onRunEvent
|
|
16746
17041
|
});
|
|
17042
|
+
import { mkdirSync as mkdirSync8 } from "fs";
|
|
17043
|
+
import { join as join13 } from "path";
|
|
16747
17044
|
import { enableNetworkLogging } from "@hasna/browser";
|
|
16748
17045
|
function onRunEvent(handler) {
|
|
16749
17046
|
eventHandler = handler;
|
|
@@ -16829,13 +17126,35 @@ async function runSingleScenario(scenario, runId, options) {
|
|
|
16829
17126
|
emit({ type: "scenario:start", scenarioId: scenario.id, scenarioName: scenario.name, resultId: result.id, runId });
|
|
16830
17127
|
let browser = null;
|
|
16831
17128
|
let page = null;
|
|
17129
|
+
let context = null;
|
|
17130
|
+
let harPath = null;
|
|
16832
17131
|
let stopNetworkLogging = null;
|
|
16833
17132
|
const networkErrors = [];
|
|
16834
17133
|
try {
|
|
16835
17134
|
browser = await launchBrowser({ headless: !(effectiveOptions.headed ?? false), engine: effectiveOptions.engine });
|
|
16836
|
-
|
|
16837
|
-
|
|
16838
|
-
|
|
17135
|
+
const useHar = effectiveOptions.engine !== "lightpanda" && effectiveOptions.engine !== "bun";
|
|
17136
|
+
if (useHar) {
|
|
17137
|
+
const testersDir = process.env["HASNA_TESTERS_DIR"] || join13(process.env["HOME"] || "", ".hasna", "testers");
|
|
17138
|
+
const harDir = join13(testersDir, "hars");
|
|
17139
|
+
mkdirSync8(harDir, { recursive: true });
|
|
17140
|
+
harPath = join13(harDir, `${result.id}.har`);
|
|
17141
|
+
const contextOptions = {
|
|
17142
|
+
viewport: config.browser.viewport,
|
|
17143
|
+
recordHar: { path: harPath, mode: "full" }
|
|
17144
|
+
};
|
|
17145
|
+
if (effectiveOptions.recordVideo) {
|
|
17146
|
+
const videoDir = join13(testersDir, "videos");
|
|
17147
|
+
mkdirSync8(videoDir, { recursive: true });
|
|
17148
|
+
contextOptions.recordVideo = { dir: videoDir, size: config.browser.viewport };
|
|
17149
|
+
}
|
|
17150
|
+
context = await browser.newContext(contextOptions);
|
|
17151
|
+
page = await context.newPage();
|
|
17152
|
+
} else {
|
|
17153
|
+
page = await getPage(browser, {
|
|
17154
|
+
viewport: config.browser.viewport,
|
|
17155
|
+
engine: effectiveOptions.engine
|
|
17156
|
+
});
|
|
17157
|
+
}
|
|
16839
17158
|
const targetUrl = scenario.targetPath ? `${options.url.replace(/\/$/, "")}${scenario.targetPath}` : options.url;
|
|
16840
17159
|
const scenarioTimeout = scenario.timeoutMs ?? options.timeout ?? config.browser.timeout ?? 60000;
|
|
16841
17160
|
registerSession({
|
|
@@ -16855,7 +17174,11 @@ async function runSingleScenario(scenario, runId, options) {
|
|
|
16855
17174
|
}
|
|
16856
17175
|
});
|
|
16857
17176
|
const consoleErrors = [];
|
|
17177
|
+
const consoleLogs = [];
|
|
17178
|
+
let currentStep = 0;
|
|
16858
17179
|
page.on("console", (msg) => {
|
|
17180
|
+
const logEntry = { step: currentStep > 0 ? currentStep : null, type: msg.type(), text: msg.text(), timestamp: Date.now() };
|
|
17181
|
+
consoleLogs.push(logEntry);
|
|
16859
17182
|
if (msg.type() === "error")
|
|
16860
17183
|
consoleErrors.push(msg.text());
|
|
16861
17184
|
});
|
|
@@ -16887,6 +17210,7 @@ async function runSingleScenario(scenario, runId, options) {
|
|
|
16887
17210
|
}
|
|
16888
17211
|
await page.goto(targetUrl, { timeout: Math.min(scenarioTimeout, 30000) });
|
|
16889
17212
|
const stepStartTimes = new Map;
|
|
17213
|
+
const stepResultIds = new Map;
|
|
16890
17214
|
const agentResult = await withTimeout(runAgentLoop({
|
|
16891
17215
|
client,
|
|
16892
17216
|
page,
|
|
@@ -16909,13 +17233,32 @@ async function runSingleScenario(scenario, runId, options) {
|
|
|
16909
17233
|
onStep: (stepEvent) => {
|
|
16910
17234
|
let stepDurationMs;
|
|
16911
17235
|
if (stepEvent.type === "tool_call") {
|
|
17236
|
+
currentStep = stepEvent.stepNumber;
|
|
16912
17237
|
stepStartTimes.set(stepEvent.stepNumber, Date.now());
|
|
17238
|
+
const stepResult = createStepResult({
|
|
17239
|
+
resultId: result.id,
|
|
17240
|
+
stepNumber: stepEvent.stepNumber,
|
|
17241
|
+
action: stepEvent.toolName ?? `step-${stepEvent.stepNumber}`,
|
|
17242
|
+
toolName: stepEvent.toolName,
|
|
17243
|
+
toolInput: stepEvent.toolInput,
|
|
17244
|
+
thinking: stepEvent.thinking
|
|
17245
|
+
});
|
|
17246
|
+
stepResultIds.set(stepEvent.stepNumber, stepResult.id);
|
|
16913
17247
|
} else if (stepEvent.type === "tool_result") {
|
|
16914
17248
|
const startTime = stepStartTimes.get(stepEvent.stepNumber);
|
|
16915
17249
|
if (startTime !== undefined) {
|
|
16916
17250
|
stepDurationMs = Date.now() - startTime;
|
|
16917
17251
|
stepStartTimes.delete(stepEvent.stepNumber);
|
|
16918
17252
|
}
|
|
17253
|
+
const stepResultId = stepResultIds.get(stepEvent.stepNumber);
|
|
17254
|
+
if (stepResultId) {
|
|
17255
|
+
const isSuccess = !stepEvent.toolResult?.toLowerCase().includes("error") && !stepEvent.toolResult?.toLowerCase().includes("failed");
|
|
17256
|
+
updateStepResult(stepResultId, {
|
|
17257
|
+
status: isSuccess ? "passed" : "failed",
|
|
17258
|
+
toolResult: stepEvent.toolResult,
|
|
17259
|
+
durationMs: stepDurationMs
|
|
17260
|
+
});
|
|
17261
|
+
}
|
|
16919
17262
|
}
|
|
16920
17263
|
emit({
|
|
16921
17264
|
type: `step:${stepEvent.type}`,
|
|
@@ -16964,7 +17307,7 @@ async function runSingleScenario(scenario, runId, options) {
|
|
|
16964
17307
|
durationMs: Date.now() - new Date(result.createdAt).getTime(),
|
|
16965
17308
|
tokensUsed: agentResult.tokensUsed,
|
|
16966
17309
|
costCents: estimateCost(model, agentResult.tokensUsed),
|
|
16967
|
-
metadata: networkErrors.length > 0 ? networkMeta :
|
|
17310
|
+
metadata: { consoleLogs, ...networkErrors.length > 0 ? networkMeta : {} }
|
|
16968
17311
|
});
|
|
16969
17312
|
if (agentResult.status === "failed" || agentResult.status === "error") {
|
|
16970
17313
|
const failureAnalysis = analyzeFailure(null, agentResult.reasoning ?? null);
|
|
@@ -17000,8 +17343,16 @@ async function runSingleScenario(scenario, runId, options) {
|
|
|
17000
17343
|
emit({ type: "scenario:error", scenarioId: scenario.id, scenarioName: scenario.name, error: errorMsg, runId });
|
|
17001
17344
|
return updatedResult;
|
|
17002
17345
|
} finally {
|
|
17003
|
-
if (
|
|
17004
|
-
|
|
17346
|
+
if (harPath) {
|
|
17347
|
+
try {
|
|
17348
|
+
updateResult(result.id, { metadata: { harPath } });
|
|
17349
|
+
} catch {}
|
|
17350
|
+
}
|
|
17351
|
+
if (browser) {
|
|
17352
|
+
try {
|
|
17353
|
+
await closeBrowser(browser, effectiveOptions.engine);
|
|
17354
|
+
} catch {}
|
|
17355
|
+
}
|
|
17005
17356
|
}
|
|
17006
17357
|
}
|
|
17007
17358
|
async function runBatch(scenarios, options) {
|
|
@@ -17297,6 +17648,7 @@ var init_runner = __esm(() => {
|
|
|
17297
17648
|
init_results();
|
|
17298
17649
|
init_costs();
|
|
17299
17650
|
init_screenshots();
|
|
17651
|
+
init_step_results();
|
|
17300
17652
|
init_scenarios();
|
|
17301
17653
|
init_personas();
|
|
17302
17654
|
init_browser();
|
|
@@ -24387,8 +24739,10 @@ var init_pii = __esm(() => {
|
|
|
24387
24739
|
// src/lib/ci.ts
|
|
24388
24740
|
var exports_ci = {};
|
|
24389
24741
|
__export(exports_ci, {
|
|
24742
|
+
resolvePullRequestNumber: () => resolvePullRequestNumber,
|
|
24390
24743
|
postGitHubComment: () => postGitHubComment,
|
|
24391
|
-
generateGitHubActionsWorkflow: () => generateGitHubActionsWorkflow
|
|
24744
|
+
generateGitHubActionsWorkflow: () => generateGitHubActionsWorkflow,
|
|
24745
|
+
formatPRComment: () => formatPRComment
|
|
24392
24746
|
});
|
|
24393
24747
|
function generateGitHubActionsWorkflow() {
|
|
24394
24748
|
return `name: AI QA Tests
|
|
@@ -24397,6 +24751,10 @@ on:
|
|
|
24397
24751
|
push:
|
|
24398
24752
|
branches: [main]
|
|
24399
24753
|
|
|
24754
|
+
permissions:
|
|
24755
|
+
contents: read
|
|
24756
|
+
pull-requests: write
|
|
24757
|
+
|
|
24400
24758
|
jobs:
|
|
24401
24759
|
test:
|
|
24402
24760
|
runs-on: ubuntu-latest
|
|
@@ -24411,6 +24769,7 @@ jobs:
|
|
|
24411
24769
|
TEST_URL: \${{ vars.TEST_URL || 'http://localhost:3000' }}
|
|
24412
24770
|
GITHUB_TOKEN: \${{ secrets.GITHUB_TOKEN }}
|
|
24413
24771
|
- run: testers report --latest --output report.html
|
|
24772
|
+
if: always()
|
|
24414
24773
|
- uses: actions/upload-artifact@v4
|
|
24415
24774
|
if: always()
|
|
24416
24775
|
with:
|
|
@@ -24421,46 +24780,73 @@ jobs:
|
|
|
24421
24780
|
`;
|
|
24422
24781
|
}
|
|
24423
24782
|
function formatPRComment(run, results, dashboardUrl) {
|
|
24424
|
-
const icon = run.status === "passed" ? "\u2705" : "\u274C";
|
|
24783
|
+
const icon = run.status === "passed" ? "\u2705" : run.status === "failed" ? "\u274C" : "\u26A0\uFE0F";
|
|
24425
24784
|
const passRate = run.total > 0 ? Math.round(run.passed / run.total * 100) : 0;
|
|
24426
|
-
const
|
|
24427
|
-
const
|
|
24785
|
+
const ordered = [...results].sort((a, b) => {
|
|
24786
|
+
const rank = (s) => s === "failed" ? 0 : s === "error" ? 1 : s === "flaky" ? 2 : s === "skipped" ? 3 : 4;
|
|
24787
|
+
return rank(a.status) - rank(b.status);
|
|
24788
|
+
});
|
|
24789
|
+
const MAX_ROWS = 20;
|
|
24790
|
+
const rows = ordered.slice(0, MAX_ROWS).map((r) => {
|
|
24791
|
+
const rowIcon = r.status === "passed" ? "\u2705" : r.status === "failed" ? "\u274C" : r.status === "error" ? "\u26A0\uFE0F" : r.status === "flaky" ? "\uD83D\uDFE1" : "\u23ED\uFE0F";
|
|
24428
24792
|
const dur = r.durationMs > 0 ? `${(r.durationMs / 1000).toFixed(1)}s` : "\u2014";
|
|
24429
|
-
const
|
|
24430
|
-
|
|
24793
|
+
const scenario = (() => {
|
|
24794
|
+
try {
|
|
24795
|
+
return getScenario(r.scenarioId);
|
|
24796
|
+
} catch {
|
|
24797
|
+
return null;
|
|
24798
|
+
}
|
|
24799
|
+
})();
|
|
24800
|
+
const name = scenario ? scenario.name : r.scenarioId.slice(0, 8);
|
|
24801
|
+
const safeName = name.replace(/\|/g, "\\|");
|
|
24802
|
+
const errSource = r.error ?? r.reasoning ?? "";
|
|
24803
|
+
const err = errSource ? ` ${errSource.replace(/\s+/g, " ").slice(0, 140).replace(/\|/g, "\\|")}` : "";
|
|
24804
|
+
return `| ${rowIcon} | ${safeName} | ${r.status} | ${dur} |${err} |`;
|
|
24431
24805
|
}).join(`
|
|
24432
24806
|
`);
|
|
24433
|
-
const truncated = results.length >
|
|
24434
|
-
_...and ${results.length -
|
|
24807
|
+
const truncated = results.length > MAX_ROWS ? `
|
|
24808
|
+
_...and ${results.length - MAX_ROWS} more_` : "";
|
|
24435
24809
|
const dashLink = dashboardUrl ? `
|
|
24436
24810
|
|
|
24437
24811
|
[View full report \u2192](${dashboardUrl}/runs/${run.id})` : "";
|
|
24812
|
+
const totalCostCents = results.reduce((sum, r) => sum + (r.costCents ?? 0), 0);
|
|
24813
|
+
const costStr = totalCostCents > 0 ? ` \xB7 $${(totalCostCents / 100).toFixed(4)}` : "";
|
|
24814
|
+
const headerRow = `| | Scenario | Status | Duration | Details |
|
|
24815
|
+
|---|---|---|---|---|`;
|
|
24816
|
+
const body = results.length > 0 ? `${headerRow}
|
|
24817
|
+
${rows}${truncated}` : `_No scenarios ran. Use \`testers add\` to create scenarios or run with a URL to auto-generate them._`;
|
|
24438
24818
|
return `## ${icon} AI QA Tests \u2014 ${run.status.toUpperCase()}
|
|
24439
24819
|
|
|
24440
|
-
**${run.passed}/${run.total} passed** (${passRate}%) \xB7 URL: \`${run.url}
|
|
24820
|
+
**${run.passed}/${run.total} passed** (${passRate}%) \xB7 URL: \`${run.url}\`${costStr}
|
|
24441
24821
|
|
|
24442
|
-
|
|
24443
|
-
|---|---|---|
|
|
24444
|
-
${rows}
|
|
24445
|
-
${truncated}${dashLink}
|
|
24822
|
+
${body}${dashLink}
|
|
24446
24823
|
|
|
24447
24824
|
_Generated by [@hasna/testers](https://www.npmjs.com/package/@hasna/testers)_`;
|
|
24448
24825
|
}
|
|
24826
|
+
function resolvePullRequestNumber(explicit) {
|
|
24827
|
+
if (explicit && Number.isFinite(explicit))
|
|
24828
|
+
return explicit;
|
|
24829
|
+
const fromEnv = process.env["GITHUB_PR_NUMBER"];
|
|
24830
|
+
if (fromEnv) {
|
|
24831
|
+
const parsed = parseInt(fromEnv, 10);
|
|
24832
|
+
if (Number.isFinite(parsed))
|
|
24833
|
+
return parsed;
|
|
24834
|
+
}
|
|
24835
|
+
const ref = process.env["GITHUB_REF"] ?? "";
|
|
24836
|
+
const match = ref.match(/refs\/pull\/(\d+)\//);
|
|
24837
|
+
if (match && match[1]) {
|
|
24838
|
+
const parsed = parseInt(match[1], 10);
|
|
24839
|
+
if (Number.isFinite(parsed))
|
|
24840
|
+
return parsed;
|
|
24841
|
+
}
|
|
24842
|
+
return null;
|
|
24843
|
+
}
|
|
24449
24844
|
async function postGitHubComment(run, results, options) {
|
|
24450
24845
|
const token = process.env["GITHUB_TOKEN"];
|
|
24451
24846
|
if (!token)
|
|
24452
24847
|
return false;
|
|
24453
|
-
|
|
24454
|
-
if (
|
|
24455
|
-
prNumber = parseInt(process.env["GITHUB_PR_NUMBER"], 10);
|
|
24456
|
-
}
|
|
24457
|
-
if (!prNumber) {
|
|
24458
|
-
const ref = process.env["GITHUB_REF"] ?? "";
|
|
24459
|
-
const match = ref.match(/refs\/pull\/(\d+)\//);
|
|
24460
|
-
if (match)
|
|
24461
|
-
prNumber = parseInt(match[1], 10);
|
|
24462
|
-
}
|
|
24463
|
-
if (!prNumber)
|
|
24848
|
+
const prNumber = resolvePullRequestNumber(options?.prNumber);
|
|
24849
|
+
if (prNumber === null)
|
|
24464
24850
|
return false;
|
|
24465
24851
|
const repo = process.env["GITHUB_REPOSITORY"];
|
|
24466
24852
|
if (!repo)
|
|
@@ -24483,6 +24869,235 @@ async function postGitHubComment(run, results, options) {
|
|
|
24483
24869
|
return false;
|
|
24484
24870
|
}
|
|
24485
24871
|
}
|
|
24872
|
+
var init_ci = __esm(() => {
|
|
24873
|
+
init_scenarios();
|
|
24874
|
+
});
|
|
24875
|
+
|
|
24876
|
+
// src/lib/crawl-and-generate.ts
|
|
24877
|
+
var exports_crawl_and_generate = {};
|
|
24878
|
+
__export(exports_crawl_and_generate, {
|
|
24879
|
+
crawlAndGenerate: () => crawlAndGenerate
|
|
24880
|
+
});
|
|
24881
|
+
function shouldSkip(href, rootOrigin, skipPaths) {
|
|
24882
|
+
try {
|
|
24883
|
+
const u = new URL(href);
|
|
24884
|
+
if (u.origin !== rootOrigin)
|
|
24885
|
+
return true;
|
|
24886
|
+
const path = u.pathname;
|
|
24887
|
+
const allSkip = [...DEFAULT_SKIP_PATTERNS, ...skipPaths];
|
|
24888
|
+
return allSkip.some((p) => path.startsWith(p) || path.includes(p));
|
|
24889
|
+
} catch {
|
|
24890
|
+
return true;
|
|
24891
|
+
}
|
|
24892
|
+
}
|
|
24893
|
+
function normaliseUrl(href) {
|
|
24894
|
+
try {
|
|
24895
|
+
const u = new URL(href);
|
|
24896
|
+
return `${u.origin}${u.pathname}`;
|
|
24897
|
+
} catch {
|
|
24898
|
+
return href;
|
|
24899
|
+
}
|
|
24900
|
+
}
|
|
24901
|
+
async function getPageContext(browser, pageUrl, timeoutMs) {
|
|
24902
|
+
const page = await getPage(browser, {});
|
|
24903
|
+
try {
|
|
24904
|
+
await page.goto(pageUrl, { timeout: timeoutMs, waitUntil: "domcontentloaded" });
|
|
24905
|
+
await page.waitForTimeout(800).catch(() => {});
|
|
24906
|
+
const [title, html, links, screenshot] = await Promise.all([
|
|
24907
|
+
page.title().catch(() => ""),
|
|
24908
|
+
page.evaluate(() => {
|
|
24909
|
+
const body = document.body;
|
|
24910
|
+
if (!body)
|
|
24911
|
+
return "";
|
|
24912
|
+
const clone = body.cloneNode(true);
|
|
24913
|
+
clone.querySelectorAll("script,style,svg,noscript,iframe").forEach((el) => el.remove());
|
|
24914
|
+
return clone.innerText?.slice(0, 3000) ?? clone.textContent?.slice(0, 3000) ?? "";
|
|
24915
|
+
}).catch(() => ""),
|
|
24916
|
+
page.evaluate((origin) => Array.from(document.querySelectorAll("a[href]")).map((a) => a.href).filter((h) => {
|
|
24917
|
+
try {
|
|
24918
|
+
return new URL(h).origin === origin;
|
|
24919
|
+
} catch {
|
|
24920
|
+
return false;
|
|
24921
|
+
}
|
|
24922
|
+
}), new URL(pageUrl).origin).catch(() => []),
|
|
24923
|
+
page.screenshot({ fullPage: false }).catch(() => null)
|
|
24924
|
+
]);
|
|
24925
|
+
return { title, path: new URL(pageUrl).pathname, html, screenshot, links };
|
|
24926
|
+
} finally {
|
|
24927
|
+
await page.close().catch(() => {});
|
|
24928
|
+
}
|
|
24929
|
+
}
|
|
24930
|
+
async function generateScenariosForPage(client, model, pageContext, baseUrl, count) {
|
|
24931
|
+
const Anthropic4 = (await import("@anthropic-ai/sdk")).default;
|
|
24932
|
+
const anthropicClient = client;
|
|
24933
|
+
const pageDesc = [
|
|
24934
|
+
`URL: ${baseUrl.replace(/\/$/, "")}${pageContext.path}`,
|
|
24935
|
+
`Title: ${pageContext.title || pageContext.path}`,
|
|
24936
|
+
pageContext.html ? `
|
|
24937
|
+
Page content (text):
|
|
24938
|
+
${pageContext.html.slice(0, 2000)}` : ""
|
|
24939
|
+
].filter(Boolean).join(`
|
|
24940
|
+
`);
|
|
24941
|
+
const prompt = `You are a QA engineer. Analyze this web page and write ${count} practical test scenarios.
|
|
24942
|
+
|
|
24943
|
+
${pageDesc}
|
|
24944
|
+
|
|
24945
|
+
Return ONLY a JSON array (no markdown, no explanation). Each scenario:
|
|
24946
|
+
{
|
|
24947
|
+
"name": "short action-oriented name (e.g. 'User can log in with valid credentials')",
|
|
24948
|
+
"description": "what this test verifies",
|
|
24949
|
+
"steps": ["step 1", "step 2", "step 3"],
|
|
24950
|
+
"tags": ["tag1"],
|
|
24951
|
+
"priority": "low|medium|high|critical"
|
|
24952
|
+
}
|
|
24953
|
+
|
|
24954
|
+
Rules:
|
|
24955
|
+
- Focus on user flows, not implementation details
|
|
24956
|
+
- Steps should be plain English instructions the browser agent can follow
|
|
24957
|
+
- Vary priorities: 1 critical/high per page for the main flow, rest medium/low
|
|
24958
|
+
- Keep steps concise (max 8 per scenario)
|
|
24959
|
+
- Tags should reflect the page area (e.g. "auth", "dashboard", "settings", "checkout")`;
|
|
24960
|
+
const contentParts = [
|
|
24961
|
+
...pageContext.screenshot ? [{
|
|
24962
|
+
type: "image",
|
|
24963
|
+
source: {
|
|
24964
|
+
type: "base64",
|
|
24965
|
+
media_type: "image/png",
|
|
24966
|
+
data: pageContext.screenshot.toString("base64")
|
|
24967
|
+
}
|
|
24968
|
+
}] : [],
|
|
24969
|
+
{ type: "text", text: prompt }
|
|
24970
|
+
];
|
|
24971
|
+
const messages = [{ role: "user", content: contentParts }];
|
|
24972
|
+
try {
|
|
24973
|
+
const response = await anthropicClient.messages.create({
|
|
24974
|
+
model,
|
|
24975
|
+
max_tokens: 2048,
|
|
24976
|
+
messages
|
|
24977
|
+
});
|
|
24978
|
+
const text = response.content.filter((b) => b.type === "text").map((b) => b.text).join("");
|
|
24979
|
+
const match = text.match(/\[[\s\S]*\]/);
|
|
24980
|
+
if (!match)
|
|
24981
|
+
return [];
|
|
24982
|
+
const parsed = JSON.parse(match[0]);
|
|
24983
|
+
return parsed.map((s) => ({
|
|
24984
|
+
name: s.name ?? "Untitled scenario",
|
|
24985
|
+
description: s.description ?? "",
|
|
24986
|
+
steps: s.steps ?? [],
|
|
24987
|
+
tags: s.tags ?? [],
|
|
24988
|
+
priority: s.priority ?? "medium"
|
|
24989
|
+
}));
|
|
24990
|
+
} catch {
|
|
24991
|
+
return [];
|
|
24992
|
+
}
|
|
24993
|
+
}
|
|
24994
|
+
async function crawlAndGenerate(options) {
|
|
24995
|
+
const {
|
|
24996
|
+
url,
|
|
24997
|
+
projectId,
|
|
24998
|
+
maxPages = 20,
|
|
24999
|
+
scenariosPerPage = 3,
|
|
25000
|
+
headed = false,
|
|
25001
|
+
skipPaths = [],
|
|
25002
|
+
tags: extraTags = []
|
|
25003
|
+
} = options;
|
|
25004
|
+
const config = loadConfig();
|
|
25005
|
+
const model = resolveModel(options.model ?? config.defaultModel ?? "thorough");
|
|
25006
|
+
const client = createClient(options.apiKey ?? config.anthropicApiKey);
|
|
25007
|
+
const rootOrigin = new URL(url).origin;
|
|
25008
|
+
const visited = new Set;
|
|
25009
|
+
const queue = [url];
|
|
25010
|
+
const pageContexts = [];
|
|
25011
|
+
const skipped = [];
|
|
25012
|
+
const browser = await launchBrowser({ headless: !headed });
|
|
25013
|
+
try {
|
|
25014
|
+
while (queue.length > 0 && visited.size < maxPages) {
|
|
25015
|
+
const pageUrl = queue.shift();
|
|
25016
|
+
const norm = normaliseUrl(pageUrl);
|
|
25017
|
+
if (visited.has(norm))
|
|
25018
|
+
continue;
|
|
25019
|
+
if (shouldSkip(pageUrl, rootOrigin, skipPaths)) {
|
|
25020
|
+
skipped.push(pageUrl);
|
|
25021
|
+
continue;
|
|
25022
|
+
}
|
|
25023
|
+
visited.add(norm);
|
|
25024
|
+
try {
|
|
25025
|
+
const ctx = await getPageContext(browser, pageUrl, 15000);
|
|
25026
|
+
pageContexts.push(ctx);
|
|
25027
|
+
for (const link of ctx.links) {
|
|
25028
|
+
const normLink = normaliseUrl(link);
|
|
25029
|
+
if (!visited.has(normLink) && !shouldSkip(link, rootOrigin, skipPaths)) {
|
|
25030
|
+
queue.push(link);
|
|
25031
|
+
}
|
|
25032
|
+
}
|
|
25033
|
+
} catch {
|
|
25034
|
+
skipped.push(pageUrl);
|
|
25035
|
+
}
|
|
25036
|
+
}
|
|
25037
|
+
} finally {
|
|
25038
|
+
await closeBrowser(browser).catch(() => {});
|
|
25039
|
+
}
|
|
25040
|
+
const pages = [];
|
|
25041
|
+
let totalCreated = 0;
|
|
25042
|
+
for (const ctx of pageContexts) {
|
|
25043
|
+
const generated = await generateScenariosForPage(client, model, ctx, url, scenariosPerPage);
|
|
25044
|
+
const createdScenarios = [];
|
|
25045
|
+
for (const s of generated) {
|
|
25046
|
+
try {
|
|
25047
|
+
const priority = ["low", "medium", "high", "critical"].includes(s.priority) ? s.priority : "medium";
|
|
25048
|
+
const scenario = createScenario({
|
|
25049
|
+
name: s.name,
|
|
25050
|
+
description: s.description,
|
|
25051
|
+
steps: s.steps,
|
|
25052
|
+
tags: [...s.tags ?? [], ...extraTags, "generated"],
|
|
25053
|
+
priority,
|
|
25054
|
+
targetPath: ctx.path,
|
|
25055
|
+
projectId
|
|
25056
|
+
});
|
|
25057
|
+
createdScenarios.push({ id: scenario.id, shortId: scenario.shortId, name: scenario.name });
|
|
25058
|
+
totalCreated++;
|
|
25059
|
+
} catch {}
|
|
25060
|
+
}
|
|
25061
|
+
if (createdScenarios.length > 0) {
|
|
25062
|
+
pages.push({
|
|
25063
|
+
path: ctx.path,
|
|
25064
|
+
title: ctx.title,
|
|
25065
|
+
scenariosCreated: createdScenarios.length,
|
|
25066
|
+
scenarios: createdScenarios
|
|
25067
|
+
});
|
|
25068
|
+
}
|
|
25069
|
+
}
|
|
25070
|
+
return {
|
|
25071
|
+
projectId: projectId ?? null,
|
|
25072
|
+
url,
|
|
25073
|
+
pagesDiscovered: pageContexts.length,
|
|
25074
|
+
pagesGenerated: pages.length,
|
|
25075
|
+
totalScenariosCreated: totalCreated,
|
|
25076
|
+
pages,
|
|
25077
|
+
skipped
|
|
25078
|
+
};
|
|
25079
|
+
}
|
|
25080
|
+
var DEFAULT_SKIP_PATTERNS;
|
|
25081
|
+
var init_crawl_and_generate = __esm(() => {
|
|
25082
|
+
init_browser();
|
|
25083
|
+
init_scenarios();
|
|
25084
|
+
init_ai_client();
|
|
25085
|
+
init_config2();
|
|
25086
|
+
init_ai_client();
|
|
25087
|
+
DEFAULT_SKIP_PATTERNS = [
|
|
25088
|
+
"/logout",
|
|
25089
|
+
"/sign-out",
|
|
25090
|
+
"/signout",
|
|
25091
|
+
"/static/",
|
|
25092
|
+
"/assets/",
|
|
25093
|
+
"/_next/",
|
|
25094
|
+
"/__/",
|
|
25095
|
+
"/favicon",
|
|
25096
|
+
"/robots.txt",
|
|
25097
|
+
"/sitemap",
|
|
25098
|
+
"#"
|
|
25099
|
+
];
|
|
25100
|
+
});
|
|
24486
25101
|
|
|
24487
25102
|
// src/lib/affected.ts
|
|
24488
25103
|
var exports_affected = {};
|
|
@@ -25044,10 +25659,14 @@ var init_generator = __esm(() => {
|
|
|
25044
25659
|
// src/lib/recorder.ts
|
|
25045
25660
|
var exports_recorder = {};
|
|
25046
25661
|
__export(exports_recorder, {
|
|
25662
|
+
replayAuthState: () => replayAuthState,
|
|
25047
25663
|
recordSession: () => recordSession,
|
|
25664
|
+
recordAuthFlow: () => recordAuthFlow,
|
|
25048
25665
|
recordAndSave: () => recordAndSave,
|
|
25666
|
+
authStateToScenarioMetadata: () => authStateToScenarioMetadata,
|
|
25049
25667
|
actionsToScenarioInput: () => actionsToScenarioInput
|
|
25050
25668
|
});
|
|
25669
|
+
import { chromium as chromium2 } from "playwright";
|
|
25051
25670
|
import { startRecording, stopRecording } from "@hasna/browser";
|
|
25052
25671
|
import { launchPlaywright as launchPlaywright2 } from "@hasna/browser";
|
|
25053
25672
|
async function recordSession(url, options) {
|
|
@@ -25212,6 +25831,99 @@ async function recordAndSave(url, name, projectId) {
|
|
|
25212
25831
|
const scenario = createScenario(input);
|
|
25213
25832
|
return { recording, scenario };
|
|
25214
25833
|
}
|
|
25834
|
+
async function recordAuthFlow(loginUrl, options) {
|
|
25835
|
+
const browser = await chromium2.launch({ headless: false });
|
|
25836
|
+
const context = await browser.newContext({ viewport: { width: 1280, height: 720 } });
|
|
25837
|
+
const page = await context.newPage();
|
|
25838
|
+
const emailSelector = options.emailSelector ?? 'input[name="email"], input[type="email"], #email';
|
|
25839
|
+
const passwordSelector = options.passwordSelector ?? 'input[name="password"], input[type="password"], #password';
|
|
25840
|
+
const submitSelector = options.submitSelector ?? 'button[type="submit"], input[type="submit"]';
|
|
25841
|
+
try {
|
|
25842
|
+
await page.goto(loginUrl, { waitUntil: "domcontentloaded", timeout: options.timeoutMs ?? 30000 });
|
|
25843
|
+
await page.fill(emailSelector, options.email);
|
|
25844
|
+
await page.fill(passwordSelector, options.password);
|
|
25845
|
+
await page.click(submitSelector);
|
|
25846
|
+
if (options.waitForUrl) {
|
|
25847
|
+
await page.waitForURL(options.waitForUrl, { timeout: options.timeoutMs ?? 30000 });
|
|
25848
|
+
} else {
|
|
25849
|
+
await page.waitForLoadState("networkidle", { timeout: options.timeoutMs ?? 30000 });
|
|
25850
|
+
}
|
|
25851
|
+
const cookies = await context.cookies();
|
|
25852
|
+
const formattedCookies = cookies.map((c) => ({
|
|
25853
|
+
name: c.name,
|
|
25854
|
+
value: c.value,
|
|
25855
|
+
domain: c.domain || "",
|
|
25856
|
+
path: c.path || "/"
|
|
25857
|
+
}));
|
|
25858
|
+
const frames = page.frames();
|
|
25859
|
+
const localStorageEntries = [];
|
|
25860
|
+
for (const frame of frames) {
|
|
25861
|
+
try {
|
|
25862
|
+
const origin = frame.url();
|
|
25863
|
+
if (origin && origin !== "about:blank") {
|
|
25864
|
+
const entries = await frame.evaluate(() => {
|
|
25865
|
+
const items = [];
|
|
25866
|
+
for (let i = 0;i < localStorage.length; i++) {
|
|
25867
|
+
const key = localStorage.key(i);
|
|
25868
|
+
if (key)
|
|
25869
|
+
items.push({ name: key, value: localStorage.getItem(key) || "" });
|
|
25870
|
+
}
|
|
25871
|
+
return items;
|
|
25872
|
+
});
|
|
25873
|
+
if (entries.length > 0) {
|
|
25874
|
+
localStorageEntries.push({ origin, entries });
|
|
25875
|
+
}
|
|
25876
|
+
}
|
|
25877
|
+
} catch {}
|
|
25878
|
+
}
|
|
25879
|
+
return {
|
|
25880
|
+
cookies: formattedCookies,
|
|
25881
|
+
localStorage: localStorageEntries,
|
|
25882
|
+
loginUrl,
|
|
25883
|
+
recordedAt: new Date().toISOString()
|
|
25884
|
+
};
|
|
25885
|
+
} finally {
|
|
25886
|
+
await browser.close();
|
|
25887
|
+
}
|
|
25888
|
+
}
|
|
25889
|
+
async function replayAuthState(context, authState) {
|
|
25890
|
+
for (const cookie of authState.cookies) {
|
|
25891
|
+
try {
|
|
25892
|
+
await context.addCookies([{
|
|
25893
|
+
name: cookie.name,
|
|
25894
|
+
value: cookie.value,
|
|
25895
|
+
domain: cookie.domain,
|
|
25896
|
+
path: cookie.path,
|
|
25897
|
+
expires: -1
|
|
25898
|
+
}]);
|
|
25899
|
+
} catch {}
|
|
25900
|
+
}
|
|
25901
|
+
const page = await context.newPage();
|
|
25902
|
+
const origin = new URL(authState.loginUrl).origin;
|
|
25903
|
+
await page.goto(`${origin}/about:blank`, { waitUntil: "domcontentloaded" }).catch(() => {});
|
|
25904
|
+
for (const entry of authState.localStorage) {
|
|
25905
|
+
try {
|
|
25906
|
+
await page.evaluate((items) => {
|
|
25907
|
+
for (const item of items) {
|
|
25908
|
+
localStorage.setItem(item.name, item.value);
|
|
25909
|
+
}
|
|
25910
|
+
}, entry.entries);
|
|
25911
|
+
} catch {}
|
|
25912
|
+
}
|
|
25913
|
+
await page.close();
|
|
25914
|
+
}
|
|
25915
|
+
function authStateToScenarioMetadata(authState, name, projectId) {
|
|
25916
|
+
return createScenario({
|
|
25917
|
+
name,
|
|
25918
|
+
description: `Authenticated test scenario from recorded auth state at ${authState.loginUrl}`,
|
|
25919
|
+
steps: [`Navigate to authenticated session`],
|
|
25920
|
+
tags: ["auth", "recorded"],
|
|
25921
|
+
requiresAuth: true,
|
|
25922
|
+
authConfig: { loginPath: new URL(authState.loginUrl).pathname },
|
|
25923
|
+
metadata: { authState: JSON.parse(JSON.stringify(authState)) },
|
|
25924
|
+
projectId
|
|
25925
|
+
});
|
|
25926
|
+
}
|
|
25215
25927
|
var init_recorder = __esm(() => {
|
|
25216
25928
|
init_scenarios();
|
|
25217
25929
|
});
|
|
@@ -25683,7 +26395,7 @@ async function scanBrokenLinks(options) {
|
|
|
25683
26395
|
try {
|
|
25684
26396
|
while (queue.length > 0 && visited.size < maxPages) {
|
|
25685
26397
|
const { pageUrl, sourceUrl } = queue.shift();
|
|
25686
|
-
const normalised =
|
|
26398
|
+
const normalised = normaliseUrl2(pageUrl);
|
|
25687
26399
|
if (visited.has(normalised))
|
|
25688
26400
|
continue;
|
|
25689
26401
|
visited.add(normalised);
|
|
@@ -25748,7 +26460,7 @@ async function scanBrokenLinks(options) {
|
|
|
25748
26460
|
issues
|
|
25749
26461
|
};
|
|
25750
26462
|
}
|
|
25751
|
-
function
|
|
26463
|
+
function normaliseUrl2(rawUrl) {
|
|
25752
26464
|
try {
|
|
25753
26465
|
const u = new URL(rawUrl);
|
|
25754
26466
|
return `${u.origin}${u.pathname}`;
|
|
@@ -26175,6 +26887,16 @@ async function runHealthScan(options) {
|
|
|
26175
26887
|
});
|
|
26176
26888
|
results.push(piiResult);
|
|
26177
26889
|
}
|
|
26890
|
+
if (scanners.includes("a11y")) {
|
|
26891
|
+
const a11yResult = await scanA11y({
|
|
26892
|
+
url,
|
|
26893
|
+
pages,
|
|
26894
|
+
wcagLevel: options.wcagLevel ?? "AA",
|
|
26895
|
+
headed,
|
|
26896
|
+
timeoutMs
|
|
26897
|
+
});
|
|
26898
|
+
results.push(a11yResult);
|
|
26899
|
+
}
|
|
26178
26900
|
const allIssues = results.flatMap((r) => r.issues);
|
|
26179
26901
|
let newCount = 0;
|
|
26180
26902
|
let regressedCount = 0;
|
|
@@ -27028,7 +27750,8 @@ async function runHybridScenario(scenario, options) {
|
|
|
27028
27750
|
createdAt: new Date().toISOString(),
|
|
27029
27751
|
updatedAt: new Date().toISOString(),
|
|
27030
27752
|
lastPassedAt: null,
|
|
27031
|
-
lastPassedUrl: null
|
|
27753
|
+
lastPassedUrl: null,
|
|
27754
|
+
parameters: null
|
|
27032
27755
|
};
|
|
27033
27756
|
try {
|
|
27034
27757
|
const agentResult = await runAgentLoop({
|
|
@@ -27119,7 +27842,7 @@ import chalk6 from "chalk";
|
|
|
27119
27842
|
// package.json
|
|
27120
27843
|
var package_default = {
|
|
27121
27844
|
name: "@hasna/testers",
|
|
27122
|
-
version: "0.0.
|
|
27845
|
+
version: "0.0.30",
|
|
27123
27846
|
description: "AI-powered QA testing CLI \u2014 spawns cheap AI agents to test web apps with headless browsers",
|
|
27124
27847
|
type: "module",
|
|
27125
27848
|
main: "dist/index.js",
|
|
@@ -27184,7 +27907,7 @@ var package_default = {
|
|
|
27184
27907
|
},
|
|
27185
27908
|
repository: {
|
|
27186
27909
|
type: "git",
|
|
27187
|
-
url: "https://github.com/hasna/
|
|
27910
|
+
url: "https://github.com/hasna/testers.git"
|
|
27188
27911
|
},
|
|
27189
27912
|
license: "Apache-2.0",
|
|
27190
27913
|
keywords: [
|
|
@@ -27214,13 +27937,13 @@ import { render, Box, Text, useInput, useApp } from "ink";
|
|
|
27214
27937
|
import React, { useState } from "react";
|
|
27215
27938
|
import { readFileSync as readFileSync8, readdirSync as readdirSync3, writeFileSync as writeFileSync4 } from "fs";
|
|
27216
27939
|
import { createInterface } from "readline";
|
|
27217
|
-
import { join as
|
|
27940
|
+
import { join as join16, resolve } from "path";
|
|
27218
27941
|
|
|
27219
27942
|
// src/lib/init.ts
|
|
27220
27943
|
init_paths();
|
|
27221
27944
|
init_scenarios();
|
|
27222
|
-
import { existsSync as existsSync11, readFileSync as readFileSync3, writeFileSync as writeFileSync3, mkdirSync as
|
|
27223
|
-
import { join as
|
|
27945
|
+
import { existsSync as existsSync11, readFileSync as readFileSync3, writeFileSync as writeFileSync3, mkdirSync as mkdirSync9 } from "fs";
|
|
27946
|
+
import { join as join14, basename } from "path";
|
|
27224
27947
|
|
|
27225
27948
|
// src/db/projects.ts
|
|
27226
27949
|
init_types();
|
|
@@ -27258,7 +27981,7 @@ function ensureProject(name, path) {
|
|
|
27258
27981
|
|
|
27259
27982
|
// src/lib/init.ts
|
|
27260
27983
|
function detectFramework(dir) {
|
|
27261
|
-
const pkgPath =
|
|
27984
|
+
const pkgPath = join14(dir, "package.json");
|
|
27262
27985
|
if (!existsSync11(pkgPath))
|
|
27263
27986
|
return null;
|
|
27264
27987
|
let pkg;
|
|
@@ -27486,9 +28209,9 @@ function initProject(options) {
|
|
|
27486
28209
|
}
|
|
27487
28210
|
}).filter((s) => s !== null);
|
|
27488
28211
|
const configDir = getTestersDir();
|
|
27489
|
-
const configPath =
|
|
28212
|
+
const configPath = join14(configDir, "config.json");
|
|
27490
28213
|
if (!existsSync11(configDir)) {
|
|
27491
|
-
|
|
28214
|
+
mkdirSync9(configDir, { recursive: true });
|
|
27492
28215
|
}
|
|
27493
28216
|
let config = {};
|
|
27494
28217
|
if (existsSync11(configPath)) {
|
|
@@ -27880,8 +28603,8 @@ init_results();
|
|
|
27880
28603
|
init_runs();
|
|
27881
28604
|
init_scenarios();
|
|
27882
28605
|
init_database();
|
|
27883
|
-
import { readFileSync as readFileSync4, existsSync as existsSync12, mkdirSync as
|
|
27884
|
-
import { join as
|
|
28606
|
+
import { readFileSync as readFileSync4, existsSync as existsSync12, mkdirSync as mkdirSync10 } from "fs";
|
|
28607
|
+
import { join as join15, dirname as dirname3 } from "path";
|
|
27885
28608
|
import chalk4 from "chalk";
|
|
27886
28609
|
var DEFAULT_THRESHOLD = 0.1;
|
|
27887
28610
|
function setBaseline(runId) {
|
|
@@ -27936,8 +28659,8 @@ async function compareImages(image1Path, image2Path, options) {
|
|
|
27936
28659
|
let diffImagePath;
|
|
27937
28660
|
if (options?.saveDiff) {
|
|
27938
28661
|
const dir = options.diffDir ?? dirname3(image2Path);
|
|
27939
|
-
|
|
27940
|
-
diffImagePath =
|
|
28662
|
+
mkdirSync10(dir, { recursive: true });
|
|
28663
|
+
diffImagePath = join15(dir, `diff-${Date.now()}.png`);
|
|
27941
28664
|
await sharp.default(diffBuffer, { raw: { width: w, height: h, channels } }).png().toFile(diffImagePath);
|
|
27942
28665
|
}
|
|
27943
28666
|
return { diffPercent, diffPixels: changedPixels, totalPixels, diffImagePath };
|
|
@@ -28249,6 +28972,383 @@ function generateLatestReport() {
|
|
|
28249
28972
|
|
|
28250
28973
|
// src/cli/index.tsx
|
|
28251
28974
|
init_costs();
|
|
28975
|
+
|
|
28976
|
+
// src/lib/prod-debug.ts
|
|
28977
|
+
init_secrets_resolver();
|
|
28978
|
+
var UUID_RE = /\b[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}\b/i;
|
|
28979
|
+
var SENSITIVE_PARAM_RE = /token|secret|key|password|code|state|cookie|session|grant|credential|auth|jwt|access/i;
|
|
28980
|
+
var SENSITIVE_TEXT_RE = /\b(Bearer\s+[A-Za-z0-9._-]{12,}|sk-[A-Za-z0-9]{12,}|pk_[A-Za-z0-9]{12,}|eyJ[A-Za-z0-9._-]{12,})\b/g;
|
|
28981
|
+
var URL_TEXT_RE = /https?:\/\/[^\s"'<>]+/g;
|
|
28982
|
+
function safeUrl(raw) {
|
|
28983
|
+
try {
|
|
28984
|
+
const url = new URL(raw);
|
|
28985
|
+
if (url.protocol !== "http:" && url.protocol !== "https:")
|
|
28986
|
+
return null;
|
|
28987
|
+
return url;
|
|
28988
|
+
} catch {
|
|
28989
|
+
return null;
|
|
28990
|
+
}
|
|
28991
|
+
}
|
|
28992
|
+
function normalizeOrigin(raw) {
|
|
28993
|
+
const url = safeUrl(raw);
|
|
28994
|
+
if (url)
|
|
28995
|
+
return url.origin;
|
|
28996
|
+
const hostUrl = safeUrl(`https://${raw}`);
|
|
28997
|
+
return hostUrl?.origin ?? null;
|
|
28998
|
+
}
|
|
28999
|
+
function redactProdDebugText(value) {
|
|
29000
|
+
return value.replace(URL_TEXT_RE, (match) => {
|
|
29001
|
+
const url = safeUrl(match);
|
|
29002
|
+
return url ? redactUrl(url) : match;
|
|
29003
|
+
}).replace(SENSITIVE_TEXT_RE, (match) => {
|
|
29004
|
+
if (match.startsWith("Bearer "))
|
|
29005
|
+
return "Bearer [redacted]";
|
|
29006
|
+
return "[redacted]";
|
|
29007
|
+
});
|
|
29008
|
+
}
|
|
29009
|
+
function redactUrl(url) {
|
|
29010
|
+
const clone = new URL(url.toString());
|
|
29011
|
+
for (const key of Array.from(clone.searchParams.keys())) {
|
|
29012
|
+
if (SENSITIVE_PARAM_RE.test(key)) {
|
|
29013
|
+
clone.searchParams.set(key, "[redacted]");
|
|
29014
|
+
}
|
|
29015
|
+
}
|
|
29016
|
+
return clone.toString();
|
|
29017
|
+
}
|
|
29018
|
+
function redactUrlString(value) {
|
|
29019
|
+
const url = safeUrl(value);
|
|
29020
|
+
return url ? redactUrl(url) : redactProdDebugText(value);
|
|
29021
|
+
}
|
|
29022
|
+
function parseProdDebugTarget(target) {
|
|
29023
|
+
const input = target.trim();
|
|
29024
|
+
const url = safeUrl(input);
|
|
29025
|
+
if (!url) {
|
|
29026
|
+
const id = (input.match(UUID_RE)?.[0] ?? input) || null;
|
|
29027
|
+
return {
|
|
29028
|
+
url: null,
|
|
29029
|
+
origin: null,
|
|
29030
|
+
orgSlug: null,
|
|
29031
|
+
projectRef: null,
|
|
29032
|
+
sessionId: null,
|
|
29033
|
+
agentId: null,
|
|
29034
|
+
requestId: input.startsWith("req_") ? input : null,
|
|
29035
|
+
rawId: id
|
|
29036
|
+
};
|
|
29037
|
+
}
|
|
29038
|
+
const parts = url.pathname.split("/").filter(Boolean);
|
|
29039
|
+
const projectsIndex = parts.indexOf("projects");
|
|
29040
|
+
const sessionsIndex = parts.indexOf("sessions");
|
|
29041
|
+
const orgSlug = projectsIndex > 0 ? parts[0] ?? null : null;
|
|
29042
|
+
const projectRef = projectsIndex >= 0 ? parts[projectsIndex + 1] ?? null : null;
|
|
29043
|
+
const sessionId = url.searchParams.get("session") ?? (sessionsIndex >= 0 ? parts[sessionsIndex + 1] ?? null : null);
|
|
29044
|
+
return {
|
|
29045
|
+
url: redactUrl(url),
|
|
29046
|
+
origin: url.origin,
|
|
29047
|
+
orgSlug,
|
|
29048
|
+
projectRef,
|
|
29049
|
+
sessionId,
|
|
29050
|
+
agentId: url.searchParams.get("agent"),
|
|
29051
|
+
requestId: url.searchParams.get("requestId") ?? url.searchParams.get("request_id"),
|
|
29052
|
+
rawId: input.match(UUID_RE)?.[0] ?? null
|
|
29053
|
+
};
|
|
29054
|
+
}
|
|
29055
|
+
function boundedTtl(value) {
|
|
29056
|
+
if (!Number.isFinite(value))
|
|
29057
|
+
return 15;
|
|
29058
|
+
return Math.min(Math.max(Math.round(value ?? 15), 1), 60);
|
|
29059
|
+
}
|
|
29060
|
+
function makeCommand(command) {
|
|
29061
|
+
return command.replace(/\s+/g, " ").trim();
|
|
29062
|
+
}
|
|
29063
|
+
function hostnameFromOrigin(origin) {
|
|
29064
|
+
if (!origin)
|
|
29065
|
+
return null;
|
|
29066
|
+
return safeUrl(origin)?.hostname ?? null;
|
|
29067
|
+
}
|
|
29068
|
+
function originMatches(pattern, origin) {
|
|
29069
|
+
if (!origin)
|
|
29070
|
+
return false;
|
|
29071
|
+
const normalizedPattern = normalizeOrigin(pattern);
|
|
29072
|
+
const normalizedOrigin = normalizeOrigin(origin);
|
|
29073
|
+
if (!normalizedOrigin)
|
|
29074
|
+
return false;
|
|
29075
|
+
if (normalizedPattern === normalizedOrigin)
|
|
29076
|
+
return true;
|
|
29077
|
+
const targetHost = hostnameFromOrigin(normalizedOrigin);
|
|
29078
|
+
const patternHost = normalizedPattern ? hostnameFromOrigin(normalizedPattern) : pattern.replace(/^https?:\/\//, "");
|
|
29079
|
+
if (!targetHost || !patternHost)
|
|
29080
|
+
return false;
|
|
29081
|
+
if (patternHost.startsWith("*.")) {
|
|
29082
|
+
const suffix = patternHost.slice(1);
|
|
29083
|
+
return targetHost.endsWith(suffix);
|
|
29084
|
+
}
|
|
29085
|
+
return targetHost === patternHost;
|
|
29086
|
+
}
|
|
29087
|
+
function resolveProfile(input, target, config) {
|
|
29088
|
+
const apps = config?.apps ?? {};
|
|
29089
|
+
const explicitKey = input.profile?.trim() || input.app?.trim() || config?.defaultProfile;
|
|
29090
|
+
if (explicitKey && apps[explicitKey]) {
|
|
29091
|
+
return {
|
|
29092
|
+
key: explicitKey,
|
|
29093
|
+
profile: apps[explicitKey],
|
|
29094
|
+
matchedOrigin: target.origin
|
|
29095
|
+
};
|
|
29096
|
+
}
|
|
29097
|
+
for (const [key, profile] of Object.entries(apps)) {
|
|
29098
|
+
const match = profile.origins?.find((origin) => originMatches(origin, target.origin));
|
|
29099
|
+
if (match) {
|
|
29100
|
+
return { key, profile, matchedOrigin: match };
|
|
29101
|
+
}
|
|
29102
|
+
}
|
|
29103
|
+
return { key: null, profile: null, matchedOrigin: null };
|
|
29104
|
+
}
|
|
29105
|
+
function firstResolvedCredential(...values) {
|
|
29106
|
+
for (const value of values) {
|
|
29107
|
+
if (!value?.trim())
|
|
29108
|
+
continue;
|
|
29109
|
+
const resolved = resolveCredential(value);
|
|
29110
|
+
if (resolved)
|
|
29111
|
+
return resolved;
|
|
29112
|
+
}
|
|
29113
|
+
return null;
|
|
29114
|
+
}
|
|
29115
|
+
function displayCredential(value, source) {
|
|
29116
|
+
if (!value)
|
|
29117
|
+
return null;
|
|
29118
|
+
if (source && isCredentialReference(source))
|
|
29119
|
+
return "[configured]";
|
|
29120
|
+
return redactProdDebugText(value);
|
|
29121
|
+
}
|
|
29122
|
+
function replacementValues(target, input, supportGrant) {
|
|
29123
|
+
const values = {
|
|
29124
|
+
targetUrl: target.url ?? input.target,
|
|
29125
|
+
origin: target.origin ?? "",
|
|
29126
|
+
org: target.orgSlug ?? "",
|
|
29127
|
+
project: target.projectRef ?? "",
|
|
29128
|
+
session: target.sessionId ?? "",
|
|
29129
|
+
agent: target.agentId ?? "",
|
|
29130
|
+
request: target.requestId ?? "",
|
|
29131
|
+
rawId: target.rawId ?? "",
|
|
29132
|
+
reason: input.reason ?? "",
|
|
29133
|
+
supportGrant: supportGrant ?? ""
|
|
29134
|
+
};
|
|
29135
|
+
for (const [key, value] of Object.entries({ ...values })) {
|
|
29136
|
+
values[`${key}Encoded`] = encodeURIComponent(value);
|
|
29137
|
+
}
|
|
29138
|
+
return values;
|
|
29139
|
+
}
|
|
29140
|
+
function renderTemplate(template, values) {
|
|
29141
|
+
return template.replace(/\{([a-zA-Z0-9_]+)\}/g, (_match, key) => values[key] ?? "");
|
|
29142
|
+
}
|
|
29143
|
+
function resolveSupportGrant(input, profile) {
|
|
29144
|
+
if (input.supportGrantId?.trim()) {
|
|
29145
|
+
return {
|
|
29146
|
+
value: input.supportGrantId.trim(),
|
|
29147
|
+
display: displayCredential(input.supportGrantId.trim()),
|
|
29148
|
+
source: "input"
|
|
29149
|
+
};
|
|
29150
|
+
}
|
|
29151
|
+
const source = profile?.supportGrantRef ?? profile?.supportGrantId ?? null;
|
|
29152
|
+
const value = firstResolvedCredential(profile?.supportGrantRef, profile?.supportGrantId);
|
|
29153
|
+
return { value, display: displayCredential(value, source ?? undefined), source };
|
|
29154
|
+
}
|
|
29155
|
+
function resolveSupportUrl(input, target, profile, supportGrant) {
|
|
29156
|
+
if (input.supportUrl?.trim())
|
|
29157
|
+
return input.supportUrl.trim();
|
|
29158
|
+
const direct = firstResolvedCredential(profile?.supportUrlRef, profile?.supportUrl);
|
|
29159
|
+
if (direct)
|
|
29160
|
+
return direct;
|
|
29161
|
+
if (profile?.supportUrlTemplate) {
|
|
29162
|
+
const rendered = renderTemplate(profile.supportUrlTemplate, replacementValues(target, input, supportGrant)).trim();
|
|
29163
|
+
return rendered || null;
|
|
29164
|
+
}
|
|
29165
|
+
return null;
|
|
29166
|
+
}
|
|
29167
|
+
function resolvePiiOrigin(profile, target) {
|
|
29168
|
+
if (!profile?.piiOrigin)
|
|
29169
|
+
return target.origin;
|
|
29170
|
+
return redactUrlString(renderTemplate(profile.piiOrigin, replacementValues(target, { target: target.url ?? "" }, null)));
|
|
29171
|
+
}
|
|
29172
|
+
function resolveSupportRunTarget(supportUrl, input, target) {
|
|
29173
|
+
if (supportUrl)
|
|
29174
|
+
return redactUrlString(supportUrl);
|
|
29175
|
+
return target.url ?? target.origin ?? redactProdDebugText(input.target);
|
|
29176
|
+
}
|
|
29177
|
+
function supportScenarioDescription(reason) {
|
|
29178
|
+
return `Prod debug: ${reason}. Reproduce the user-visible issue, capture console and network errors, and do not enter secrets.`;
|
|
29179
|
+
}
|
|
29180
|
+
function configuredMissing(profile, supportUrl, supportGrant, includeLogs) {
|
|
29181
|
+
const missing = [];
|
|
29182
|
+
if (!profile) {
|
|
29183
|
+
missing.push("optional: add prodDebug.apps.<profile>.origins to match this app automatically");
|
|
29184
|
+
}
|
|
29185
|
+
if (!supportUrl) {
|
|
29186
|
+
missing.push("supportUrl/supportUrlRef/supportUrlTemplate for scoped browser debugging");
|
|
29187
|
+
}
|
|
29188
|
+
if (!supportGrant) {
|
|
29189
|
+
missing.push("supportGrantId/supportGrantRef for auditable support access");
|
|
29190
|
+
}
|
|
29191
|
+
if (includeLogs && !profile?.logCommand) {
|
|
29192
|
+
missing.push("logCommand for sanitized app/provider log lookup");
|
|
29193
|
+
}
|
|
29194
|
+
return missing;
|
|
29195
|
+
}
|
|
29196
|
+
function createProdDebugPlan(input, config) {
|
|
29197
|
+
const target = parseProdDebugTarget(input.target);
|
|
29198
|
+
const browserRequested = input.includeBrowser !== false;
|
|
29199
|
+
const resolvedProfile = resolveProfile(input, target, config);
|
|
29200
|
+
const supportGrant = resolveSupportGrant(input, resolvedProfile.profile);
|
|
29201
|
+
const supportUrl = resolveSupportUrl(input, target, resolvedProfile.profile, supportGrant.value);
|
|
29202
|
+
const supportBrowserReady = Boolean(supportUrl);
|
|
29203
|
+
const app = input.app?.trim() || resolvedProfile.profile?.name || resolvedProfile.key || (target.origin ? new URL(target.origin).hostname : "app");
|
|
29204
|
+
const reason = input.reason?.trim() || "production debug requested";
|
|
29205
|
+
const actor = input.actor?.trim() || process.env["USER"] || "agent";
|
|
29206
|
+
const ttlMinutes = boundedTtl(input.ttlMinutes);
|
|
29207
|
+
const piiOrigin = resolvePiiOrigin(resolvedProfile.profile, target);
|
|
29208
|
+
const logCommand = resolvedProfile.profile?.logCommand ? redactUrlString(renderTemplate(resolvedProfile.profile.logCommand, replacementValues(target, { ...input, reason }, supportGrant.value))) : null;
|
|
29209
|
+
const safety = [
|
|
29210
|
+
"read-only by default",
|
|
29211
|
+
"no customer passwords or raw cookies",
|
|
29212
|
+
"redact tokens, OAuth codes, session values, support grants, and secrets",
|
|
29213
|
+
"verify org/user/session scope before reading data",
|
|
29214
|
+
"require explicit approval for production writes",
|
|
29215
|
+
`support access TTL capped at ${ttlMinutes} minutes`
|
|
29216
|
+
];
|
|
29217
|
+
const checks = [];
|
|
29218
|
+
const blocked = [];
|
|
29219
|
+
if (target.url) {
|
|
29220
|
+
checks.push({
|
|
29221
|
+
id: "public-route-smoke",
|
|
29222
|
+
status: "ready",
|
|
29223
|
+
description: "Open the supplied production URL and capture console/network errors without credentials.",
|
|
29224
|
+
command: makeCommand(`testers scan all ${JSON.stringify(target.url)} --json`)
|
|
29225
|
+
});
|
|
29226
|
+
}
|
|
29227
|
+
checks.push({
|
|
29228
|
+
id: "pii-redaction-scan",
|
|
29229
|
+
status: piiOrigin ? "ready" : "blocked",
|
|
29230
|
+
description: "Scan public/API responses for accidental sensitive data leakage.",
|
|
29231
|
+
command: piiOrigin ? makeCommand(`testers scan pii ${JSON.stringify(piiOrigin)} --json`) : undefined,
|
|
29232
|
+
reason: piiOrigin ? undefined : "Need a URL origin or prodDebug app profile piiOrigin to run the PII scan."
|
|
29233
|
+
});
|
|
29234
|
+
if (browserRequested) {
|
|
29235
|
+
if (supportBrowserReady) {
|
|
29236
|
+
checks.push({
|
|
29237
|
+
id: "support-browser-repro",
|
|
29238
|
+
status: "ready",
|
|
29239
|
+
description: "Use an audited support browser/session URL to reproduce the user-visible issue.",
|
|
29240
|
+
command: makeCommand(`testers run ${JSON.stringify(resolveSupportRunTarget(supportUrl, input, target))} ${JSON.stringify(supportScenarioDescription(reason))} --headed --json --overall-timeout 600000`)
|
|
29241
|
+
});
|
|
29242
|
+
} else {
|
|
29243
|
+
const reasonText = supportGrant.value ? "An audited support grant was supplied, but open-testers still needs supportUrl/supportUrlRef/supportUrlTemplate or an app adapter to open a scoped browser session." : "No audited support browser/session grant was supplied. Do not use customer passwords, copied cookies, bearer tokens, or magic links.";
|
|
29244
|
+
blocked.push(reasonText);
|
|
29245
|
+
checks.push({
|
|
29246
|
+
id: "support-browser-repro",
|
|
29247
|
+
status: "blocked",
|
|
29248
|
+
description: "Browser reproduction as the target user requires a short-lived audited support session.",
|
|
29249
|
+
reason: reasonText
|
|
29250
|
+
});
|
|
29251
|
+
}
|
|
29252
|
+
}
|
|
29253
|
+
if (input.includeLogs) {
|
|
29254
|
+
if (logCommand) {
|
|
29255
|
+
checks.push({
|
|
29256
|
+
id: "log-timeline",
|
|
29257
|
+
status: "ready",
|
|
29258
|
+
description: "Read sanitized app/provider logs by request ID, session ID, project ID, or support access ID.",
|
|
29259
|
+
command: makeCommand(logCommand)
|
|
29260
|
+
});
|
|
29261
|
+
} else {
|
|
29262
|
+
checks.push({
|
|
29263
|
+
id: "log-timeline",
|
|
29264
|
+
status: "blocked",
|
|
29265
|
+
description: "Read sanitized app/provider logs by request ID, session ID, project ID, or support access ID.",
|
|
29266
|
+
reason: "Configure prodDebug.apps.<profile>.logCommand or use an app-specific log MCP. Do not paste raw provider logs with headers/secrets."
|
|
29267
|
+
});
|
|
29268
|
+
}
|
|
29269
|
+
}
|
|
29270
|
+
if (input.allowWrites) {
|
|
29271
|
+
blocked.push("Production writes are not part of prod-debug. Require a separate explicit approval and app-specific write tool.");
|
|
29272
|
+
}
|
|
29273
|
+
return {
|
|
29274
|
+
target,
|
|
29275
|
+
app,
|
|
29276
|
+
actor,
|
|
29277
|
+
reason,
|
|
29278
|
+
ttlMinutes,
|
|
29279
|
+
setup: {
|
|
29280
|
+
profile: resolvedProfile.key,
|
|
29281
|
+
matchedOrigin: resolvedProfile.matchedOrigin,
|
|
29282
|
+
configured: {
|
|
29283
|
+
supportUrl: Boolean(supportUrl),
|
|
29284
|
+
supportGrant: Boolean(supportGrant.value),
|
|
29285
|
+
piiOrigin: Boolean(piiOrigin),
|
|
29286
|
+
logCommand: Boolean(logCommand)
|
|
29287
|
+
},
|
|
29288
|
+
missing: configuredMissing(resolvedProfile.profile, supportUrl, supportGrant.value, Boolean(input.includeLogs))
|
|
29289
|
+
},
|
|
29290
|
+
supportAccess: {
|
|
29291
|
+
required: browserRequested,
|
|
29292
|
+
grantId: supportGrant.display,
|
|
29293
|
+
browserReady: supportBrowserReady,
|
|
29294
|
+
note: supportBrowserReady ? "Use the provided audited support access; never print token/cookie values." : "Configure an audited support-browser/session URL, URL ref, or template before user-scoped browser debugging."
|
|
29295
|
+
},
|
|
29296
|
+
safety,
|
|
29297
|
+
checks,
|
|
29298
|
+
blocked
|
|
29299
|
+
};
|
|
29300
|
+
}
|
|
29301
|
+
function formatProdDebugPlan(plan) {
|
|
29302
|
+
const lines = [];
|
|
29303
|
+
lines.push(`Prod debug plan for ${plan.app}`);
|
|
29304
|
+
lines.push("");
|
|
29305
|
+
lines.push("Target");
|
|
29306
|
+
lines.push(`- url: ${plan.target.url ?? "(none)"}`);
|
|
29307
|
+
lines.push(`- org: ${plan.target.orgSlug ?? "(unknown)"}`);
|
|
29308
|
+
lines.push(`- project: ${plan.target.projectRef ?? "(unknown)"}`);
|
|
29309
|
+
lines.push(`- session: ${plan.target.sessionId ?? "(unknown)"}`);
|
|
29310
|
+
lines.push(`- agent: ${plan.target.agentId ?? "(unknown)"}`);
|
|
29311
|
+
lines.push(`- request: ${plan.target.requestId ?? "(unknown)"}`);
|
|
29312
|
+
lines.push("");
|
|
29313
|
+
lines.push("Setup");
|
|
29314
|
+
lines.push(`- profile: ${plan.setup.profile ?? "(none)"}`);
|
|
29315
|
+
lines.push(`- matched origin: ${plan.setup.matchedOrigin ?? "(none)"}`);
|
|
29316
|
+
if (plan.setup.missing.length > 0) {
|
|
29317
|
+
for (const item of plan.setup.missing)
|
|
29318
|
+
lines.push(`- missing: ${item}`);
|
|
29319
|
+
}
|
|
29320
|
+
lines.push("");
|
|
29321
|
+
lines.push("Support access");
|
|
29322
|
+
lines.push(`- actor: ${plan.actor}`);
|
|
29323
|
+
lines.push(`- reason: ${plan.reason}`);
|
|
29324
|
+
lines.push(`- ttl: ${plan.ttlMinutes} minutes`);
|
|
29325
|
+
lines.push(`- grant: ${plan.supportAccess.grantId ?? "(none)"}`);
|
|
29326
|
+
lines.push(`- browser ready: ${plan.supportAccess.browserReady ? "yes" : "no"}`);
|
|
29327
|
+
lines.push(`- note: ${plan.supportAccess.note}`);
|
|
29328
|
+
lines.push("");
|
|
29329
|
+
lines.push("Checks");
|
|
29330
|
+
for (const check of plan.checks) {
|
|
29331
|
+
lines.push(`- ${check.id}: ${check.status} - ${check.description}`);
|
|
29332
|
+
if (check.command)
|
|
29333
|
+
lines.push(` command: ${check.command}`);
|
|
29334
|
+
if (check.reason)
|
|
29335
|
+
lines.push(` blocked: ${check.reason}`);
|
|
29336
|
+
}
|
|
29337
|
+
if (plan.blocked.length > 0) {
|
|
29338
|
+
lines.push("");
|
|
29339
|
+
lines.push("Blocked");
|
|
29340
|
+
for (const item of plan.blocked)
|
|
29341
|
+
lines.push(`- ${item}`);
|
|
29342
|
+
}
|
|
29343
|
+
lines.push("");
|
|
29344
|
+
lines.push("Safety");
|
|
29345
|
+
for (const item of plan.safety)
|
|
29346
|
+
lines.push(`- ${item}`);
|
|
29347
|
+
return lines.join(`
|
|
29348
|
+
`);
|
|
29349
|
+
}
|
|
29350
|
+
|
|
29351
|
+
// src/cli/index.tsx
|
|
28252
29352
|
init_personas();
|
|
28253
29353
|
init_api_checks();
|
|
28254
29354
|
|
|
@@ -28535,6 +29635,18 @@ var SCENARIO_TEMPLATES = {
|
|
|
28535
29635
|
a11y: [
|
|
28536
29636
|
{ name: "Keyboard navigation", description: "Navigate the page using only keyboard (Tab, Enter, Escape). Verify all interactive elements are reachable and focusable.", tags: ["a11y", "keyboard"], priority: "high", steps: ["Press Tab to move through elements", "Verify focus indicators are visible", "Press Enter on buttons/links", "Verify actions trigger correctly", "Press Escape to close modals/dropdowns"] },
|
|
28537
29637
|
{ name: "Image alt text", description: "Check that all images have meaningful alt text attributes.", tags: ["a11y"], priority: "medium", steps: ["Find all images on the page", "Check each image has an alt attribute", "Verify alt text is descriptive, not empty or generic"] }
|
|
29638
|
+
],
|
|
29639
|
+
checkout: [
|
|
29640
|
+
{ name: "Add item to cart", description: "Navigate to a product page, add an item to cart, and verify cart count increments.", tags: ["checkout", "cart"], priority: "critical", steps: ["Navigate to product detail page", "Click add to cart button", "Verify cart count badge updates", "Verify success notification appears"] },
|
|
29641
|
+
{ name: "Cart page shows added items", description: "Navigate to the cart page and verify previously added items are listed with correct prices and quantities.", tags: ["checkout", "cart"], priority: "high", steps: ["Navigate to cart page", "Verify all added items are listed", "Verify prices and quantities are correct", "Verify subtotal is calculated"] },
|
|
29642
|
+
{ name: "Checkout flow completion", description: "Complete the full checkout flow: cart -> shipping -> payment -> confirmation. Verify order is placed successfully.", tags: ["checkout", "smoke"], priority: "critical", steps: ["Navigate to cart", "Click proceed to checkout", "Fill shipping address", "Select shipping method", "Fill payment details", "Review order summary", "Place order", "Verify order confirmation page"] },
|
|
29643
|
+
{ name: "Apply coupon/discount code", description: "Enter a valid coupon code on the cart or checkout page and verify discount is applied to the total.", tags: ["checkout", "discount"], priority: "high", steps: ["Navigate to cart or checkout", "Enter coupon code", "Click apply", "Verify discount amount is shown", "Verify total is updated"] }
|
|
29644
|
+
],
|
|
29645
|
+
search: [
|
|
29646
|
+
{ name: "Search returns relevant results", description: "Enter a known search term and verify relevant results appear. Check that results match the query intent.", tags: ["search"], priority: "high", steps: ["Navigate to search page or focus search bar", "Enter known search term", "Submit search", "Verify results appear", "Verify result titles contain search term"] },
|
|
29647
|
+
{ name: "Empty search handling", description: "Submit an empty search and verify the system handles it gracefully (show all results or a prompt, no errors).", tags: ["search", "validation"], priority: "medium", steps: ["Focus search bar", "Submit empty search", "Verify no crash or error state", "Verify fallback behavior (all results or prompt)"] },
|
|
29648
|
+
{ name: "No results handling", description: "Search for a term that returns no results and verify appropriate 'no results found' messaging is displayed.", tags: ["search"], priority: "medium", steps: ["Navigate to search", "Enter nonsense term like 'xyzqwrtp'", "Submit search", "Verify no results found message", "Verify no error states"] },
|
|
29649
|
+
{ name: "Search filters work", description: "Perform a search, apply a filter (category, date, price range), and verify results are narrowed down.", tags: ["search", "filter"], priority: "high", steps: ["Perform initial search", "Note initial result count", "Apply a filter", "Verify result count decreased", "Verify results match filter criteria"] }
|
|
28538
29650
|
]
|
|
28539
29651
|
};
|
|
28540
29652
|
function getTemplate(name) {
|
|
@@ -28653,6 +29765,10 @@ function getDefaultEnvironment() {
|
|
|
28653
29765
|
const row = db2.query("SELECT * FROM environments WHERE is_default = 1 LIMIT 1").get();
|
|
28654
29766
|
return row ? fromRow3(row) : null;
|
|
28655
29767
|
}
|
|
29768
|
+
|
|
29769
|
+
// src/cli/index.tsx
|
|
29770
|
+
init_ci();
|
|
29771
|
+
|
|
28656
29772
|
// src/lib/assertions.ts
|
|
28657
29773
|
function parseAssertionString(str) {
|
|
28658
29774
|
const trimmed = str.trim();
|
|
@@ -28709,12 +29825,44 @@ function parseAssertionString(str) {
|
|
|
28709
29825
|
}
|
|
28710
29826
|
throw new Error(`Unknown selector action: "${action}". Expected "visible" or "not-visible"`);
|
|
28711
29827
|
}
|
|
29828
|
+
if (trimmed.startsWith("cookie:exists:")) {
|
|
29829
|
+
const name = trimmed.slice("cookie:exists:".length);
|
|
29830
|
+
return { type: "cookie_exists", expected: name, description: `Cookie "${name}" exists` };
|
|
29831
|
+
}
|
|
29832
|
+
if (trimmed.startsWith("cookie:not-exists:")) {
|
|
29833
|
+
const name = trimmed.slice("cookie:not-exists:".length);
|
|
29834
|
+
return { type: "cookie_not_exists", expected: name, description: `Cookie "${name}" does not exist` };
|
|
29835
|
+
}
|
|
29836
|
+
if (trimmed.startsWith("cookie:value:")) {
|
|
29837
|
+
const valueStr = trimmed.slice("cookie:value:".length);
|
|
29838
|
+
return { type: "cookie_value", expected: valueStr, description: `Cookie value is "${valueStr}"` };
|
|
29839
|
+
}
|
|
29840
|
+
if (trimmed.startsWith("local:exists:")) {
|
|
29841
|
+
const key = trimmed.slice("local:exists:".length);
|
|
29842
|
+
return { type: "local_storage_exists", expected: key, description: `LocalStorage key "${key}" exists` };
|
|
29843
|
+
}
|
|
29844
|
+
if (trimmed.startsWith("local:not-exists:")) {
|
|
29845
|
+
const key = trimmed.slice("local:not-exists:".length);
|
|
29846
|
+
return { type: "local_storage_not_exists", expected: key, description: `LocalStorage key "${key}" does not exist` };
|
|
29847
|
+
}
|
|
29848
|
+
if (trimmed.startsWith("local:value:")) {
|
|
29849
|
+
const valueStr = trimmed.slice("local:value:".length);
|
|
29850
|
+
return { type: "local_storage_value", expected: valueStr, description: `LocalStorage value is "${valueStr}"` };
|
|
29851
|
+
}
|
|
29852
|
+
if (trimmed.startsWith("session:value:")) {
|
|
29853
|
+
const valueStr = trimmed.slice("session:value:".length);
|
|
29854
|
+
return { type: "session_storage_value", expected: valueStr, description: `SessionStorage value is "${valueStr}"` };
|
|
29855
|
+
}
|
|
29856
|
+
if (trimmed.startsWith("session:not-exists:")) {
|
|
29857
|
+
const key = trimmed.slice("session:not-exists:".length);
|
|
29858
|
+
return { type: "session_storage_not_exists", expected: key, description: `SessionStorage key "${key}" does not exist` };
|
|
29859
|
+
}
|
|
28712
29860
|
throw new Error(`Cannot parse assertion: "${str}". See --help for assertion formats.`);
|
|
28713
29861
|
}
|
|
28714
29862
|
|
|
28715
29863
|
// src/cli/index.tsx
|
|
28716
29864
|
init_paths();
|
|
28717
|
-
import { existsSync as existsSync14, mkdirSync as
|
|
29865
|
+
import { existsSync as existsSync14, mkdirSync as mkdirSync11 } from "fs";
|
|
28718
29866
|
import { jsxDEV } from "react/jsx-dev-runtime";
|
|
28719
29867
|
var PRIORITIES = ["low", "medium", "high", "critical"];
|
|
28720
29868
|
function AddForm({ onComplete }) {
|
|
@@ -28973,8 +30121,31 @@ function logError(...args) {
|
|
|
28973
30121
|
console.error(...args);
|
|
28974
30122
|
}
|
|
28975
30123
|
program2.name("testers").version(package_default.version).description("AI-powered browser testing CLI").option("-q, --quiet", "Suppress all output", false).option("--no-color", "Disable color output");
|
|
30124
|
+
program2.command("prod-debug <target>").description("Create a safe production debug plan for a URL/session/request without leaking secrets").option("--app <name>", "App name for reporting").option("--profile <name>", "prodDebug app profile from testers config").option("--actor <name>", "Operator/agent identity for support audit context").option("--reason <text>", "Debug reason or support context").option("--support-url <url>", "Audited support browser/session URL minted by the target app").option("--support-grant <id>", "Audited support access grant ID").option("--ttl <minutes>", "Support access TTL in minutes, capped at 60", "15").option("--no-browser", "Do not include user-scoped browser reproduction").option("--logs", "Include log timeline adapter requirement", false).option("--allow-writes", "Document that a separate explicit approval is required for writes", false).option("--json", "Output JSON", false).option("-o, --output <filepath>", "Write plan to file").action((target, opts) => {
|
|
30125
|
+
const config = loadConfig();
|
|
30126
|
+
const plan = createProdDebugPlan({
|
|
30127
|
+
target,
|
|
30128
|
+
app: opts.app,
|
|
30129
|
+
profile: opts.profile,
|
|
30130
|
+
actor: opts.actor,
|
|
30131
|
+
reason: opts.reason,
|
|
30132
|
+
supportUrl: opts.supportUrl,
|
|
30133
|
+
supportGrantId: opts.supportGrant,
|
|
30134
|
+
ttlMinutes: parseInt(opts.ttl, 10),
|
|
30135
|
+
includeBrowser: opts.browser,
|
|
30136
|
+
includeLogs: opts.logs,
|
|
30137
|
+
allowWrites: opts.allowWrites
|
|
30138
|
+
}, config.prodDebug);
|
|
30139
|
+
const output = opts.json ? JSON.stringify(plan, null, 2) : formatProdDebugPlan(plan);
|
|
30140
|
+
if (opts.output) {
|
|
30141
|
+
writeFileSync4(resolve(opts.output), output + `
|
|
30142
|
+
`);
|
|
30143
|
+
} else {
|
|
30144
|
+
log(output);
|
|
30145
|
+
}
|
|
30146
|
+
});
|
|
28976
30147
|
var CONFIG_DIR5 = getTestersDir();
|
|
28977
|
-
var CONFIG_PATH3 =
|
|
30148
|
+
var CONFIG_PATH3 = join16(CONFIG_DIR5, "config.json");
|
|
28978
30149
|
function getActiveProject() {
|
|
28979
30150
|
try {
|
|
28980
30151
|
if (existsSync14(CONFIG_PATH3)) {
|
|
@@ -29242,7 +30413,7 @@ program2.command("remove <id>").alias("uninstall").description("Remove a scenari
|
|
|
29242
30413
|
program2.command("run [url] [description]").alias("test").description("Run test scenarios against a URL").option("-t, --tag <tag>", "Filter by tag (repeatable)", (val, acc) => {
|
|
29243
30414
|
acc.push(val);
|
|
29244
30415
|
return acc;
|
|
29245
|
-
}, []).option("-s, --scenario <id>", "Run specific scenario ID").option("-p, --priority <level>", "Filter by priority").option("--headed", "Run browser in headed mode", false).option("-m, --model <model>", "AI model to use").option("--parallel <n>", "Number of parallel browsers", "1").option("--json", "Output results as JSON", false).option("-o, --output <filepath>", "Write JSON results to file").option("--timeout <ms>", "Timeout in milliseconds").option("--from-todos", "Import scenarios from todos before running", false).option("--project <id>", "Project ID").option("-b, --background", "Start run in background and return immediately", false).option("--browser <engine>", "Browser engine: playwright (default), lightpanda (9x faster, no screenshots), or bun (native WKWebView, 11x faster, Bun canary required)", "playwright").option("--env <name>", "Use a named environment for the URL").option("--dry-run", "Print what would run without launching browser", false).option("--retry <n>", "Retry failed scenarios up to n times", "0").option("--samples <n>", "Run each scenario N times and report flakiness (pass rate)", "1").option("--flakiness-threshold <n>", "Pass rate threshold below which a scenario is marked flaky (0-1)", "0.95").option("--a11y [level]", "Run axe-core WCAG accessibility scan after each navigation (level: A, AA, AAA \u2014 default AA)").option("--self-heal", "Enable AI-powered selector repair when elements can't be found (requires judgeModel or ANTHROPIC_API_KEY)", false).option("--verbose", "Show per-step timing and full tool results", false).option("--watch-results", "When used with --background, poll and display live results table until run completes", false).option("--failed-only", "Only show failed/error scenarios in output (passed count shown as summary)", false).option("--smoke", "Run only smoke-tagged scenarios (fast validation suite, <2 min)", false).option("--minimal", "Fastest possible run: cheapest model, max parallelism, min turns (ideal for CI)", false).option("--github-comment", "Post pass/fail summary as a GitHub PR comment (requires GITHUB_TOKEN env var)", false).option("--pr <number>", "GitHub PR number (auto-detected from GITHUB_REF if not provided)").option("--persona <id>", "Override persona for this run (comma-separated IDs for divergence testing)").option("--max-cost <dollars>", "Hard budget cap in dollars \u2014 abort if estimated cost exceeds this (e.g. 0.50 for 50 cents)").option("--cache-max-age <seconds>", "Skip scenarios that passed at the same URL within this many seconds (0 = disabled)", "0").option("--diff", "Auto-detect changed files from git diff and run only relevant scenarios", false).action(async (urlArg, description, opts) => {
|
|
30416
|
+
}, []).option("-s, --scenario <id>", "Run specific scenario ID").option("-p, --priority <level>", "Filter by priority").option("--headed", "Run browser in headed mode", false).option("-m, --model <model>", "AI model to use").option("--parallel <n>", "Number of parallel browsers", "1").option("--json", "Output results as JSON", false).option("-o, --output <filepath>", "Write JSON results to file").option("--timeout <ms>", "Timeout in milliseconds").option("--from-todos", "Import scenarios from todos before running", false).option("--project <id>", "Project ID").option("-b, --background", "Start run in background and return immediately", false).option("--browser <engine>", "Browser engine: playwright (default), lightpanda (9x faster, no screenshots), or bun (native WKWebView, 11x faster, Bun canary required)", "playwright").option("--env <name>", "Use a named environment for the URL").option("--dry-run", "Print what would run without launching browser", false).option("--retry <n>", "Retry failed scenarios up to n times", "0").option("--samples <n>", "Run each scenario N times and report flakiness (pass rate)", "1").option("--flakiness-threshold <n>", "Pass rate threshold below which a scenario is marked flaky (0-1)", "0.95").option("--a11y [level]", "Run axe-core WCAG accessibility scan after each navigation (level: A, AA, AAA \u2014 default AA)").option("--self-heal", "Enable AI-powered selector repair when elements can't be found (requires judgeModel or ANTHROPIC_API_KEY)", false).option("--verbose", "Show per-step timing and full tool results", false).option("--watch-results", "When used with --background, poll and display live results table until run completes", false).option("--failed-only", "Only show failed/error scenarios in output (passed count shown as summary)", false).option("--smoke", "Run only smoke-tagged scenarios (fast validation suite, <2 min)", false).option("--minimal", "Fastest possible run: cheapest model, max parallelism, min turns (ideal for CI)", false).option("--github-comment", "Post pass/fail summary as a GitHub PR comment (requires GITHUB_TOKEN env var)", false).option("--pr <number>", "GitHub PR number (auto-detected from GITHUB_REF if not provided)").option("--persona <id>", "Override persona for this run (comma-separated IDs for divergence testing)").option("--max-cost <dollars>", "Hard budget cap in dollars \u2014 abort if estimated cost exceeds this (e.g. 0.50 for 50 cents)").option("--cache-max-age <seconds>", "Skip scenarios that passed at the same URL within this many seconds (0 = disabled)", "0").option("--diff", "Auto-detect changed files from git diff and run only relevant scenarios", false).option("--auto-generate", "If no scenarios exist, crawl the URL and generate scenarios automatically (enabled by default when a URL is given as the first arg)").option("--no-auto-generate", "Disable automatic scenario generation when no scenarios exist").option("--overall-timeout <ms>", "Hard overall timeout for the whole run in milliseconds (default 10 minutes)").option("-y, --yes", "Skip confirmation prompts (e.g. proceed past budget warnings)", false).action(async (urlArg, description, opts) => {
|
|
29246
30417
|
try {
|
|
29247
30418
|
const projectId = resolveProject(opts.project);
|
|
29248
30419
|
let url = urlArg;
|
|
@@ -29263,7 +30434,51 @@ program2.command("run [url] [description]").alias("test").description("Run test
|
|
|
29263
30434
|
}
|
|
29264
30435
|
if (!url) {
|
|
29265
30436
|
logError(chalk6.red("No URL provided. Pass a URL argument, use --env <name>, or set a default environment with 'testers env use <name>'."));
|
|
29266
|
-
process.exit(
|
|
30437
|
+
process.exit(2);
|
|
30438
|
+
}
|
|
30439
|
+
if (!opts.dryRun) {
|
|
30440
|
+
const hasAnthropic = Boolean(process.env["ANTHROPIC_API_KEY"]);
|
|
30441
|
+
const hasOpenAI = Boolean(process.env["OPENAI_API_KEY"]);
|
|
30442
|
+
const hasGoogle = Boolean(process.env["GOOGLE_API_KEY"]);
|
|
30443
|
+
const hasCerebras = Boolean(process.env["CEREBRAS_API_KEY"]);
|
|
30444
|
+
if (!hasAnthropic && !hasOpenAI && !hasGoogle && !hasCerebras) {
|
|
30445
|
+
logError(chalk6.red("No AI API key found. Set ANTHROPIC_API_KEY (recommended), or OPENAI_API_KEY / GOOGLE_API_KEY / CEREBRAS_API_KEY."));
|
|
30446
|
+
logError(chalk6.red("For GitHub Actions, add ANTHROPIC_API_KEY to your repo secrets and pass it via: env: ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}"));
|
|
30447
|
+
process.exit(2);
|
|
30448
|
+
}
|
|
30449
|
+
}
|
|
30450
|
+
if (!opts.dryRun && !opts.background) {
|
|
30451
|
+
const reachable = await (async () => {
|
|
30452
|
+
try {
|
|
30453
|
+
const ctrl = new AbortController;
|
|
30454
|
+
const t = setTimeout(() => ctrl.abort(), 1e4);
|
|
30455
|
+
let res;
|
|
30456
|
+
try {
|
|
30457
|
+
res = await fetch(url, { method: "HEAD", signal: ctrl.signal, redirect: "follow" });
|
|
30458
|
+
} catch {
|
|
30459
|
+
res = await fetch(url, { method: "GET", signal: ctrl.signal, redirect: "follow" });
|
|
30460
|
+
}
|
|
30461
|
+
clearTimeout(t);
|
|
30462
|
+
if (res.status >= 500)
|
|
30463
|
+
return { ok: false, reason: `HTTP ${res.status}` };
|
|
30464
|
+
return { ok: true };
|
|
30465
|
+
} catch (err) {
|
|
30466
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
30467
|
+
return { ok: false, reason: msg };
|
|
30468
|
+
}
|
|
30469
|
+
})();
|
|
30470
|
+
if (!reachable.ok) {
|
|
30471
|
+
logError(chalk6.red(`URL unreachable: ${url}${reachable.reason ? ` (${reachable.reason})` : ""}`));
|
|
30472
|
+
logError(chalk6.red("Check that your preview deployment is up and the URL is correct."));
|
|
30473
|
+
process.exit(2);
|
|
30474
|
+
}
|
|
30475
|
+
}
|
|
30476
|
+
const overallTimeoutMs = opts.overallTimeout ? parseInt(opts.overallTimeout, 10) : 10 * 60 * 1000;
|
|
30477
|
+
if (!opts.dryRun && !opts.background && overallTimeoutMs > 0) {
|
|
30478
|
+
setTimeout(() => {
|
|
30479
|
+
logError(chalk6.red(`Overall timeout reached (${Math.round(overallTimeoutMs / 1000)}s). Aborting.`));
|
|
30480
|
+
process.exit(2);
|
|
30481
|
+
}, overallTimeoutMs).unref();
|
|
29267
30482
|
}
|
|
29268
30483
|
if (!opts.dryRun && !opts.background) {
|
|
29269
30484
|
const budgetResult = checkBudget(0);
|
|
@@ -29485,13 +30700,15 @@ program2.command("run [url] [description]").alias("test").description("Run test
|
|
|
29485
30700
|
log(formatTerminal(run2, results2, { failedOnly: opts.failedOnly }));
|
|
29486
30701
|
}
|
|
29487
30702
|
if (opts.githubComment) {
|
|
29488
|
-
const { postGitHubComment: postGitHubComment2 } = await Promise.resolve().then(() => exports_ci);
|
|
30703
|
+
const { postGitHubComment: postGitHubComment2 } = await Promise.resolve().then(() => (init_ci(), exports_ci));
|
|
29489
30704
|
const prNumber = opts.pr ? parseInt(opts.pr, 10) : undefined;
|
|
29490
30705
|
const posted = await postGitHubComment2(run2, results2, { prNumber });
|
|
29491
30706
|
if (posted) {
|
|
29492
30707
|
log(chalk6.green(" GitHub PR comment posted."));
|
|
29493
30708
|
} else if (!process.env["GITHUB_TOKEN"]) {
|
|
29494
30709
|
log(chalk6.yellow(" --github-comment: GITHUB_TOKEN not set, skipping PR comment."));
|
|
30710
|
+
} else {
|
|
30711
|
+
log(chalk6.yellow(" --github-comment: could not post PR comment (check GITHUB_PR_NUMBER / GITHUB_REF / GITHUB_REPOSITORY env)."));
|
|
29495
30712
|
}
|
|
29496
30713
|
}
|
|
29497
30714
|
process.exit(getExitCode(run2));
|
|
@@ -29502,6 +30719,33 @@ program2.command("run [url] [description]").alias("test").description("Run test
|
|
|
29502
30719
|
log(chalk6.bold(` Running all ${allScenarios.length} scenarios...`));
|
|
29503
30720
|
log("");
|
|
29504
30721
|
}
|
|
30722
|
+
const shouldAutoGenerate = urlArg !== undefined && opts.autoGenerate !== false && noFilters;
|
|
30723
|
+
if (shouldAutoGenerate) {
|
|
30724
|
+
const existingScenarios = listScenarios({ projectId });
|
|
30725
|
+
if (existingScenarios.length === 0) {
|
|
30726
|
+
log(chalk6.blue(" No scenarios found \u2014 crawling URL to auto-generate scenarios..."));
|
|
30727
|
+
log(chalk6.dim(` (disable with --no-auto-generate)`));
|
|
30728
|
+
try {
|
|
30729
|
+
const { crawlAndGenerate: crawlAndGenerate2 } = await Promise.resolve().then(() => (init_crawl_and_generate(), exports_crawl_and_generate));
|
|
30730
|
+
const crawlResult = await crawlAndGenerate2({
|
|
30731
|
+
url,
|
|
30732
|
+
projectId,
|
|
30733
|
+
maxPages: 5,
|
|
30734
|
+
scenariosPerPage: 2,
|
|
30735
|
+
model: opts.model,
|
|
30736
|
+
apiKey: process.env["ANTHROPIC_API_KEY"],
|
|
30737
|
+
headed: opts.headed,
|
|
30738
|
+
tags: ["auto-generated"]
|
|
30739
|
+
});
|
|
30740
|
+
log(chalk6.green(` Generated ${crawlResult.totalScenariosCreated} scenarios from ${crawlResult.pagesGenerated} page(s).`));
|
|
30741
|
+
log("");
|
|
30742
|
+
} catch (err) {
|
|
30743
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
30744
|
+
log(chalk6.yellow(` Auto-generate failed: ${msg}`));
|
|
30745
|
+
log(chalk6.yellow(` Continuing with 0 scenarios \u2014 the run will exit cleanly (0 passed, 0 failed).`));
|
|
30746
|
+
}
|
|
30747
|
+
}
|
|
30748
|
+
}
|
|
29505
30749
|
let diffScenarioIds;
|
|
29506
30750
|
if (opts.diff) {
|
|
29507
30751
|
try {
|
|
@@ -29564,6 +30808,18 @@ program2.command("run [url] [description]").alias("test").description("Run test
|
|
|
29564
30808
|
} else {
|
|
29565
30809
|
log(formatTerminal(run, results, { failedOnly: opts.failedOnly }));
|
|
29566
30810
|
}
|
|
30811
|
+
if (opts.githubComment) {
|
|
30812
|
+
const { postGitHubComment: postGitHubComment2 } = await Promise.resolve().then(() => (init_ci(), exports_ci));
|
|
30813
|
+
const prNumber = opts.pr ? parseInt(opts.pr, 10) : undefined;
|
|
30814
|
+
const posted = await postGitHubComment2(run, results, { prNumber });
|
|
30815
|
+
if (posted) {
|
|
30816
|
+
log(chalk6.green(" GitHub PR comment posted."));
|
|
30817
|
+
} else if (!process.env["GITHUB_TOKEN"]) {
|
|
30818
|
+
log(chalk6.yellow(" --github-comment: GITHUB_TOKEN not set, skipping PR comment."));
|
|
30819
|
+
} else {
|
|
30820
|
+
log(chalk6.yellow(" --github-comment: could not post PR comment (check GITHUB_PR_NUMBER / GITHUB_REF / GITHUB_REPOSITORY env)."));
|
|
30821
|
+
}
|
|
30822
|
+
}
|
|
29567
30823
|
process.exit(getExitCode(run));
|
|
29568
30824
|
} catch (error) {
|
|
29569
30825
|
logError(chalk6.red(`Error: ${error instanceof Error ? error.message : String(error)}`));
|
|
@@ -29741,7 +30997,7 @@ program2.command("import <dir>").description("Import markdown test files as scen
|
|
|
29741
30997
|
}
|
|
29742
30998
|
let imported = 0;
|
|
29743
30999
|
for (const file of files) {
|
|
29744
|
-
const content = readFileSync8(
|
|
31000
|
+
const content = readFileSync8(join16(absDir, file), "utf-8");
|
|
29745
31001
|
const lines = content.split(`
|
|
29746
31002
|
`);
|
|
29747
31003
|
let name = file.replace(/\.md$/, "");
|
|
@@ -29802,7 +31058,7 @@ program2.command("export [format]").description("Export scenarios as JSON (defau
|
|
|
29802
31058
|
}
|
|
29803
31059
|
const outputDir = opts.output ?? ".";
|
|
29804
31060
|
if (!existsSync14(outputDir)) {
|
|
29805
|
-
|
|
31061
|
+
mkdirSync11(outputDir, { recursive: true });
|
|
29806
31062
|
}
|
|
29807
31063
|
for (const s of scenarios) {
|
|
29808
31064
|
const lines = [];
|
|
@@ -29829,7 +31085,7 @@ program2.command("export [format]").description("Export scenarios as JSON (defau
|
|
|
29829
31085
|
lines.push("");
|
|
29830
31086
|
}
|
|
29831
31087
|
const safeFilename = s.name.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "").slice(0, 80);
|
|
29832
|
-
const filePath =
|
|
31088
|
+
const filePath = join16(outputDir, `${s.shortId}-${safeFilename}.md`);
|
|
29833
31089
|
writeFileSync4(filePath, lines.join(`
|
|
29834
31090
|
`), "utf-8");
|
|
29835
31091
|
log(chalk6.dim(` ${s.shortId}: ${s.name} \u2192 ${filePath}`));
|
|
@@ -29854,7 +31110,7 @@ program2.command("status").description("Show database and auth status").action((
|
|
|
29854
31110
|
try {
|
|
29855
31111
|
const config = loadConfig();
|
|
29856
31112
|
const hasApiKey = !!config.anthropicApiKey || !!process.env["ANTHROPIC_API_KEY"];
|
|
29857
|
-
const dbPath =
|
|
31113
|
+
const dbPath = join16(getTestersDir(), "testers.db");
|
|
29858
31114
|
log("");
|
|
29859
31115
|
log(chalk6.bold(" Open Testers Status"));
|
|
29860
31116
|
log("");
|
|
@@ -29974,7 +31230,7 @@ projectCmd.command("use <name>").description("Set active project (find or create
|
|
|
29974
31230
|
try {
|
|
29975
31231
|
const project = ensureProject(name, process.cwd());
|
|
29976
31232
|
if (!existsSync14(CONFIG_DIR5)) {
|
|
29977
|
-
|
|
31233
|
+
mkdirSync11(CONFIG_DIR5, { recursive: true });
|
|
29978
31234
|
}
|
|
29979
31235
|
let config = {};
|
|
29980
31236
|
if (existsSync14(CONFIG_PATH3)) {
|
|
@@ -30207,6 +31463,25 @@ Shutting down scheduler daemon...`));
|
|
|
30207
31463
|
process.exit(1);
|
|
30208
31464
|
}
|
|
30209
31465
|
});
|
|
31466
|
+
program2.command("ci [provider]").description("Print or write a CI workflow (default provider: github)").option("-o, --output <path>", "Write the workflow to a file (e.g. .github/workflows/qa.yml)").action(async (providerArg, opts) => {
|
|
31467
|
+
const provider = (providerArg ?? "github").toLowerCase();
|
|
31468
|
+
if (provider !== "github") {
|
|
31469
|
+
logError(chalk6.red(`Unknown CI provider: ${provider}. Supported: github`));
|
|
31470
|
+
process.exit(2);
|
|
31471
|
+
}
|
|
31472
|
+
const workflow = generateGitHubActionsWorkflow();
|
|
31473
|
+
if (opts.output) {
|
|
31474
|
+
const outPath = resolve(opts.output);
|
|
31475
|
+
const outDir = outPath.replace(/\/[^/]*$/, "");
|
|
31476
|
+
if (outDir && !existsSync14(outDir)) {
|
|
31477
|
+
mkdirSync11(outDir, { recursive: true });
|
|
31478
|
+
}
|
|
31479
|
+
writeFileSync4(outPath, workflow, "utf-8");
|
|
31480
|
+
log(chalk6.green(`Workflow written to ${outPath}`));
|
|
31481
|
+
return;
|
|
31482
|
+
}
|
|
31483
|
+
process.stdout.write(workflow);
|
|
31484
|
+
});
|
|
30210
31485
|
program2.command("init").description("Initialize a new testing project").option("-n, --name <name>", "Project name").option("-u, --url <url>", "Base URL").option("-p, --path <path>", "Project path").option("--ci <provider>", "Generate CI workflow (github)").option("-y, --yes", "Skip interactive prompts (non-interactive mode)", false).action(async (opts) => {
|
|
30211
31486
|
try {
|
|
30212
31487
|
const { project, scenarios, framework, url } = initProject({
|
|
@@ -30232,11 +31507,11 @@ program2.command("init").description("Initialize a new testing project").option(
|
|
|
30232
31507
|
log(` ${chalk6.dim(s.shortId)} ${s.name} ${chalk6.dim(`[${s.tags.join(", ")}]`)}`);
|
|
30233
31508
|
}
|
|
30234
31509
|
if (opts.ci === "github") {
|
|
30235
|
-
const workflowDir =
|
|
31510
|
+
const workflowDir = join16(process.cwd(), ".github", "workflows");
|
|
30236
31511
|
if (!existsSync14(workflowDir)) {
|
|
30237
|
-
|
|
31512
|
+
mkdirSync11(workflowDir, { recursive: true });
|
|
30238
31513
|
}
|
|
30239
|
-
const workflowPath =
|
|
31514
|
+
const workflowPath = join16(workflowDir, "testers.yml");
|
|
30240
31515
|
writeFileSync4(workflowPath, generateGitHubActionsWorkflow(), "utf-8");
|
|
30241
31516
|
log(` CI: ${chalk6.green("GitHub Actions workflow written to .github/workflows/testers.yml")}`);
|
|
30242
31517
|
} else if (opts.ci) {
|
|
@@ -31255,7 +32530,7 @@ program2.command("doctor").description("Check system setup and configuration").a
|
|
|
31255
32530
|
log(chalk6.red("\u2717") + " ANTHROPIC_API_KEY is not set (required for AI-powered tests)");
|
|
31256
32531
|
allPassed = false;
|
|
31257
32532
|
}
|
|
31258
|
-
const dbPath =
|
|
32533
|
+
const dbPath = join16(getTestersDir(), "testers.db");
|
|
31259
32534
|
try {
|
|
31260
32535
|
const { Database: Database4 } = await import("bun:sqlite");
|
|
31261
32536
|
const db2 = new Database4(dbPath, { create: true });
|
|
@@ -31266,8 +32541,8 @@ program2.command("doctor").description("Check system setup and configuration").a
|
|
|
31266
32541
|
allPassed = false;
|
|
31267
32542
|
}
|
|
31268
32543
|
try {
|
|
31269
|
-
const { chromium:
|
|
31270
|
-
const execPath =
|
|
32544
|
+
const { chromium: chromium3 } = await import("playwright");
|
|
32545
|
+
const execPath = chromium3.executablePath();
|
|
31271
32546
|
const { existsSync: fsExists } = await import("fs");
|
|
31272
32547
|
if (fsExists(execPath)) {
|
|
31273
32548
|
log(chalk6.green("\u2713") + " Playwright chromium is installed");
|
|
@@ -31307,7 +32582,7 @@ program2.command("serve").description("Start the Open Testers web dashboard").op
|
|
|
31307
32582
|
try {
|
|
31308
32583
|
const port = parseInt(opts.port, 10);
|
|
31309
32584
|
const url = `http://localhost:${port}`;
|
|
31310
|
-
const serverBin =
|
|
32585
|
+
const serverBin = join16(resolve(process.execPath, ".."), "..", "dist", "server", "index.js");
|
|
31311
32586
|
const { join: pathJoin, resolve: pathResolve, dirname: dirname4 } = await import("path");
|
|
31312
32587
|
const { fileURLToPath } = await import("url");
|
|
31313
32588
|
const serverPath = pathJoin(dirname4(fileURLToPath(import.meta.url)), "..", "server", "index.js");
|