@hasna/testers 0.0.28 → 0.0.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -154
- package/README.md +92 -0
- package/dist/cli/index.js +1354 -79
- package/dist/db/database.d.ts.map +1 -1
- package/dist/db/personas.d.ts.map +1 -1
- package/dist/db/runs.d.ts +29 -0
- package/dist/db/runs.d.ts.map +1 -1
- package/dist/db/scenarios.d.ts +12 -0
- package/dist/db/scenarios.d.ts.map +1 -1
- package/dist/db/sessions.d.ts +36 -0
- package/dist/db/sessions.d.ts.map +1 -0
- package/dist/db/step-results.d.ts +30 -0
- package/dist/db/step-results.d.ts.map +1 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +818 -25
- package/dist/lib/a11y-audit.d.ts +54 -0
- package/dist/lib/a11y-audit.d.ts.map +1 -0
- package/dist/lib/api-discovery.d.ts +46 -0
- package/dist/lib/api-discovery.d.ts.map +1 -0
- package/dist/lib/assertions.d.ts.map +1 -1
- package/dist/lib/auth-profiles.d.ts +16 -0
- package/dist/lib/auth-profiles.d.ts.map +1 -0
- package/dist/lib/auth-session-pool.d.ts +57 -0
- package/dist/lib/auth-session-pool.d.ts.map +1 -0
- package/dist/lib/batch-actions.d.ts +44 -0
- package/dist/lib/batch-actions.d.ts.map +1 -0
- package/dist/lib/browser-compat.d.ts +14 -0
- package/dist/lib/browser-compat.d.ts.map +1 -0
- package/dist/lib/browser.d.ts +6 -7
- package/dist/lib/browser.d.ts.map +1 -1
- package/dist/lib/ci.d.ts +12 -0
- package/dist/lib/ci.d.ts.map +1 -1
- package/dist/lib/config.d.ts.map +1 -1
- package/dist/lib/discovery.d.ts +33 -0
- package/dist/lib/discovery.d.ts.map +1 -0
- package/dist/lib/dom-mutation.d.ts +53 -0
- package/dist/lib/dom-mutation.d.ts.map +1 -0
- package/dist/lib/environment.d.ts +26 -0
- package/dist/lib/environment.d.ts.map +1 -0
- package/dist/lib/health-scan.d.ts +2 -1
- package/dist/lib/health-scan.d.ts.map +1 -1
- package/dist/lib/hybrid-runner.d.ts.map +1 -1
- package/dist/lib/junit-export.d.ts +24 -0
- package/dist/lib/junit-export.d.ts.map +1 -0
- package/dist/lib/network-mock.d.ts +38 -0
- package/dist/lib/network-mock.d.ts.map +1 -0
- package/dist/lib/offline-mode.d.ts +31 -0
- package/dist/lib/offline-mode.d.ts.map +1 -0
- package/dist/lib/pdf-export.d.ts +27 -0
- package/dist/lib/pdf-export.d.ts.map +1 -0
- package/dist/lib/performance.d.ts +65 -0
- package/dist/lib/performance.d.ts.map +1 -0
- package/dist/lib/pr-comment.d.ts +27 -0
- package/dist/lib/pr-comment.d.ts.map +1 -0
- package/dist/lib/preview-detect.d.ts +27 -0
- package/dist/lib/preview-detect.d.ts.map +1 -0
- package/dist/lib/prod-debug.d.ts +77 -0
- package/dist/lib/prod-debug.d.ts.map +1 -0
- package/dist/lib/recorder.d.ts +42 -0
- package/dist/lib/recorder.d.ts.map +1 -1
- package/dist/lib/repo-discovery.d.ts +102 -0
- package/dist/lib/repo-discovery.d.ts.map +1 -0
- package/dist/lib/repo-executor.d.ts +56 -0
- package/dist/lib/repo-executor.d.ts.map +1 -0
- package/dist/lib/responsive.d.ts +43 -0
- package/dist/lib/responsive.d.ts.map +1 -0
- package/dist/lib/runner.d.ts +1 -0
- package/dist/lib/runner.d.ts.map +1 -1
- package/dist/lib/scenario-chain.d.ts +52 -0
- package/dist/lib/scenario-chain.d.ts.map +1 -0
- package/dist/lib/templates.d.ts.map +1 -1
- package/dist/lib/webhooks.d.ts +3 -0
- package/dist/lib/webhooks.d.ts.map +1 -1
- package/dist/mcp/index.js +852 -74
- package/dist/sdk/index.d.ts +48 -0
- package/dist/sdk/index.d.ts.map +1 -0
- package/dist/server/index.js +276 -29
- package/dist/types/index.d.ts +66 -2
- package/dist/types/index.d.ts.map +1 -1
- package/package.json +2 -2
package/dist/index.js
CHANGED
|
@@ -65,7 +65,8 @@ function scenarioFromRow(row) {
|
|
|
65
65
|
createdAt: row.created_at,
|
|
66
66
|
updatedAt: row.updated_at,
|
|
67
67
|
lastPassedAt: row.last_passed_at ?? null,
|
|
68
|
-
lastPassedUrl: row.last_passed_url ?? null
|
|
68
|
+
lastPassedUrl: row.last_passed_url ?? null,
|
|
69
|
+
parameters: row.parameters ? JSON.parse(row.parameters) : null
|
|
69
70
|
};
|
|
70
71
|
}
|
|
71
72
|
function runFromRow(row) {
|
|
@@ -85,7 +86,14 @@ function runFromRow(row) {
|
|
|
85
86
|
metadata: row.metadata ? JSON.parse(row.metadata) : null,
|
|
86
87
|
isBaseline: row.is_baseline === 1,
|
|
87
88
|
samples: row.samples ?? 1,
|
|
88
|
-
flakinessThreshold: row.flakiness_threshold ?? 0.95
|
|
89
|
+
flakinessThreshold: row.flakiness_threshold ?? 0.95,
|
|
90
|
+
prNumber: row.pr_number ?? null,
|
|
91
|
+
prTitle: row.pr_title ?? null,
|
|
92
|
+
prBranch: row.pr_branch ?? null,
|
|
93
|
+
prBaseBranch: row.pr_base_branch ?? null,
|
|
94
|
+
prCommitSha: row.pr_commit_sha ?? null,
|
|
95
|
+
prUrl: row.pr_url ?? null,
|
|
96
|
+
ghAppInstallationId: row.gh_app_installation_id ?? null
|
|
89
97
|
};
|
|
90
98
|
}
|
|
91
99
|
function resultFromRow(row) {
|
|
@@ -106,7 +114,8 @@ function resultFromRow(row) {
|
|
|
106
114
|
createdAt: row.created_at,
|
|
107
115
|
personaId: row.persona_id ?? null,
|
|
108
116
|
personaName: row.persona_name ?? null,
|
|
109
|
-
failureAnalysis: row.failure_analysis ? JSON.parse(row.failure_analysis) : null
|
|
117
|
+
failureAnalysis: row.failure_analysis ? JSON.parse(row.failure_analysis) : null,
|
|
118
|
+
harPath: row.har_path ?? null
|
|
110
119
|
};
|
|
111
120
|
}
|
|
112
121
|
function screenshotFromRow(row) {
|
|
@@ -180,7 +189,10 @@ function personaFromRow(row) {
|
|
|
180
189
|
email: row.auth_email,
|
|
181
190
|
password: row.auth_password,
|
|
182
191
|
loginPath: row.auth_login_path ?? "/login",
|
|
183
|
-
cookies: row.auth_cookies ? JSON.parse(row.auth_cookies) : null
|
|
192
|
+
cookies: row.auth_cookies ? JSON.parse(row.auth_cookies) : null,
|
|
193
|
+
strategy: row.auth_strategy ?? "form-login",
|
|
194
|
+
headers: row.auth_headers ? JSON.parse(row.auth_headers) : undefined,
|
|
195
|
+
customScript: row.auth_script ?? undefined
|
|
184
196
|
} : null
|
|
185
197
|
};
|
|
186
198
|
}
|
|
@@ -10122,6 +10134,43 @@ ALTER TABLE scenarios ADD COLUMN required_role TEXT;
|
|
|
10122
10134
|
machine_id TEXT,
|
|
10123
10135
|
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
10124
10136
|
);
|
|
10137
|
+
`,
|
|
10138
|
+
`
|
|
10139
|
+
ALTER TABLE results ADD COLUMN har_path TEXT;
|
|
10140
|
+
`,
|
|
10141
|
+
`
|
|
10142
|
+
ALTER TABLE scenarios ADD COLUMN parameters TEXT;
|
|
10143
|
+
`,
|
|
10144
|
+
`
|
|
10145
|
+
ALTER TABLE personas ADD COLUMN auth_strategy TEXT DEFAULT 'form-login';
|
|
10146
|
+
ALTER TABLE personas ADD COLUMN auth_headers TEXT;
|
|
10147
|
+
ALTER TABLE personas ADD COLUMN auth_script TEXT;
|
|
10148
|
+
`,
|
|
10149
|
+
`
|
|
10150
|
+
CREATE TABLE IF NOT EXISTS step_results (
|
|
10151
|
+
id TEXT PRIMARY KEY,
|
|
10152
|
+
result_id TEXT NOT NULL REFERENCES results(id) ON DELETE CASCADE,
|
|
10153
|
+
step_number INTEGER NOT NULL,
|
|
10154
|
+
action TEXT NOT NULL,
|
|
10155
|
+
status TEXT NOT NULL DEFAULT 'running' CHECK(status IN ('passed','failed','error','running','skipped')),
|
|
10156
|
+
tool_name TEXT,
|
|
10157
|
+
tool_input TEXT,
|
|
10158
|
+
tool_result TEXT,
|
|
10159
|
+
thinking TEXT,
|
|
10160
|
+
error TEXT,
|
|
10161
|
+
duration_ms INTEGER,
|
|
10162
|
+
screenshot_id TEXT REFERENCES screenshots(id),
|
|
10163
|
+
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
10164
|
+
);
|
|
10165
|
+
`,
|
|
10166
|
+
`
|
|
10167
|
+
ALTER TABLE runs ADD COLUMN pr_number INTEGER;
|
|
10168
|
+
ALTER TABLE runs ADD COLUMN pr_title TEXT;
|
|
10169
|
+
ALTER TABLE runs ADD COLUMN pr_branch TEXT;
|
|
10170
|
+
ALTER TABLE runs ADD COLUMN pr_base_branch TEXT;
|
|
10171
|
+
ALTER TABLE runs ADD COLUMN pr_commit_sha TEXT;
|
|
10172
|
+
ALTER TABLE runs ADD COLUMN pr_url TEXT;
|
|
10173
|
+
ALTER TABLE runs ADD COLUMN gh_app_installation_id TEXT;
|
|
10125
10174
|
`
|
|
10126
10175
|
];
|
|
10127
10176
|
});
|
|
@@ -10130,8 +10179,12 @@ ALTER TABLE scenarios ADD COLUMN required_role TEXT;
|
|
|
10130
10179
|
var exports_runs = {};
|
|
10131
10180
|
__export(exports_runs, {
|
|
10132
10181
|
updateRun: () => updateRun,
|
|
10182
|
+
updatePrRunMetadata: () => updatePrRunMetadata,
|
|
10133
10183
|
listRuns: () => listRuns,
|
|
10184
|
+
listPrRuns: () => listPrRuns,
|
|
10185
|
+
getRunsByPr: () => getRunsByPr,
|
|
10134
10186
|
getRun: () => getRun,
|
|
10187
|
+
getLatestPrRun: () => getLatestPrRun,
|
|
10135
10188
|
deleteRun: () => deleteRun,
|
|
10136
10189
|
createRun: () => createRun,
|
|
10137
10190
|
countRuns: () => countRuns
|
|
@@ -10141,9 +10194,9 @@ function createRun(input) {
|
|
|
10141
10194
|
const id = uuid();
|
|
10142
10195
|
const timestamp = now();
|
|
10143
10196
|
db2.query(`
|
|
10144
|
-
INSERT INTO runs (id, project_id, status, url, model, headed, parallel, total, passed, failed, started_at, finished_at, metadata, samples, flakiness_threshold)
|
|
10145
|
-
VALUES (?, ?, 'pending', ?, ?, ?, ?, 0, 0, 0, ?, NULL, ?, ?, ?)
|
|
10146
|
-
`).run(id, input.projectId ?? null, input.url, input.model, input.headed ? 1 : 0, input.parallel ?? 1, timestamp,
|
|
10197
|
+
INSERT INTO runs (id, project_id, status, url, model, headed, parallel, total, passed, failed, started_at, finished_at, metadata, samples, flakiness_threshold, pr_number, pr_title, pr_branch, pr_base_branch, pr_commit_sha, pr_url, gh_app_installation_id)
|
|
10198
|
+
VALUES (?, ?, 'pending', ?, ?, ?, ?, 0, 0, 0, ?, NULL, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
10199
|
+
`).run(id, input.projectId ?? null, input.url, input.model, input.headed ? 1 : 0, input.parallel ?? 1, timestamp, JSON.stringify({}), input.samples ?? 1, input.flakinessThreshold ?? 0.95, input.prNumber ?? null, input.prTitle ?? null, input.prBranch ?? null, input.prBaseBranch ?? null, input.prCommitSha ?? null, input.prUrl ?? null, input.ghAppInstallationId ?? null);
|
|
10147
10200
|
return getRun(id);
|
|
10148
10201
|
}
|
|
10149
10202
|
function getRun(id) {
|
|
@@ -10171,6 +10224,14 @@ function listRuns(filter) {
|
|
|
10171
10224
|
conditions.push("status = ?");
|
|
10172
10225
|
params.push(filter.status);
|
|
10173
10226
|
}
|
|
10227
|
+
if (filter?.since) {
|
|
10228
|
+
conditions.push("started_at >= ?");
|
|
10229
|
+
params.push(filter.since);
|
|
10230
|
+
}
|
|
10231
|
+
if (filter?.until) {
|
|
10232
|
+
conditions.push("started_at <= ?");
|
|
10233
|
+
params.push(filter.until);
|
|
10234
|
+
}
|
|
10174
10235
|
let sql = "SELECT * FROM runs";
|
|
10175
10236
|
if (conditions.length > 0) {
|
|
10176
10237
|
sql += " WHERE " + conditions.join(" AND ");
|
|
@@ -10202,6 +10263,14 @@ function countRuns(filter) {
|
|
|
10202
10263
|
conditions.push("status = ?");
|
|
10203
10264
|
params.push(filter.status);
|
|
10204
10265
|
}
|
|
10266
|
+
if (filter?.since) {
|
|
10267
|
+
conditions.push("started_at >= ?");
|
|
10268
|
+
params.push(filter.since);
|
|
10269
|
+
}
|
|
10270
|
+
if (filter?.until) {
|
|
10271
|
+
conditions.push("started_at <= ?");
|
|
10272
|
+
params.push(filter.until);
|
|
10273
|
+
}
|
|
10205
10274
|
let sql = "SELECT COUNT(*) as count FROM runs";
|
|
10206
10275
|
if (conditions.length > 0)
|
|
10207
10276
|
sql += " WHERE " + conditions.join(" AND ");
|
|
@@ -10279,6 +10348,52 @@ function deleteRun(id) {
|
|
|
10279
10348
|
const result = db2.query("DELETE FROM runs WHERE id = ?").run(run.id);
|
|
10280
10349
|
return result.changes > 0;
|
|
10281
10350
|
}
|
|
10351
|
+
function getRunsByPr(prNumber) {
|
|
10352
|
+
const db2 = getDatabase();
|
|
10353
|
+
const rows = db2.query("SELECT * FROM runs WHERE pr_number = ? ORDER BY started_at DESC").all(prNumber);
|
|
10354
|
+
return rows.map(runFromRow);
|
|
10355
|
+
}
|
|
10356
|
+
function getLatestPrRun(prNumber) {
|
|
10357
|
+
const db2 = getDatabase();
|
|
10358
|
+
const row = db2.query("SELECT * FROM runs WHERE pr_number = ? ORDER BY started_at DESC, rowid DESC LIMIT 1").get(prNumber);
|
|
10359
|
+
return row ? runFromRow(row) : null;
|
|
10360
|
+
}
|
|
10361
|
+
function listPrRuns(filter) {
|
|
10362
|
+
const db2 = getDatabase();
|
|
10363
|
+
const conditions = ["pr_number IS NOT NULL"];
|
|
10364
|
+
const params = [];
|
|
10365
|
+
if (filter?.branch) {
|
|
10366
|
+
conditions.push("pr_branch = ?");
|
|
10367
|
+
params.push(filter.branch);
|
|
10368
|
+
}
|
|
10369
|
+
if (filter?.baseBranch) {
|
|
10370
|
+
conditions.push("pr_base_branch = ?");
|
|
10371
|
+
params.push(filter.baseBranch);
|
|
10372
|
+
}
|
|
10373
|
+
let sql = "SELECT * FROM runs WHERE " + conditions.join(" AND ") + " ORDER BY started_at DESC";
|
|
10374
|
+
if (filter?.limit) {
|
|
10375
|
+
sql += " LIMIT ?";
|
|
10376
|
+
params.push(filter.limit);
|
|
10377
|
+
}
|
|
10378
|
+
if (filter?.offset) {
|
|
10379
|
+
sql += " OFFSET ?";
|
|
10380
|
+
params.push(filter.offset);
|
|
10381
|
+
}
|
|
10382
|
+
const rows = db2.query(sql).all(...params);
|
|
10383
|
+
return rows.map(runFromRow);
|
|
10384
|
+
}
|
|
10385
|
+
function updatePrRunMetadata(runId, prData) {
|
|
10386
|
+
const db2 = getDatabase();
|
|
10387
|
+
const existing = getRun(runId);
|
|
10388
|
+
if (!existing) {
|
|
10389
|
+
throw new Error(`Run not found: ${runId}`);
|
|
10390
|
+
}
|
|
10391
|
+
db2.query(`
|
|
10392
|
+
UPDATE runs SET pr_number = ?, pr_title = ?, pr_branch = ?, pr_base_branch = ?, pr_commit_sha = ?, pr_url = ?, gh_app_installation_id = ?
|
|
10393
|
+
WHERE id = ?
|
|
10394
|
+
`).run(prData.prNumber, prData.prTitle ?? null, prData.prBranch ?? null, prData.prBaseBranch ?? null, prData.prCommitSha ?? null, prData.prUrl ?? null, prData.ghAppInstallationId ?? null, runId);
|
|
10395
|
+
return getRun(runId);
|
|
10396
|
+
}
|
|
10282
10397
|
var init_runs = __esm(() => {
|
|
10283
10398
|
init_types();
|
|
10284
10399
|
init_database();
|
|
@@ -10467,7 +10582,7 @@ function getDefaultConfig() {
|
|
|
10467
10582
|
browser: {
|
|
10468
10583
|
headless: true,
|
|
10469
10584
|
viewport: { width: 1280, height: 720 },
|
|
10470
|
-
timeout:
|
|
10585
|
+
timeout: 120000
|
|
10471
10586
|
},
|
|
10472
10587
|
screenshots: {
|
|
10473
10588
|
dir: join8(getTestersDir(), "screenshots"),
|
|
@@ -10497,7 +10612,8 @@ function loadConfig() {
|
|
|
10497
10612
|
judgeModel: fileConfig.judgeModel,
|
|
10498
10613
|
judgeProvider: fileConfig.judgeProvider,
|
|
10499
10614
|
selfHeal: fileConfig.selfHeal ?? false,
|
|
10500
|
-
conversationsSpace: fileConfig.conversationsSpace
|
|
10615
|
+
conversationsSpace: fileConfig.conversationsSpace,
|
|
10616
|
+
prodDebug: fileConfig.prodDebug
|
|
10501
10617
|
};
|
|
10502
10618
|
const envModel = process.env["TESTERS_MODEL"];
|
|
10503
10619
|
if (envModel) {
|
|
@@ -11151,6 +11267,16 @@ async function launchBrowser(options) {
|
|
|
11151
11267
|
const headless = options?.headless ?? true;
|
|
11152
11268
|
const viewport = options?.viewport ?? DEFAULT_VIEWPORT;
|
|
11153
11269
|
try {
|
|
11270
|
+
if (engine === "playwright-firefox") {
|
|
11271
|
+
const { firefox } = await import("playwright");
|
|
11272
|
+
const browser = await firefox.launch({ headless });
|
|
11273
|
+
return browser;
|
|
11274
|
+
}
|
|
11275
|
+
if (engine === "playwright-webkit") {
|
|
11276
|
+
const { webkit } = await import("playwright");
|
|
11277
|
+
const browser = await webkit.launch({ headless });
|
|
11278
|
+
return browser;
|
|
11279
|
+
}
|
|
11154
11280
|
return await launchPlaywright({ headless, viewport });
|
|
11155
11281
|
} catch (error) {
|
|
11156
11282
|
const message = error instanceof Error ? error.message : String(error);
|
|
@@ -11267,8 +11393,9 @@ async function installBrowser(engine) {
|
|
|
11267
11393
|
const { installLightpanda: installLightpanda2 } = await Promise.resolve().then(() => (init_browser_lightpanda(), exports_browser_lightpanda));
|
|
11268
11394
|
return installLightpanda2();
|
|
11269
11395
|
}
|
|
11396
|
+
const browserName = engine === "playwright-firefox" ? "firefox" : engine === "playwright-webkit" ? "webkit" : "chromium";
|
|
11270
11397
|
try {
|
|
11271
|
-
execSync(
|
|
11398
|
+
execSync(`bunx playwright install ${browserName}`, {
|
|
11272
11399
|
stdio: "inherit"
|
|
11273
11400
|
});
|
|
11274
11401
|
} catch (error) {
|
|
@@ -12410,9 +12537,9 @@ function createScenario(input) {
|
|
|
12410
12537
|
const short_id = nextShortId(input.projectId);
|
|
12411
12538
|
const timestamp = now();
|
|
12412
12539
|
db2.query(`
|
|
12413
|
-
INSERT INTO scenarios (id, short_id, project_id, name, description, steps, tags, priority, model, timeout_ms, target_path, requires_auth, auth_config, metadata, assertions, version, created_at, updated_at)
|
|
12414
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1, ?, ?)
|
|
12415
|
-
`).run(id, short_id, input.projectId ?? null, input.name, input.description, JSON.stringify(input.steps ?? []), JSON.stringify(input.tags ?? []), input.priority ?? "medium", input.model ?? null, input.timeoutMs ?? null, input.targetPath ?? null, input.requiresAuth ? 1 : 0, input.authConfig ? JSON.stringify(input.authConfig) : null, input.metadata ? JSON.stringify(input.metadata) : null, JSON.stringify(input.assertions ?? []), timestamp, timestamp);
|
|
12540
|
+
INSERT INTO scenarios (id, short_id, project_id, name, description, steps, tags, priority, model, timeout_ms, target_path, requires_auth, auth_config, metadata, assertions, parameters, version, created_at, updated_at)
|
|
12541
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1, ?, ?)
|
|
12542
|
+
`).run(id, short_id, input.projectId ?? null, input.name, input.description, JSON.stringify(input.steps ?? []), JSON.stringify(input.tags ?? []), input.priority ?? "medium", input.model ?? null, input.timeoutMs ?? null, input.targetPath ?? null, input.requiresAuth ? 1 : 0, input.authConfig ? JSON.stringify(input.authConfig) : null, input.metadata ? JSON.stringify(input.metadata) : null, JSON.stringify(input.assertions ?? []), input.parameters ? JSON.stringify(input.parameters) : null, timestamp, timestamp);
|
|
12416
12543
|
return getScenario(id);
|
|
12417
12544
|
}
|
|
12418
12545
|
function getScenario(id) {
|
|
@@ -12562,6 +12689,10 @@ function updateScenario(id, input, version) {
|
|
|
12562
12689
|
sets.push("assertions = ?");
|
|
12563
12690
|
params.push(JSON.stringify(input.assertions));
|
|
12564
12691
|
}
|
|
12692
|
+
if (input.parameters !== undefined) {
|
|
12693
|
+
sets.push("parameters = ?");
|
|
12694
|
+
params.push(JSON.stringify(input.parameters));
|
|
12695
|
+
}
|
|
12565
12696
|
if (sets.length === 0) {
|
|
12566
12697
|
return existing;
|
|
12567
12698
|
}
|
|
@@ -13597,6 +13728,8 @@ async function runPipelineScenario(scenario, options) {
|
|
|
13597
13728
|
|
|
13598
13729
|
// src/lib/runner.ts
|
|
13599
13730
|
init_runs();
|
|
13731
|
+
import { mkdirSync as mkdirSync8 } from "fs";
|
|
13732
|
+
import { join as join13 } from "path";
|
|
13600
13733
|
|
|
13601
13734
|
// src/lib/failure-analyzer.ts
|
|
13602
13735
|
function analyzeFailure(error, reasoning) {
|
|
@@ -14378,6 +14511,74 @@ function formatCostsJSON(summary) {
|
|
|
14378
14511
|
return JSON.stringify(summary, null, 2);
|
|
14379
14512
|
}
|
|
14380
14513
|
|
|
14514
|
+
// src/db/step-results.ts
|
|
14515
|
+
init_database();
|
|
14516
|
+
function createStepResult(input) {
|
|
14517
|
+
const db2 = getDatabase();
|
|
14518
|
+
const id = uuid();
|
|
14519
|
+
const timestamp = now();
|
|
14520
|
+
db2.query(`
|
|
14521
|
+
INSERT INTO step_results (id, result_id, step_number, action, status, tool_name, tool_input, thinking, created_at)
|
|
14522
|
+
VALUES (?, ?, ?, ?, 'running', ?, ?, ?, ?)
|
|
14523
|
+
`).run(id, input.resultId, input.stepNumber, input.action, input.toolName ?? null, input.toolInput ? JSON.stringify(input.toolInput) : null, input.thinking ?? null, timestamp);
|
|
14524
|
+
return getStepResult(id);
|
|
14525
|
+
}
|
|
14526
|
+
function getStepResult(id) {
|
|
14527
|
+
const db2 = getDatabase();
|
|
14528
|
+
const row = db2.query("SELECT * FROM step_results WHERE id = ?").get(id);
|
|
14529
|
+
return row ? stepResultFromRow(row) : null;
|
|
14530
|
+
}
|
|
14531
|
+
function updateStepResult(id, updates) {
|
|
14532
|
+
const db2 = getDatabase();
|
|
14533
|
+
const existing = getStepResult(id);
|
|
14534
|
+
if (!existing)
|
|
14535
|
+
return null;
|
|
14536
|
+
const sets = [];
|
|
14537
|
+
const params = [];
|
|
14538
|
+
if (updates.status !== undefined) {
|
|
14539
|
+
sets.push("status = ?");
|
|
14540
|
+
params.push(updates.status);
|
|
14541
|
+
}
|
|
14542
|
+
if (updates.toolResult !== undefined) {
|
|
14543
|
+
sets.push("tool_result = ?");
|
|
14544
|
+
params.push(updates.toolResult);
|
|
14545
|
+
}
|
|
14546
|
+
if (updates.error !== undefined) {
|
|
14547
|
+
sets.push("error = ?");
|
|
14548
|
+
params.push(updates.error);
|
|
14549
|
+
}
|
|
14550
|
+
if (updates.durationMs !== undefined) {
|
|
14551
|
+
sets.push("duration_ms = ?");
|
|
14552
|
+
params.push(updates.durationMs);
|
|
14553
|
+
}
|
|
14554
|
+
if (updates.screenshotId !== undefined) {
|
|
14555
|
+
sets.push("screenshot_id = ?");
|
|
14556
|
+
params.push(updates.screenshotId);
|
|
14557
|
+
}
|
|
14558
|
+
if (sets.length === 0)
|
|
14559
|
+
return existing;
|
|
14560
|
+
params.push(id);
|
|
14561
|
+
db2.query(`UPDATE step_results SET ${sets.join(", ")} WHERE id = ?`).run(...params);
|
|
14562
|
+
return getStepResult(id);
|
|
14563
|
+
}
|
|
14564
|
+
function stepResultFromRow(row) {
|
|
14565
|
+
return {
|
|
14566
|
+
id: row.id,
|
|
14567
|
+
resultId: row.result_id,
|
|
14568
|
+
stepNumber: row.step_number,
|
|
14569
|
+
action: row.action,
|
|
14570
|
+
status: row.status,
|
|
14571
|
+
toolName: row.tool_name,
|
|
14572
|
+
toolInput: row.tool_input ? JSON.parse(row.tool_input) : null,
|
|
14573
|
+
toolResult: row.tool_result,
|
|
14574
|
+
thinking: row.thinking,
|
|
14575
|
+
error: row.error,
|
|
14576
|
+
durationMs: row.duration_ms,
|
|
14577
|
+
screenshotId: row.screenshot_id,
|
|
14578
|
+
createdAt: row.created_at
|
|
14579
|
+
};
|
|
14580
|
+
}
|
|
14581
|
+
|
|
14381
14582
|
// src/db/personas.ts
|
|
14382
14583
|
init_types();
|
|
14383
14584
|
init_database();
|
|
@@ -14734,6 +14935,24 @@ function signPayload(body, secret) {
|
|
|
14734
14935
|
}
|
|
14735
14936
|
return `sha256=${Math.abs(hash).toString(16).padStart(16, "0")}`;
|
|
14736
14937
|
}
|
|
14938
|
+
function formatDiscordPayload(payload) {
|
|
14939
|
+
const isPassed = payload.run.status === "passed";
|
|
14940
|
+
const color = isPassed ? 2278750 : 15680580;
|
|
14941
|
+
return {
|
|
14942
|
+
username: "open-testers",
|
|
14943
|
+
embeds: [
|
|
14944
|
+
{
|
|
14945
|
+
title: `Test Run ${payload.run.status.toUpperCase()}`,
|
|
14946
|
+
color,
|
|
14947
|
+
description: `URL: ${payload.run.url}
|
|
14948
|
+
` + `Results: ${payload.run.passed}/${payload.run.total} passed` + (payload.run.failed > 0 ? ` (${payload.run.failed} failed)` : "") + (payload.schedule ? `
|
|
14949
|
+
Schedule: ${payload.schedule.name}` : ""),
|
|
14950
|
+
timestamp: payload.timestamp,
|
|
14951
|
+
footer: { text: "open-testers" }
|
|
14952
|
+
}
|
|
14953
|
+
]
|
|
14954
|
+
};
|
|
14955
|
+
}
|
|
14737
14956
|
function formatSlackPayload(payload) {
|
|
14738
14957
|
const status = payload.run.status === "passed" ? ":white_check_mark:" : ":x:";
|
|
14739
14958
|
const color = payload.run.status === "passed" ? "#22c55e" : "#ef4444";
|
|
@@ -14776,7 +14995,8 @@ async function dispatchWebhooks(event, run, schedule) {
|
|
|
14776
14995
|
if (!webhook.events.includes(event) && !webhook.events.includes("*"))
|
|
14777
14996
|
continue;
|
|
14778
14997
|
const isSlack = webhook.url.includes("hooks.slack.com");
|
|
14779
|
-
const
|
|
14998
|
+
const isDiscord = webhook.url.includes("discord.com/api/webhooks") || webhook.url.includes("discordapp.com/api/webhooks");
|
|
14999
|
+
const body = isSlack ? JSON.stringify(formatSlackPayload(payload)) : isDiscord ? JSON.stringify(formatDiscordPayload(payload)) : JSON.stringify(payload);
|
|
14780
15000
|
const headers = {
|
|
14781
15001
|
"Content-Type": "application/json"
|
|
14782
15002
|
};
|
|
@@ -15186,13 +15406,35 @@ async function runSingleScenario(scenario, runId, options) {
|
|
|
15186
15406
|
emit({ type: "scenario:start", scenarioId: scenario.id, scenarioName: scenario.name, resultId: result.id, runId });
|
|
15187
15407
|
let browser = null;
|
|
15188
15408
|
let page = null;
|
|
15409
|
+
let context = null;
|
|
15410
|
+
let harPath = null;
|
|
15189
15411
|
let stopNetworkLogging = null;
|
|
15190
15412
|
const networkErrors = [];
|
|
15191
15413
|
try {
|
|
15192
15414
|
browser = await launchBrowser({ headless: !(effectiveOptions.headed ?? false), engine: effectiveOptions.engine });
|
|
15193
|
-
|
|
15194
|
-
|
|
15195
|
-
|
|
15415
|
+
const useHar = effectiveOptions.engine !== "lightpanda" && effectiveOptions.engine !== "bun";
|
|
15416
|
+
if (useHar) {
|
|
15417
|
+
const testersDir = process.env["HASNA_TESTERS_DIR"] || join13(process.env["HOME"] || "", ".hasna", "testers");
|
|
15418
|
+
const harDir = join13(testersDir, "hars");
|
|
15419
|
+
mkdirSync8(harDir, { recursive: true });
|
|
15420
|
+
harPath = join13(harDir, `${result.id}.har`);
|
|
15421
|
+
const contextOptions = {
|
|
15422
|
+
viewport: config.browser.viewport,
|
|
15423
|
+
recordHar: { path: harPath, mode: "full" }
|
|
15424
|
+
};
|
|
15425
|
+
if (effectiveOptions.recordVideo) {
|
|
15426
|
+
const videoDir = join13(testersDir, "videos");
|
|
15427
|
+
mkdirSync8(videoDir, { recursive: true });
|
|
15428
|
+
contextOptions.recordVideo = { dir: videoDir, size: config.browser.viewport };
|
|
15429
|
+
}
|
|
15430
|
+
context = await browser.newContext(contextOptions);
|
|
15431
|
+
page = await context.newPage();
|
|
15432
|
+
} else {
|
|
15433
|
+
page = await getPage(browser, {
|
|
15434
|
+
viewport: config.browser.viewport,
|
|
15435
|
+
engine: effectiveOptions.engine
|
|
15436
|
+
});
|
|
15437
|
+
}
|
|
15196
15438
|
const targetUrl = scenario.targetPath ? `${options.url.replace(/\/$/, "")}${scenario.targetPath}` : options.url;
|
|
15197
15439
|
const scenarioTimeout = scenario.timeoutMs ?? options.timeout ?? config.browser.timeout ?? 60000;
|
|
15198
15440
|
registerSession({
|
|
@@ -15212,7 +15454,11 @@ async function runSingleScenario(scenario, runId, options) {
|
|
|
15212
15454
|
}
|
|
15213
15455
|
});
|
|
15214
15456
|
const consoleErrors = [];
|
|
15457
|
+
const consoleLogs = [];
|
|
15458
|
+
let currentStep = 0;
|
|
15215
15459
|
page.on("console", (msg) => {
|
|
15460
|
+
const logEntry = { step: currentStep > 0 ? currentStep : null, type: msg.type(), text: msg.text(), timestamp: Date.now() };
|
|
15461
|
+
consoleLogs.push(logEntry);
|
|
15216
15462
|
if (msg.type() === "error")
|
|
15217
15463
|
consoleErrors.push(msg.text());
|
|
15218
15464
|
});
|
|
@@ -15244,6 +15490,7 @@ async function runSingleScenario(scenario, runId, options) {
|
|
|
15244
15490
|
}
|
|
15245
15491
|
await page.goto(targetUrl, { timeout: Math.min(scenarioTimeout, 30000) });
|
|
15246
15492
|
const stepStartTimes = new Map;
|
|
15493
|
+
const stepResultIds = new Map;
|
|
15247
15494
|
const agentResult = await withTimeout(runAgentLoop({
|
|
15248
15495
|
client,
|
|
15249
15496
|
page,
|
|
@@ -15266,13 +15513,32 @@ async function runSingleScenario(scenario, runId, options) {
|
|
|
15266
15513
|
onStep: (stepEvent) => {
|
|
15267
15514
|
let stepDurationMs;
|
|
15268
15515
|
if (stepEvent.type === "tool_call") {
|
|
15516
|
+
currentStep = stepEvent.stepNumber;
|
|
15269
15517
|
stepStartTimes.set(stepEvent.stepNumber, Date.now());
|
|
15518
|
+
const stepResult = createStepResult({
|
|
15519
|
+
resultId: result.id,
|
|
15520
|
+
stepNumber: stepEvent.stepNumber,
|
|
15521
|
+
action: stepEvent.toolName ?? `step-${stepEvent.stepNumber}`,
|
|
15522
|
+
toolName: stepEvent.toolName,
|
|
15523
|
+
toolInput: stepEvent.toolInput,
|
|
15524
|
+
thinking: stepEvent.thinking
|
|
15525
|
+
});
|
|
15526
|
+
stepResultIds.set(stepEvent.stepNumber, stepResult.id);
|
|
15270
15527
|
} else if (stepEvent.type === "tool_result") {
|
|
15271
15528
|
const startTime = stepStartTimes.get(stepEvent.stepNumber);
|
|
15272
15529
|
if (startTime !== undefined) {
|
|
15273
15530
|
stepDurationMs = Date.now() - startTime;
|
|
15274
15531
|
stepStartTimes.delete(stepEvent.stepNumber);
|
|
15275
15532
|
}
|
|
15533
|
+
const stepResultId = stepResultIds.get(stepEvent.stepNumber);
|
|
15534
|
+
if (stepResultId) {
|
|
15535
|
+
const isSuccess = !stepEvent.toolResult?.toLowerCase().includes("error") && !stepEvent.toolResult?.toLowerCase().includes("failed");
|
|
15536
|
+
updateStepResult(stepResultId, {
|
|
15537
|
+
status: isSuccess ? "passed" : "failed",
|
|
15538
|
+
toolResult: stepEvent.toolResult,
|
|
15539
|
+
durationMs: stepDurationMs
|
|
15540
|
+
});
|
|
15541
|
+
}
|
|
15276
15542
|
}
|
|
15277
15543
|
emit({
|
|
15278
15544
|
type: `step:${stepEvent.type}`,
|
|
@@ -15321,7 +15587,7 @@ async function runSingleScenario(scenario, runId, options) {
|
|
|
15321
15587
|
durationMs: Date.now() - new Date(result.createdAt).getTime(),
|
|
15322
15588
|
tokensUsed: agentResult.tokensUsed,
|
|
15323
15589
|
costCents: estimateCost(model, agentResult.tokensUsed),
|
|
15324
|
-
metadata: networkErrors.length > 0 ? networkMeta :
|
|
15590
|
+
metadata: { consoleLogs, ...networkErrors.length > 0 ? networkMeta : {} }
|
|
15325
15591
|
});
|
|
15326
15592
|
if (agentResult.status === "failed" || agentResult.status === "error") {
|
|
15327
15593
|
const failureAnalysis = analyzeFailure(null, agentResult.reasoning ?? null);
|
|
@@ -15357,8 +15623,16 @@ async function runSingleScenario(scenario, runId, options) {
|
|
|
15357
15623
|
emit({ type: "scenario:error", scenarioId: scenario.id, scenarioName: scenario.name, error: errorMsg, runId });
|
|
15358
15624
|
return updatedResult;
|
|
15359
15625
|
} finally {
|
|
15360
|
-
if (
|
|
15361
|
-
|
|
15626
|
+
if (harPath) {
|
|
15627
|
+
try {
|
|
15628
|
+
updateResult(result.id, { metadata: { harPath } });
|
|
15629
|
+
} catch {}
|
|
15630
|
+
}
|
|
15631
|
+
if (browser) {
|
|
15632
|
+
try {
|
|
15633
|
+
await closeBrowser(browser, effectiveOptions.engine);
|
|
15634
|
+
} catch {}
|
|
15635
|
+
}
|
|
15362
15636
|
}
|
|
15363
15637
|
}
|
|
15364
15638
|
async function runBatch(scenarios, options) {
|
|
@@ -16111,10 +16385,10 @@ class Scheduler {
|
|
|
16111
16385
|
}
|
|
16112
16386
|
// src/lib/init.ts
|
|
16113
16387
|
init_paths();
|
|
16114
|
-
import { existsSync as existsSync11, readFileSync as readFileSync3, writeFileSync as writeFileSync3, mkdirSync as
|
|
16115
|
-
import { join as
|
|
16388
|
+
import { existsSync as existsSync11, readFileSync as readFileSync3, writeFileSync as writeFileSync3, mkdirSync as mkdirSync9 } from "fs";
|
|
16389
|
+
import { join as join14, basename } from "path";
|
|
16116
16390
|
function detectFramework(dir) {
|
|
16117
|
-
const pkgPath =
|
|
16391
|
+
const pkgPath = join14(dir, "package.json");
|
|
16118
16392
|
if (!existsSync11(pkgPath))
|
|
16119
16393
|
return null;
|
|
16120
16394
|
let pkg;
|
|
@@ -16342,9 +16616,9 @@ function initProject(options) {
|
|
|
16342
16616
|
}
|
|
16343
16617
|
}).filter((s) => s !== null);
|
|
16344
16618
|
const configDir = getTestersDir();
|
|
16345
|
-
const configPath =
|
|
16619
|
+
const configPath = join14(configDir, "config.json");
|
|
16346
16620
|
if (!existsSync11(configDir)) {
|
|
16347
|
-
|
|
16621
|
+
mkdirSync9(configDir, { recursive: true });
|
|
16348
16622
|
}
|
|
16349
16623
|
let config = {};
|
|
16350
16624
|
if (existsSync11(configPath)) {
|
|
@@ -16747,6 +17021,18 @@ var SCENARIO_TEMPLATES = {
|
|
|
16747
17021
|
a11y: [
|
|
16748
17022
|
{ name: "Keyboard navigation", description: "Navigate the page using only keyboard (Tab, Enter, Escape). Verify all interactive elements are reachable and focusable.", tags: ["a11y", "keyboard"], priority: "high", steps: ["Press Tab to move through elements", "Verify focus indicators are visible", "Press Enter on buttons/links", "Verify actions trigger correctly", "Press Escape to close modals/dropdowns"] },
|
|
16749
17023
|
{ name: "Image alt text", description: "Check that all images have meaningful alt text attributes.", tags: ["a11y"], priority: "medium", steps: ["Find all images on the page", "Check each image has an alt attribute", "Verify alt text is descriptive, not empty or generic"] }
|
|
17024
|
+
],
|
|
17025
|
+
checkout: [
|
|
17026
|
+
{ name: "Add item to cart", description: "Navigate to a product page, add an item to cart, and verify cart count increments.", tags: ["checkout", "cart"], priority: "critical", steps: ["Navigate to product detail page", "Click add to cart button", "Verify cart count badge updates", "Verify success notification appears"] },
|
|
17027
|
+
{ name: "Cart page shows added items", description: "Navigate to the cart page and verify previously added items are listed with correct prices and quantities.", tags: ["checkout", "cart"], priority: "high", steps: ["Navigate to cart page", "Verify all added items are listed", "Verify prices and quantities are correct", "Verify subtotal is calculated"] },
|
|
17028
|
+
{ name: "Checkout flow completion", description: "Complete the full checkout flow: cart -> shipping -> payment -> confirmation. Verify order is placed successfully.", tags: ["checkout", "smoke"], priority: "critical", steps: ["Navigate to cart", "Click proceed to checkout", "Fill shipping address", "Select shipping method", "Fill payment details", "Review order summary", "Place order", "Verify order confirmation page"] },
|
|
17029
|
+
{ name: "Apply coupon/discount code", description: "Enter a valid coupon code on the cart or checkout page and verify discount is applied to the total.", tags: ["checkout", "discount"], priority: "high", steps: ["Navigate to cart or checkout", "Enter coupon code", "Click apply", "Verify discount amount is shown", "Verify total is updated"] }
|
|
17030
|
+
],
|
|
17031
|
+
search: [
|
|
17032
|
+
{ name: "Search returns relevant results", description: "Enter a known search term and verify relevant results appear. Check that results match the query intent.", tags: ["search"], priority: "high", steps: ["Navigate to search page or focus search bar", "Enter known search term", "Submit search", "Verify results appear", "Verify result titles contain search term"] },
|
|
17033
|
+
{ name: "Empty search handling", description: "Submit an empty search and verify the system handles it gracefully (show all results or a prompt, no errors).", tags: ["search", "validation"], priority: "medium", steps: ["Focus search bar", "Submit empty search", "Verify no crash or error state", "Verify fallback behavior (all results or prompt)"] },
|
|
17034
|
+
{ name: "No results handling", description: "Search for a term that returns no results and verify appropriate 'no results found' messaging is displayed.", tags: ["search"], priority: "medium", steps: ["Navigate to search", "Enter nonsense term like 'xyzqwrtp'", "Submit search", "Verify no results found message", "Verify no error states"] },
|
|
17035
|
+
{ name: "Search filters work", description: "Perform a search, apply a filter (category, date, price range), and verify results are narrowed down.", tags: ["search", "filter"], priority: "high", steps: ["Perform initial search", "Note initial result count", "Apply a filter", "Verify result count decreased", "Verify results match filter criteria"] }
|
|
16750
17036
|
]
|
|
16751
17037
|
};
|
|
16752
17038
|
function getTemplate(name) {
|
|
@@ -17084,6 +17370,505 @@ async function startWatcher(options) {
|
|
|
17084
17370
|
process.on("SIGTERM", cleanup);
|
|
17085
17371
|
await new Promise(() => {});
|
|
17086
17372
|
}
|
|
17373
|
+
// src/lib/prod-debug.ts
|
|
17374
|
+
var UUID_RE = /\b[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}\b/i;
|
|
17375
|
+
var SENSITIVE_PARAM_RE = /token|secret|key|password|code|state|cookie|session|grant|credential|auth|jwt|access/i;
|
|
17376
|
+
var SENSITIVE_TEXT_RE = /\b(Bearer\s+[A-Za-z0-9._-]{12,}|sk-[A-Za-z0-9]{12,}|pk_[A-Za-z0-9]{12,}|eyJ[A-Za-z0-9._-]{12,})\b/g;
|
|
17377
|
+
var URL_TEXT_RE = /https?:\/\/[^\s"'<>]+/g;
|
|
17378
|
+
function safeUrl(raw) {
|
|
17379
|
+
try {
|
|
17380
|
+
const url = new URL(raw);
|
|
17381
|
+
if (url.protocol !== "http:" && url.protocol !== "https:")
|
|
17382
|
+
return null;
|
|
17383
|
+
return url;
|
|
17384
|
+
} catch {
|
|
17385
|
+
return null;
|
|
17386
|
+
}
|
|
17387
|
+
}
|
|
17388
|
+
function normalizeOrigin(raw) {
|
|
17389
|
+
const url = safeUrl(raw);
|
|
17390
|
+
if (url)
|
|
17391
|
+
return url.origin;
|
|
17392
|
+
const hostUrl = safeUrl(`https://${raw}`);
|
|
17393
|
+
return hostUrl?.origin ?? null;
|
|
17394
|
+
}
|
|
17395
|
+
function redactProdDebugText(value) {
|
|
17396
|
+
return value.replace(URL_TEXT_RE, (match) => {
|
|
17397
|
+
const url = safeUrl(match);
|
|
17398
|
+
return url ? redactUrl(url) : match;
|
|
17399
|
+
}).replace(SENSITIVE_TEXT_RE, (match) => {
|
|
17400
|
+
if (match.startsWith("Bearer "))
|
|
17401
|
+
return "Bearer [redacted]";
|
|
17402
|
+
return "[redacted]";
|
|
17403
|
+
});
|
|
17404
|
+
}
|
|
17405
|
+
function redactUrl(url) {
|
|
17406
|
+
const clone = new URL(url.toString());
|
|
17407
|
+
for (const key of Array.from(clone.searchParams.keys())) {
|
|
17408
|
+
if (SENSITIVE_PARAM_RE.test(key)) {
|
|
17409
|
+
clone.searchParams.set(key, "[redacted]");
|
|
17410
|
+
}
|
|
17411
|
+
}
|
|
17412
|
+
return clone.toString();
|
|
17413
|
+
}
|
|
17414
|
+
function redactUrlString(value) {
|
|
17415
|
+
const url = safeUrl(value);
|
|
17416
|
+
return url ? redactUrl(url) : redactProdDebugText(value);
|
|
17417
|
+
}
|
|
17418
|
+
function parseProdDebugTarget(target) {
|
|
17419
|
+
const input = target.trim();
|
|
17420
|
+
const url = safeUrl(input);
|
|
17421
|
+
if (!url) {
|
|
17422
|
+
const id = (input.match(UUID_RE)?.[0] ?? input) || null;
|
|
17423
|
+
return {
|
|
17424
|
+
url: null,
|
|
17425
|
+
origin: null,
|
|
17426
|
+
orgSlug: null,
|
|
17427
|
+
projectRef: null,
|
|
17428
|
+
sessionId: null,
|
|
17429
|
+
agentId: null,
|
|
17430
|
+
requestId: input.startsWith("req_") ? input : null,
|
|
17431
|
+
rawId: id
|
|
17432
|
+
};
|
|
17433
|
+
}
|
|
17434
|
+
const parts = url.pathname.split("/").filter(Boolean);
|
|
17435
|
+
const projectsIndex = parts.indexOf("projects");
|
|
17436
|
+
const sessionsIndex = parts.indexOf("sessions");
|
|
17437
|
+
const orgSlug = projectsIndex > 0 ? parts[0] ?? null : null;
|
|
17438
|
+
const projectRef = projectsIndex >= 0 ? parts[projectsIndex + 1] ?? null : null;
|
|
17439
|
+
const sessionId = url.searchParams.get("session") ?? (sessionsIndex >= 0 ? parts[sessionsIndex + 1] ?? null : null);
|
|
17440
|
+
return {
|
|
17441
|
+
url: redactUrl(url),
|
|
17442
|
+
origin: url.origin,
|
|
17443
|
+
orgSlug,
|
|
17444
|
+
projectRef,
|
|
17445
|
+
sessionId,
|
|
17446
|
+
agentId: url.searchParams.get("agent"),
|
|
17447
|
+
requestId: url.searchParams.get("requestId") ?? url.searchParams.get("request_id"),
|
|
17448
|
+
rawId: input.match(UUID_RE)?.[0] ?? null
|
|
17449
|
+
};
|
|
17450
|
+
}
|
|
17451
|
+
function boundedTtl(value) {
|
|
17452
|
+
if (!Number.isFinite(value))
|
|
17453
|
+
return 15;
|
|
17454
|
+
return Math.min(Math.max(Math.round(value ?? 15), 1), 60);
|
|
17455
|
+
}
|
|
17456
|
+
function makeCommand(command) {
|
|
17457
|
+
return command.replace(/\s+/g, " ").trim();
|
|
17458
|
+
}
|
|
17459
|
+
function hostnameFromOrigin(origin) {
|
|
17460
|
+
if (!origin)
|
|
17461
|
+
return null;
|
|
17462
|
+
return safeUrl(origin)?.hostname ?? null;
|
|
17463
|
+
}
|
|
17464
|
+
function originMatches(pattern, origin) {
|
|
17465
|
+
if (!origin)
|
|
17466
|
+
return false;
|
|
17467
|
+
const normalizedPattern = normalizeOrigin(pattern);
|
|
17468
|
+
const normalizedOrigin = normalizeOrigin(origin);
|
|
17469
|
+
if (!normalizedOrigin)
|
|
17470
|
+
return false;
|
|
17471
|
+
if (normalizedPattern === normalizedOrigin)
|
|
17472
|
+
return true;
|
|
17473
|
+
const targetHost = hostnameFromOrigin(normalizedOrigin);
|
|
17474
|
+
const patternHost = normalizedPattern ? hostnameFromOrigin(normalizedPattern) : pattern.replace(/^https?:\/\//, "");
|
|
17475
|
+
if (!targetHost || !patternHost)
|
|
17476
|
+
return false;
|
|
17477
|
+
if (patternHost.startsWith("*.")) {
|
|
17478
|
+
const suffix = patternHost.slice(1);
|
|
17479
|
+
return targetHost.endsWith(suffix);
|
|
17480
|
+
}
|
|
17481
|
+
return targetHost === patternHost;
|
|
17482
|
+
}
|
|
17483
|
+
function resolveProfile(input, target, config) {
|
|
17484
|
+
const apps = config?.apps ?? {};
|
|
17485
|
+
const explicitKey = input.profile?.trim() || input.app?.trim() || config?.defaultProfile;
|
|
17486
|
+
if (explicitKey && apps[explicitKey]) {
|
|
17487
|
+
return {
|
|
17488
|
+
key: explicitKey,
|
|
17489
|
+
profile: apps[explicitKey],
|
|
17490
|
+
matchedOrigin: target.origin
|
|
17491
|
+
};
|
|
17492
|
+
}
|
|
17493
|
+
for (const [key, profile] of Object.entries(apps)) {
|
|
17494
|
+
const match = profile.origins?.find((origin) => originMatches(origin, target.origin));
|
|
17495
|
+
if (match) {
|
|
17496
|
+
return { key, profile, matchedOrigin: match };
|
|
17497
|
+
}
|
|
17498
|
+
}
|
|
17499
|
+
return { key: null, profile: null, matchedOrigin: null };
|
|
17500
|
+
}
|
|
17501
|
+
function firstResolvedCredential(...values) {
|
|
17502
|
+
for (const value of values) {
|
|
17503
|
+
if (!value?.trim())
|
|
17504
|
+
continue;
|
|
17505
|
+
const resolved = resolveCredential(value);
|
|
17506
|
+
if (resolved)
|
|
17507
|
+
return resolved;
|
|
17508
|
+
}
|
|
17509
|
+
return null;
|
|
17510
|
+
}
|
|
17511
|
+
function displayCredential(value, source) {
|
|
17512
|
+
if (!value)
|
|
17513
|
+
return null;
|
|
17514
|
+
if (source && isCredentialReference(source))
|
|
17515
|
+
return "[configured]";
|
|
17516
|
+
return redactProdDebugText(value);
|
|
17517
|
+
}
|
|
17518
|
+
function replacementValues(target, input, supportGrant) {
|
|
17519
|
+
const values = {
|
|
17520
|
+
targetUrl: target.url ?? input.target,
|
|
17521
|
+
origin: target.origin ?? "",
|
|
17522
|
+
org: target.orgSlug ?? "",
|
|
17523
|
+
project: target.projectRef ?? "",
|
|
17524
|
+
session: target.sessionId ?? "",
|
|
17525
|
+
agent: target.agentId ?? "",
|
|
17526
|
+
request: target.requestId ?? "",
|
|
17527
|
+
rawId: target.rawId ?? "",
|
|
17528
|
+
reason: input.reason ?? "",
|
|
17529
|
+
supportGrant: supportGrant ?? ""
|
|
17530
|
+
};
|
|
17531
|
+
for (const [key, value] of Object.entries({ ...values })) {
|
|
17532
|
+
values[`${key}Encoded`] = encodeURIComponent(value);
|
|
17533
|
+
}
|
|
17534
|
+
return values;
|
|
17535
|
+
}
|
|
17536
|
+
function renderTemplate(template, values) {
|
|
17537
|
+
return template.replace(/\{([a-zA-Z0-9_]+)\}/g, (_match, key) => values[key] ?? "");
|
|
17538
|
+
}
|
|
17539
|
+
function resolveSupportGrant(input, profile) {
|
|
17540
|
+
if (input.supportGrantId?.trim()) {
|
|
17541
|
+
return {
|
|
17542
|
+
value: input.supportGrantId.trim(),
|
|
17543
|
+
display: displayCredential(input.supportGrantId.trim()),
|
|
17544
|
+
source: "input"
|
|
17545
|
+
};
|
|
17546
|
+
}
|
|
17547
|
+
const source = profile?.supportGrantRef ?? profile?.supportGrantId ?? null;
|
|
17548
|
+
const value = firstResolvedCredential(profile?.supportGrantRef, profile?.supportGrantId);
|
|
17549
|
+
return { value, display: displayCredential(value, source ?? undefined), source };
|
|
17550
|
+
}
|
|
17551
|
+
function resolveSupportUrl(input, target, profile, supportGrant) {
|
|
17552
|
+
if (input.supportUrl?.trim())
|
|
17553
|
+
return input.supportUrl.trim();
|
|
17554
|
+
const direct = firstResolvedCredential(profile?.supportUrlRef, profile?.supportUrl);
|
|
17555
|
+
if (direct)
|
|
17556
|
+
return direct;
|
|
17557
|
+
if (profile?.supportUrlTemplate) {
|
|
17558
|
+
const rendered = renderTemplate(profile.supportUrlTemplate, replacementValues(target, input, supportGrant)).trim();
|
|
17559
|
+
return rendered || null;
|
|
17560
|
+
}
|
|
17561
|
+
return null;
|
|
17562
|
+
}
|
|
17563
|
+
function resolvePiiOrigin(profile, target) {
|
|
17564
|
+
if (!profile?.piiOrigin)
|
|
17565
|
+
return target.origin;
|
|
17566
|
+
return redactUrlString(renderTemplate(profile.piiOrigin, replacementValues(target, { target: target.url ?? "" }, null)));
|
|
17567
|
+
}
|
|
17568
|
+
function resolveSupportRunTarget(supportUrl, input, target) {
|
|
17569
|
+
if (supportUrl)
|
|
17570
|
+
return redactUrlString(supportUrl);
|
|
17571
|
+
return target.url ?? target.origin ?? redactProdDebugText(input.target);
|
|
17572
|
+
}
|
|
17573
|
+
function supportScenarioDescription(reason) {
|
|
17574
|
+
return `Prod debug: ${reason}. Reproduce the user-visible issue, capture console and network errors, and do not enter secrets.`;
|
|
17575
|
+
}
|
|
17576
|
+
function configuredMissing(profile, supportUrl, supportGrant, includeLogs) {
|
|
17577
|
+
const missing = [];
|
|
17578
|
+
if (!profile) {
|
|
17579
|
+
missing.push("optional: add prodDebug.apps.<profile>.origins to match this app automatically");
|
|
17580
|
+
}
|
|
17581
|
+
if (!supportUrl) {
|
|
17582
|
+
missing.push("supportUrl/supportUrlRef/supportUrlTemplate for scoped browser debugging");
|
|
17583
|
+
}
|
|
17584
|
+
if (!supportGrant) {
|
|
17585
|
+
missing.push("supportGrantId/supportGrantRef for auditable support access");
|
|
17586
|
+
}
|
|
17587
|
+
if (includeLogs && !profile?.logCommand) {
|
|
17588
|
+
missing.push("logCommand for sanitized app/provider log lookup");
|
|
17589
|
+
}
|
|
17590
|
+
return missing;
|
|
17591
|
+
}
|
|
17592
|
+
function createProdDebugPlan(input, config) {
|
|
17593
|
+
const target = parseProdDebugTarget(input.target);
|
|
17594
|
+
const browserRequested = input.includeBrowser !== false;
|
|
17595
|
+
const resolvedProfile = resolveProfile(input, target, config);
|
|
17596
|
+
const supportGrant = resolveSupportGrant(input, resolvedProfile.profile);
|
|
17597
|
+
const supportUrl = resolveSupportUrl(input, target, resolvedProfile.profile, supportGrant.value);
|
|
17598
|
+
const supportBrowserReady = Boolean(supportUrl);
|
|
17599
|
+
const app = input.app?.trim() || resolvedProfile.profile?.name || resolvedProfile.key || (target.origin ? new URL(target.origin).hostname : "app");
|
|
17600
|
+
const reason = input.reason?.trim() || "production debug requested";
|
|
17601
|
+
const actor = input.actor?.trim() || process.env["USER"] || "agent";
|
|
17602
|
+
const ttlMinutes = boundedTtl(input.ttlMinutes);
|
|
17603
|
+
const piiOrigin = resolvePiiOrigin(resolvedProfile.profile, target);
|
|
17604
|
+
const logCommand = resolvedProfile.profile?.logCommand ? redactUrlString(renderTemplate(resolvedProfile.profile.logCommand, replacementValues(target, { ...input, reason }, supportGrant.value))) : null;
|
|
17605
|
+
const safety = [
|
|
17606
|
+
"read-only by default",
|
|
17607
|
+
"no customer passwords or raw cookies",
|
|
17608
|
+
"redact tokens, OAuth codes, session values, support grants, and secrets",
|
|
17609
|
+
"verify org/user/session scope before reading data",
|
|
17610
|
+
"require explicit approval for production writes",
|
|
17611
|
+
`support access TTL capped at ${ttlMinutes} minutes`
|
|
17612
|
+
];
|
|
17613
|
+
const checks = [];
|
|
17614
|
+
const blocked = [];
|
|
17615
|
+
if (target.url) {
|
|
17616
|
+
checks.push({
|
|
17617
|
+
id: "public-route-smoke",
|
|
17618
|
+
status: "ready",
|
|
17619
|
+
description: "Open the supplied production URL and capture console/network errors without credentials.",
|
|
17620
|
+
command: makeCommand(`testers scan all ${JSON.stringify(target.url)} --json`)
|
|
17621
|
+
});
|
|
17622
|
+
}
|
|
17623
|
+
checks.push({
|
|
17624
|
+
id: "pii-redaction-scan",
|
|
17625
|
+
status: piiOrigin ? "ready" : "blocked",
|
|
17626
|
+
description: "Scan public/API responses for accidental sensitive data leakage.",
|
|
17627
|
+
command: piiOrigin ? makeCommand(`testers scan pii ${JSON.stringify(piiOrigin)} --json`) : undefined,
|
|
17628
|
+
reason: piiOrigin ? undefined : "Need a URL origin or prodDebug app profile piiOrigin to run the PII scan."
|
|
17629
|
+
});
|
|
17630
|
+
if (browserRequested) {
|
|
17631
|
+
if (supportBrowserReady) {
|
|
17632
|
+
checks.push({
|
|
17633
|
+
id: "support-browser-repro",
|
|
17634
|
+
status: "ready",
|
|
17635
|
+
description: "Use an audited support browser/session URL to reproduce the user-visible issue.",
|
|
17636
|
+
command: makeCommand(`testers run ${JSON.stringify(resolveSupportRunTarget(supportUrl, input, target))} ${JSON.stringify(supportScenarioDescription(reason))} --headed --json --overall-timeout 600000`)
|
|
17637
|
+
});
|
|
17638
|
+
} else {
|
|
17639
|
+
const reasonText = supportGrant.value ? "An audited support grant was supplied, but open-testers still needs supportUrl/supportUrlRef/supportUrlTemplate or an app adapter to open a scoped browser session." : "No audited support browser/session grant was supplied. Do not use customer passwords, copied cookies, bearer tokens, or magic links.";
|
|
17640
|
+
blocked.push(reasonText);
|
|
17641
|
+
checks.push({
|
|
17642
|
+
id: "support-browser-repro",
|
|
17643
|
+
status: "blocked",
|
|
17644
|
+
description: "Browser reproduction as the target user requires a short-lived audited support session.",
|
|
17645
|
+
reason: reasonText
|
|
17646
|
+
});
|
|
17647
|
+
}
|
|
17648
|
+
}
|
|
17649
|
+
if (input.includeLogs) {
|
|
17650
|
+
if (logCommand) {
|
|
17651
|
+
checks.push({
|
|
17652
|
+
id: "log-timeline",
|
|
17653
|
+
status: "ready",
|
|
17654
|
+
description: "Read sanitized app/provider logs by request ID, session ID, project ID, or support access ID.",
|
|
17655
|
+
command: makeCommand(logCommand)
|
|
17656
|
+
});
|
|
17657
|
+
} else {
|
|
17658
|
+
checks.push({
|
|
17659
|
+
id: "log-timeline",
|
|
17660
|
+
status: "blocked",
|
|
17661
|
+
description: "Read sanitized app/provider logs by request ID, session ID, project ID, or support access ID.",
|
|
17662
|
+
reason: "Configure prodDebug.apps.<profile>.logCommand or use an app-specific log MCP. Do not paste raw provider logs with headers/secrets."
|
|
17663
|
+
});
|
|
17664
|
+
}
|
|
17665
|
+
}
|
|
17666
|
+
if (input.allowWrites) {
|
|
17667
|
+
blocked.push("Production writes are not part of prod-debug. Require a separate explicit approval and app-specific write tool.");
|
|
17668
|
+
}
|
|
17669
|
+
return {
|
|
17670
|
+
target,
|
|
17671
|
+
app,
|
|
17672
|
+
actor,
|
|
17673
|
+
reason,
|
|
17674
|
+
ttlMinutes,
|
|
17675
|
+
setup: {
|
|
17676
|
+
profile: resolvedProfile.key,
|
|
17677
|
+
matchedOrigin: resolvedProfile.matchedOrigin,
|
|
17678
|
+
configured: {
|
|
17679
|
+
supportUrl: Boolean(supportUrl),
|
|
17680
|
+
supportGrant: Boolean(supportGrant.value),
|
|
17681
|
+
piiOrigin: Boolean(piiOrigin),
|
|
17682
|
+
logCommand: Boolean(logCommand)
|
|
17683
|
+
},
|
|
17684
|
+
missing: configuredMissing(resolvedProfile.profile, supportUrl, supportGrant.value, Boolean(input.includeLogs))
|
|
17685
|
+
},
|
|
17686
|
+
supportAccess: {
|
|
17687
|
+
required: browserRequested,
|
|
17688
|
+
grantId: supportGrant.display,
|
|
17689
|
+
browserReady: supportBrowserReady,
|
|
17690
|
+
note: supportBrowserReady ? "Use the provided audited support access; never print token/cookie values." : "Configure an audited support-browser/session URL, URL ref, or template before user-scoped browser debugging."
|
|
17691
|
+
},
|
|
17692
|
+
safety,
|
|
17693
|
+
checks,
|
|
17694
|
+
blocked
|
|
17695
|
+
};
|
|
17696
|
+
}
|
|
17697
|
+
function formatProdDebugPlan(plan) {
|
|
17698
|
+
const lines = [];
|
|
17699
|
+
lines.push(`Prod debug plan for ${plan.app}`);
|
|
17700
|
+
lines.push("");
|
|
17701
|
+
lines.push("Target");
|
|
17702
|
+
lines.push(`- url: ${plan.target.url ?? "(none)"}`);
|
|
17703
|
+
lines.push(`- org: ${plan.target.orgSlug ?? "(unknown)"}`);
|
|
17704
|
+
lines.push(`- project: ${plan.target.projectRef ?? "(unknown)"}`);
|
|
17705
|
+
lines.push(`- session: ${plan.target.sessionId ?? "(unknown)"}`);
|
|
17706
|
+
lines.push(`- agent: ${plan.target.agentId ?? "(unknown)"}`);
|
|
17707
|
+
lines.push(`- request: ${plan.target.requestId ?? "(unknown)"}`);
|
|
17708
|
+
lines.push("");
|
|
17709
|
+
lines.push("Setup");
|
|
17710
|
+
lines.push(`- profile: ${plan.setup.profile ?? "(none)"}`);
|
|
17711
|
+
lines.push(`- matched origin: ${plan.setup.matchedOrigin ?? "(none)"}`);
|
|
17712
|
+
if (plan.setup.missing.length > 0) {
|
|
17713
|
+
for (const item of plan.setup.missing)
|
|
17714
|
+
lines.push(`- missing: ${item}`);
|
|
17715
|
+
}
|
|
17716
|
+
lines.push("");
|
|
17717
|
+
lines.push("Support access");
|
|
17718
|
+
lines.push(`- actor: ${plan.actor}`);
|
|
17719
|
+
lines.push(`- reason: ${plan.reason}`);
|
|
17720
|
+
lines.push(`- ttl: ${plan.ttlMinutes} minutes`);
|
|
17721
|
+
lines.push(`- grant: ${plan.supportAccess.grantId ?? "(none)"}`);
|
|
17722
|
+
lines.push(`- browser ready: ${plan.supportAccess.browserReady ? "yes" : "no"}`);
|
|
17723
|
+
lines.push(`- note: ${plan.supportAccess.note}`);
|
|
17724
|
+
lines.push("");
|
|
17725
|
+
lines.push("Checks");
|
|
17726
|
+
for (const check of plan.checks) {
|
|
17727
|
+
lines.push(`- ${check.id}: ${check.status} - ${check.description}`);
|
|
17728
|
+
if (check.command)
|
|
17729
|
+
lines.push(` command: ${check.command}`);
|
|
17730
|
+
if (check.reason)
|
|
17731
|
+
lines.push(` blocked: ${check.reason}`);
|
|
17732
|
+
}
|
|
17733
|
+
if (plan.blocked.length > 0) {
|
|
17734
|
+
lines.push("");
|
|
17735
|
+
lines.push("Blocked");
|
|
17736
|
+
for (const item of plan.blocked)
|
|
17737
|
+
lines.push(`- ${item}`);
|
|
17738
|
+
}
|
|
17739
|
+
lines.push("");
|
|
17740
|
+
lines.push("Safety");
|
|
17741
|
+
for (const item of plan.safety)
|
|
17742
|
+
lines.push(`- ${item}`);
|
|
17743
|
+
return lines.join(`
|
|
17744
|
+
`);
|
|
17745
|
+
}
|
|
17746
|
+
// src/lib/ci.ts
|
|
17747
|
+
function generateGitHubActionsWorkflow() {
|
|
17748
|
+
return `name: AI QA Tests
|
|
17749
|
+
on:
|
|
17750
|
+
pull_request:
|
|
17751
|
+
push:
|
|
17752
|
+
branches: [main]
|
|
17753
|
+
|
|
17754
|
+
permissions:
|
|
17755
|
+
contents: read
|
|
17756
|
+
pull-requests: write
|
|
17757
|
+
|
|
17758
|
+
jobs:
|
|
17759
|
+
test:
|
|
17760
|
+
runs-on: ubuntu-latest
|
|
17761
|
+
steps:
|
|
17762
|
+
- uses: actions/checkout@v4
|
|
17763
|
+
- uses: oven-sh/setup-bun@v2
|
|
17764
|
+
- run: bun install -g @hasna/testers
|
|
17765
|
+
- run: testers install-browser
|
|
17766
|
+
- run: testers run \${{ env.TEST_URL }} --json --output results.json --github-comment
|
|
17767
|
+
env:
|
|
17768
|
+
ANTHROPIC_API_KEY: \${{ secrets.ANTHROPIC_API_KEY }}
|
|
17769
|
+
TEST_URL: \${{ vars.TEST_URL || 'http://localhost:3000' }}
|
|
17770
|
+
GITHUB_TOKEN: \${{ secrets.GITHUB_TOKEN }}
|
|
17771
|
+
- run: testers report --latest --output report.html
|
|
17772
|
+
if: always()
|
|
17773
|
+
- uses: actions/upload-artifact@v4
|
|
17774
|
+
if: always()
|
|
17775
|
+
with:
|
|
17776
|
+
name: test-report
|
|
17777
|
+
path: |
|
|
17778
|
+
report.html
|
|
17779
|
+
results.json
|
|
17780
|
+
`;
|
|
17781
|
+
}
|
|
17782
|
+
function formatPRComment(run, results, dashboardUrl) {
|
|
17783
|
+
const icon = run.status === "passed" ? "\u2705" : run.status === "failed" ? "\u274C" : "\u26A0\uFE0F";
|
|
17784
|
+
const passRate = run.total > 0 ? Math.round(run.passed / run.total * 100) : 0;
|
|
17785
|
+
const ordered = [...results].sort((a, b) => {
|
|
17786
|
+
const rank = (s) => s === "failed" ? 0 : s === "error" ? 1 : s === "flaky" ? 2 : s === "skipped" ? 3 : 4;
|
|
17787
|
+
return rank(a.status) - rank(b.status);
|
|
17788
|
+
});
|
|
17789
|
+
const MAX_ROWS = 20;
|
|
17790
|
+
const rows = ordered.slice(0, MAX_ROWS).map((r) => {
|
|
17791
|
+
const rowIcon = r.status === "passed" ? "\u2705" : r.status === "failed" ? "\u274C" : r.status === "error" ? "\u26A0\uFE0F" : r.status === "flaky" ? "\uD83D\uDFE1" : "\u23ED\uFE0F";
|
|
17792
|
+
const dur = r.durationMs > 0 ? `${(r.durationMs / 1000).toFixed(1)}s` : "\u2014";
|
|
17793
|
+
const scenario = (() => {
|
|
17794
|
+
try {
|
|
17795
|
+
return getScenario(r.scenarioId);
|
|
17796
|
+
} catch {
|
|
17797
|
+
return null;
|
|
17798
|
+
}
|
|
17799
|
+
})();
|
|
17800
|
+
const name = scenario ? scenario.name : r.scenarioId.slice(0, 8);
|
|
17801
|
+
const safeName = name.replace(/\|/g, "\\|");
|
|
17802
|
+
const errSource = r.error ?? r.reasoning ?? "";
|
|
17803
|
+
const err = errSource ? ` ${errSource.replace(/\s+/g, " ").slice(0, 140).replace(/\|/g, "\\|")}` : "";
|
|
17804
|
+
return `| ${rowIcon} | ${safeName} | ${r.status} | ${dur} |${err} |`;
|
|
17805
|
+
}).join(`
|
|
17806
|
+
`);
|
|
17807
|
+
const truncated = results.length > MAX_ROWS ? `
|
|
17808
|
+
_...and ${results.length - MAX_ROWS} more_` : "";
|
|
17809
|
+
const dashLink = dashboardUrl ? `
|
|
17810
|
+
|
|
17811
|
+
[View full report \u2192](${dashboardUrl}/runs/${run.id})` : "";
|
|
17812
|
+
const totalCostCents = results.reduce((sum, r) => sum + (r.costCents ?? 0), 0);
|
|
17813
|
+
const costStr = totalCostCents > 0 ? ` \xB7 $${(totalCostCents / 100).toFixed(4)}` : "";
|
|
17814
|
+
const headerRow = `| | Scenario | Status | Duration | Details |
|
|
17815
|
+
|---|---|---|---|---|`;
|
|
17816
|
+
const body = results.length > 0 ? `${headerRow}
|
|
17817
|
+
${rows}${truncated}` : `_No scenarios ran. Use \`testers add\` to create scenarios or run with a URL to auto-generate them._`;
|
|
17818
|
+
return `## ${icon} AI QA Tests \u2014 ${run.status.toUpperCase()}
|
|
17819
|
+
|
|
17820
|
+
**${run.passed}/${run.total} passed** (${passRate}%) \xB7 URL: \`${run.url}\`${costStr}
|
|
17821
|
+
|
|
17822
|
+
${body}${dashLink}
|
|
17823
|
+
|
|
17824
|
+
_Generated by [@hasna/testers](https://www.npmjs.com/package/@hasna/testers)_`;
|
|
17825
|
+
}
|
|
17826
|
+
function resolvePullRequestNumber(explicit) {
|
|
17827
|
+
if (explicit && Number.isFinite(explicit))
|
|
17828
|
+
return explicit;
|
|
17829
|
+
const fromEnv = process.env["GITHUB_PR_NUMBER"];
|
|
17830
|
+
if (fromEnv) {
|
|
17831
|
+
const parsed = parseInt(fromEnv, 10);
|
|
17832
|
+
if (Number.isFinite(parsed))
|
|
17833
|
+
return parsed;
|
|
17834
|
+
}
|
|
17835
|
+
const ref = process.env["GITHUB_REF"] ?? "";
|
|
17836
|
+
const match = ref.match(/refs\/pull\/(\d+)\//);
|
|
17837
|
+
if (match && match[1]) {
|
|
17838
|
+
const parsed = parseInt(match[1], 10);
|
|
17839
|
+
if (Number.isFinite(parsed))
|
|
17840
|
+
return parsed;
|
|
17841
|
+
}
|
|
17842
|
+
return null;
|
|
17843
|
+
}
|
|
17844
|
+
async function postGitHubComment(run, results, options) {
|
|
17845
|
+
const token = process.env["GITHUB_TOKEN"];
|
|
17846
|
+
if (!token)
|
|
17847
|
+
return false;
|
|
17848
|
+
const prNumber = resolvePullRequestNumber(options?.prNumber);
|
|
17849
|
+
if (prNumber === null)
|
|
17850
|
+
return false;
|
|
17851
|
+
const repo = process.env["GITHUB_REPOSITORY"];
|
|
17852
|
+
if (!repo)
|
|
17853
|
+
return false;
|
|
17854
|
+
const body = formatPRComment(run, results, options?.dashboardUrl ?? process.env["TESTERS_DASHBOARD_URL"]);
|
|
17855
|
+
const apiUrl = `https://api.github.com/repos/${repo}/issues/${prNumber}/comments`;
|
|
17856
|
+
try {
|
|
17857
|
+
const response = await fetch(apiUrl, {
|
|
17858
|
+
method: "POST",
|
|
17859
|
+
headers: {
|
|
17860
|
+
Authorization: `Bearer ${token}`,
|
|
17861
|
+
"Content-Type": "application/json",
|
|
17862
|
+
Accept: "application/vnd.github+json",
|
|
17863
|
+
"X-GitHub-Api-Version": "2022-11-28"
|
|
17864
|
+
},
|
|
17865
|
+
body: JSON.stringify({ body })
|
|
17866
|
+
});
|
|
17867
|
+
return response.ok;
|
|
17868
|
+
} catch {
|
|
17869
|
+
return false;
|
|
17870
|
+
}
|
|
17871
|
+
}
|
|
17087
17872
|
export {
|
|
17088
17873
|
writeScenarioMeta,
|
|
17089
17874
|
writeRunMeta,
|
|
@@ -17111,6 +17896,7 @@ export {
|
|
|
17111
17896
|
runBatch,
|
|
17112
17897
|
runAgentLoop,
|
|
17113
17898
|
resultFromRow,
|
|
17899
|
+
resolvePullRequestNumber,
|
|
17114
17900
|
resolvePartialId,
|
|
17115
17901
|
resolveModel as resolveModelConfig,
|
|
17116
17902
|
resolveModel2 as resolveModel,
|
|
@@ -17118,9 +17904,12 @@ export {
|
|
|
17118
17904
|
resetDatabase,
|
|
17119
17905
|
removeDependency,
|
|
17120
17906
|
registerAgent,
|
|
17907
|
+
redactProdDebugText,
|
|
17121
17908
|
pullTasks,
|
|
17122
17909
|
projectFromRow,
|
|
17910
|
+
postGitHubComment,
|
|
17123
17911
|
parseSmokeIssues,
|
|
17912
|
+
parseProdDebugTarget,
|
|
17124
17913
|
parseCronField,
|
|
17125
17914
|
parseCron,
|
|
17126
17915
|
onRunEvent,
|
|
@@ -17179,6 +17968,7 @@ export {
|
|
|
17179
17968
|
getAgent,
|
|
17180
17969
|
generateLatestReport,
|
|
17181
17970
|
generateHtmlReport,
|
|
17971
|
+
generateGitHubActionsWorkflow,
|
|
17182
17972
|
generateFilename,
|
|
17183
17973
|
formatTerminal,
|
|
17184
17974
|
formatSummary,
|
|
@@ -17186,6 +17976,8 @@ export {
|
|
|
17186
17976
|
formatScenarioList,
|
|
17187
17977
|
formatRunList,
|
|
17188
17978
|
formatResultDetail,
|
|
17979
|
+
formatProdDebugPlan,
|
|
17980
|
+
formatPRComment,
|
|
17189
17981
|
formatJSON,
|
|
17190
17982
|
formatDiffTerminal,
|
|
17191
17983
|
formatDiffJSON,
|
|
@@ -17212,6 +18004,7 @@ export {
|
|
|
17212
18004
|
createRun,
|
|
17213
18005
|
createResult,
|
|
17214
18006
|
createProject,
|
|
18007
|
+
createProdDebugPlan,
|
|
17215
18008
|
createFlow,
|
|
17216
18009
|
createClient,
|
|
17217
18010
|
createAuthPreset,
|