@hasna/testers 0.0.1 → 0.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +72 -1
- package/dist/cli/index.js +2444 -373
- package/dist/db/auth-presets.d.ts +20 -0
- package/dist/db/auth-presets.d.ts.map +1 -0
- package/dist/db/database.d.ts.map +1 -1
- package/dist/db/schedules.d.ts +9 -0
- package/dist/db/schedules.d.ts.map +1 -0
- package/dist/db/screenshots.d.ts +3 -0
- package/dist/db/screenshots.d.ts.map +1 -1
- package/dist/index.d.ts +21 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2426 -399
- package/dist/lib/ai-client.d.ts +6 -0
- package/dist/lib/ai-client.d.ts.map +1 -1
- package/dist/lib/costs.d.ts +36 -0
- package/dist/lib/costs.d.ts.map +1 -0
- package/dist/lib/diff.d.ts +25 -0
- package/dist/lib/diff.d.ts.map +1 -0
- package/dist/lib/init.d.ts +28 -0
- package/dist/lib/init.d.ts.map +1 -0
- package/dist/lib/report.d.ts +4 -0
- package/dist/lib/report.d.ts.map +1 -0
- package/dist/lib/runner.d.ts.map +1 -1
- package/dist/lib/scheduler.d.ts +71 -0
- package/dist/lib/scheduler.d.ts.map +1 -0
- package/dist/lib/screenshotter.d.ts +27 -25
- package/dist/lib/screenshotter.d.ts.map +1 -1
- package/dist/lib/smoke.d.ts +25 -0
- package/dist/lib/smoke.d.ts.map +1 -0
- package/dist/lib/templates.d.ts +5 -0
- package/dist/lib/templates.d.ts.map +1 -0
- package/dist/lib/watch.d.ts +9 -0
- package/dist/lib/watch.d.ts.map +1 -0
- package/dist/lib/webhooks.d.ts +41 -0
- package/dist/lib/webhooks.d.ts.map +1 -0
- package/dist/mcp/index.js +839 -25
- package/dist/server/index.js +818 -25
- package/dist/types/index.d.ts +86 -0
- package/dist/types/index.d.ts.map +1 -1
- package/package.json +1 -1
package/dist/cli/index.js
CHANGED
|
@@ -17,6 +17,16 @@ var __toESM = (mod, isNodeMode, target) => {
|
|
|
17
17
|
return to;
|
|
18
18
|
};
|
|
19
19
|
var __commonJS = (cb, mod) => () => (mod || cb((mod = { exports: {} }).exports, mod), mod.exports);
|
|
20
|
+
var __export = (target, all) => {
|
|
21
|
+
for (var name in all)
|
|
22
|
+
__defProp(target, name, {
|
|
23
|
+
get: all[name],
|
|
24
|
+
enumerable: true,
|
|
25
|
+
configurable: true,
|
|
26
|
+
set: (newValue) => all[name] = () => newValue
|
|
27
|
+
});
|
|
28
|
+
};
|
|
29
|
+
var __esm = (fn, res) => () => (fn && (res = fn(fn = 0)), res);
|
|
20
30
|
var __require = import.meta.require;
|
|
21
31
|
|
|
22
32
|
// node_modules/commander/lib/error.js
|
|
@@ -2053,33 +2063,17 @@ var require_commander = __commonJS((exports) => {
|
|
|
2053
2063
|
exports.InvalidOptionArgumentError = InvalidArgumentError;
|
|
2054
2064
|
});
|
|
2055
2065
|
|
|
2056
|
-
// node_modules/commander/esm.mjs
|
|
2057
|
-
var import__ = __toESM(require_commander(), 1);
|
|
2058
|
-
var {
|
|
2059
|
-
program,
|
|
2060
|
-
createCommand,
|
|
2061
|
-
createArgument,
|
|
2062
|
-
createOption,
|
|
2063
|
-
CommanderError,
|
|
2064
|
-
InvalidArgumentError,
|
|
2065
|
-
InvalidOptionArgumentError,
|
|
2066
|
-
Command,
|
|
2067
|
-
Argument,
|
|
2068
|
-
Option,
|
|
2069
|
-
Help
|
|
2070
|
-
} = import__.default;
|
|
2071
|
-
|
|
2072
|
-
// src/cli/index.tsx
|
|
2073
|
-
import chalk2 from "chalk";
|
|
2074
|
-
import { readFileSync as readFileSync2, readdirSync, writeFileSync } from "fs";
|
|
2075
|
-
import { join as join5, resolve } from "path";
|
|
2076
|
-
|
|
2077
2066
|
// src/types/index.ts
|
|
2078
|
-
|
|
2079
|
-
|
|
2080
|
-
|
|
2081
|
-
|
|
2082
|
-
|
|
2067
|
+
function projectFromRow(row) {
|
|
2068
|
+
return {
|
|
2069
|
+
id: row.id,
|
|
2070
|
+
name: row.name,
|
|
2071
|
+
path: row.path,
|
|
2072
|
+
description: row.description,
|
|
2073
|
+
createdAt: row.created_at,
|
|
2074
|
+
updatedAt: row.updated_at
|
|
2075
|
+
};
|
|
2076
|
+
}
|
|
2083
2077
|
function scenarioFromRow(row) {
|
|
2084
2078
|
return {
|
|
2085
2079
|
id: row.id,
|
|
@@ -2145,43 +2139,76 @@ function screenshotFromRow(row) {
|
|
|
2145
2139
|
filePath: row.file_path,
|
|
2146
2140
|
width: row.width,
|
|
2147
2141
|
height: row.height,
|
|
2148
|
-
timestamp: row.timestamp
|
|
2142
|
+
timestamp: row.timestamp,
|
|
2143
|
+
description: row.description,
|
|
2144
|
+
pageUrl: row.page_url,
|
|
2145
|
+
thumbnailPath: row.thumbnail_path
|
|
2149
2146
|
};
|
|
2150
2147
|
}
|
|
2151
|
-
|
|
2152
|
-
|
|
2153
|
-
|
|
2154
|
-
|
|
2155
|
-
|
|
2156
|
-
|
|
2157
|
-
|
|
2158
|
-
|
|
2159
|
-
|
|
2160
|
-
|
|
2161
|
-
|
|
2162
|
-
|
|
2163
|
-
|
|
2164
|
-
|
|
2165
|
-
|
|
2166
|
-
|
|
2167
|
-
|
|
2168
|
-
|
|
2169
|
-
}
|
|
2170
|
-
}
|
|
2171
|
-
|
|
2172
|
-
class TodosConnectionError extends Error {
|
|
2173
|
-
constructor(message) {
|
|
2174
|
-
super(message);
|
|
2175
|
-
this.name = "TodosConnectionError";
|
|
2176
|
-
}
|
|
2148
|
+
function scheduleFromRow(row) {
|
|
2149
|
+
return {
|
|
2150
|
+
id: row.id,
|
|
2151
|
+
projectId: row.project_id,
|
|
2152
|
+
name: row.name,
|
|
2153
|
+
cronExpression: row.cron_expression,
|
|
2154
|
+
url: row.url,
|
|
2155
|
+
scenarioFilter: JSON.parse(row.scenario_filter),
|
|
2156
|
+
model: row.model,
|
|
2157
|
+
headed: row.headed === 1,
|
|
2158
|
+
parallel: row.parallel,
|
|
2159
|
+
timeoutMs: row.timeout_ms,
|
|
2160
|
+
enabled: row.enabled === 1,
|
|
2161
|
+
lastRunId: row.last_run_id,
|
|
2162
|
+
lastRunAt: row.last_run_at,
|
|
2163
|
+
nextRunAt: row.next_run_at,
|
|
2164
|
+
createdAt: row.created_at,
|
|
2165
|
+
updatedAt: row.updated_at
|
|
2166
|
+
};
|
|
2177
2167
|
}
|
|
2168
|
+
var MODEL_MAP, VersionConflictError, BrowserError, AIClientError, TodosConnectionError, ScheduleNotFoundError;
|
|
2169
|
+
var init_types = __esm(() => {
|
|
2170
|
+
MODEL_MAP = {
|
|
2171
|
+
quick: "claude-haiku-4-5-20251001",
|
|
2172
|
+
thorough: "claude-sonnet-4-6-20260311",
|
|
2173
|
+
deep: "claude-opus-4-6-20260311"
|
|
2174
|
+
};
|
|
2175
|
+
VersionConflictError = class VersionConflictError extends Error {
|
|
2176
|
+
constructor(entity, id) {
|
|
2177
|
+
super(`Version conflict on ${entity}: ${id}`);
|
|
2178
|
+
this.name = "VersionConflictError";
|
|
2179
|
+
}
|
|
2180
|
+
};
|
|
2181
|
+
BrowserError = class BrowserError extends Error {
|
|
2182
|
+
constructor(message) {
|
|
2183
|
+
super(message);
|
|
2184
|
+
this.name = "BrowserError";
|
|
2185
|
+
}
|
|
2186
|
+
};
|
|
2187
|
+
AIClientError = class AIClientError extends Error {
|
|
2188
|
+
constructor(message) {
|
|
2189
|
+
super(message);
|
|
2190
|
+
this.name = "AIClientError";
|
|
2191
|
+
}
|
|
2192
|
+
};
|
|
2193
|
+
TodosConnectionError = class TodosConnectionError extends Error {
|
|
2194
|
+
constructor(message) {
|
|
2195
|
+
super(message);
|
|
2196
|
+
this.name = "TodosConnectionError";
|
|
2197
|
+
}
|
|
2198
|
+
};
|
|
2199
|
+
ScheduleNotFoundError = class ScheduleNotFoundError extends Error {
|
|
2200
|
+
constructor(id) {
|
|
2201
|
+
super(`Schedule not found: ${id}`);
|
|
2202
|
+
this.name = "ScheduleNotFoundError";
|
|
2203
|
+
}
|
|
2204
|
+
};
|
|
2205
|
+
});
|
|
2178
2206
|
|
|
2179
2207
|
// src/db/database.ts
|
|
2180
2208
|
import { Database } from "bun:sqlite";
|
|
2181
2209
|
import { mkdirSync, existsSync } from "fs";
|
|
2182
2210
|
import { dirname, join } from "path";
|
|
2183
2211
|
import { homedir } from "os";
|
|
2184
|
-
var db = null;
|
|
2185
2212
|
function now() {
|
|
2186
2213
|
return new Date().toISOString();
|
|
2187
2214
|
}
|
|
@@ -2200,8 +2227,50 @@ function resolveDbPath() {
|
|
|
2200
2227
|
mkdirSync(dir, { recursive: true });
|
|
2201
2228
|
return join(dir, "testers.db");
|
|
2202
2229
|
}
|
|
2203
|
-
|
|
2204
|
-
|
|
2230
|
+
function applyMigrations(database) {
|
|
2231
|
+
const applied = database.query("SELECT id FROM _migrations ORDER BY id").all();
|
|
2232
|
+
const appliedIds = new Set(applied.map((r) => r.id));
|
|
2233
|
+
for (let i = 0;i < MIGRATIONS.length; i++) {
|
|
2234
|
+
const migrationId = i + 1;
|
|
2235
|
+
if (appliedIds.has(migrationId))
|
|
2236
|
+
continue;
|
|
2237
|
+
const migration = MIGRATIONS[i];
|
|
2238
|
+
database.exec(migration);
|
|
2239
|
+
database.query("INSERT INTO _migrations (id, applied_at) VALUES (?, ?)").run(migrationId, now());
|
|
2240
|
+
}
|
|
2241
|
+
}
|
|
2242
|
+
function getDatabase() {
|
|
2243
|
+
if (db)
|
|
2244
|
+
return db;
|
|
2245
|
+
const dbPath = resolveDbPath();
|
|
2246
|
+
const dir = dirname(dbPath);
|
|
2247
|
+
if (dbPath !== ":memory:" && !existsSync(dir)) {
|
|
2248
|
+
mkdirSync(dir, { recursive: true });
|
|
2249
|
+
}
|
|
2250
|
+
db = new Database(dbPath);
|
|
2251
|
+
db.exec("PRAGMA journal_mode = WAL");
|
|
2252
|
+
db.exec("PRAGMA foreign_keys = ON");
|
|
2253
|
+
db.exec("PRAGMA busy_timeout = 5000");
|
|
2254
|
+
db.exec(`
|
|
2255
|
+
CREATE TABLE IF NOT EXISTS _migrations (
|
|
2256
|
+
id INTEGER PRIMARY KEY,
|
|
2257
|
+
applied_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
2258
|
+
);
|
|
2259
|
+
`);
|
|
2260
|
+
applyMigrations(db);
|
|
2261
|
+
return db;
|
|
2262
|
+
}
|
|
2263
|
+
function resolvePartialId(table, partialId) {
|
|
2264
|
+
const database = getDatabase();
|
|
2265
|
+
const rows = database.query(`SELECT id FROM ${table} WHERE id LIKE ? || '%'`).all(partialId);
|
|
2266
|
+
if (rows.length === 1)
|
|
2267
|
+
return rows[0].id;
|
|
2268
|
+
return null;
|
|
2269
|
+
}
|
|
2270
|
+
var db = null, MIGRATIONS;
|
|
2271
|
+
var init_database = __esm(() => {
|
|
2272
|
+
MIGRATIONS = [
|
|
2273
|
+
`
|
|
2205
2274
|
CREATE TABLE IF NOT EXISTS projects (
|
|
2206
2275
|
id TEXT PRIMARY KEY,
|
|
2207
2276
|
name TEXT NOT NULL UNIQUE,
|
|
@@ -2290,7 +2359,7 @@ var MIGRATIONS = [
|
|
|
2290
2359
|
applied_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
2291
2360
|
);
|
|
2292
2361
|
`,
|
|
2293
|
-
|
|
2362
|
+
`
|
|
2294
2363
|
CREATE INDEX IF NOT EXISTS idx_scenarios_project ON scenarios(project_id);
|
|
2295
2364
|
CREATE INDEX IF NOT EXISTS idx_scenarios_priority ON scenarios(priority);
|
|
2296
2365
|
CREATE INDEX IF NOT EXISTS idx_scenarios_short_id ON scenarios(short_id);
|
|
@@ -2301,98 +2370,98 @@ var MIGRATIONS = [
|
|
|
2301
2370
|
CREATE INDEX IF NOT EXISTS idx_results_status ON results(status);
|
|
2302
2371
|
CREATE INDEX IF NOT EXISTS idx_screenshots_result ON screenshots(result_id);
|
|
2303
2372
|
`,
|
|
2304
|
-
|
|
2373
|
+
`
|
|
2305
2374
|
ALTER TABLE projects ADD COLUMN scenario_prefix TEXT DEFAULT 'TST';
|
|
2306
2375
|
ALTER TABLE projects ADD COLUMN scenario_counter INTEGER DEFAULT 0;
|
|
2376
|
+
`,
|
|
2377
|
+
`
|
|
2378
|
+
CREATE TABLE IF NOT EXISTS schedules (
|
|
2379
|
+
id TEXT PRIMARY KEY,
|
|
2380
|
+
project_id TEXT REFERENCES projects(id) ON DELETE CASCADE,
|
|
2381
|
+
name TEXT NOT NULL,
|
|
2382
|
+
cron_expression TEXT NOT NULL,
|
|
2383
|
+
url TEXT NOT NULL,
|
|
2384
|
+
scenario_filter TEXT NOT NULL DEFAULT '{}',
|
|
2385
|
+
model TEXT,
|
|
2386
|
+
headed INTEGER NOT NULL DEFAULT 0,
|
|
2387
|
+
parallel INTEGER NOT NULL DEFAULT 1,
|
|
2388
|
+
timeout_ms INTEGER,
|
|
2389
|
+
enabled INTEGER NOT NULL DEFAULT 1,
|
|
2390
|
+
last_run_id TEXT REFERENCES runs(id) ON DELETE SET NULL,
|
|
2391
|
+
last_run_at TEXT,
|
|
2392
|
+
next_run_at TEXT,
|
|
2393
|
+
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
2394
|
+
updated_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
2395
|
+
);
|
|
2396
|
+
|
|
2397
|
+
CREATE INDEX IF NOT EXISTS idx_schedules_project ON schedules(project_id);
|
|
2398
|
+
CREATE INDEX IF NOT EXISTS idx_schedules_enabled ON schedules(enabled);
|
|
2399
|
+
CREATE INDEX IF NOT EXISTS idx_schedules_next_run ON schedules(next_run_at);
|
|
2400
|
+
`,
|
|
2401
|
+
`
|
|
2402
|
+
ALTER TABLE screenshots ADD COLUMN description TEXT;
|
|
2403
|
+
ALTER TABLE screenshots ADD COLUMN page_url TEXT;
|
|
2404
|
+
ALTER TABLE screenshots ADD COLUMN thumbnail_path TEXT;
|
|
2405
|
+
`,
|
|
2406
|
+
`
|
|
2407
|
+
CREATE TABLE IF NOT EXISTS auth_presets (
|
|
2408
|
+
id TEXT PRIMARY KEY,
|
|
2409
|
+
name TEXT NOT NULL UNIQUE,
|
|
2410
|
+
email TEXT NOT NULL,
|
|
2411
|
+
password TEXT NOT NULL,
|
|
2412
|
+
login_path TEXT DEFAULT '/login',
|
|
2413
|
+
metadata TEXT DEFAULT '{}',
|
|
2414
|
+
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
2415
|
+
);
|
|
2416
|
+
`,
|
|
2417
|
+
`
|
|
2418
|
+
CREATE TABLE IF NOT EXISTS webhooks (
|
|
2419
|
+
id TEXT PRIMARY KEY,
|
|
2420
|
+
url TEXT NOT NULL,
|
|
2421
|
+
events TEXT NOT NULL DEFAULT '["failed"]',
|
|
2422
|
+
project_id TEXT REFERENCES projects(id) ON DELETE CASCADE,
|
|
2423
|
+
secret TEXT,
|
|
2424
|
+
active INTEGER NOT NULL DEFAULT 1,
|
|
2425
|
+
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
2426
|
+
);
|
|
2427
|
+
CREATE INDEX IF NOT EXISTS idx_webhooks_active ON webhooks(active);
|
|
2307
2428
|
`
|
|
2308
|
-
];
|
|
2309
|
-
|
|
2310
|
-
const applied = database.query("SELECT id FROM _migrations ORDER BY id").all();
|
|
2311
|
-
const appliedIds = new Set(applied.map((r) => r.id));
|
|
2312
|
-
for (let i = 0;i < MIGRATIONS.length; i++) {
|
|
2313
|
-
const migrationId = i + 1;
|
|
2314
|
-
if (appliedIds.has(migrationId))
|
|
2315
|
-
continue;
|
|
2316
|
-
const migration = MIGRATIONS[i];
|
|
2317
|
-
database.exec(migration);
|
|
2318
|
-
database.query("INSERT INTO _migrations (id, applied_at) VALUES (?, ?)").run(migrationId, now());
|
|
2319
|
-
}
|
|
2320
|
-
}
|
|
2321
|
-
function getDatabase() {
|
|
2322
|
-
if (db)
|
|
2323
|
-
return db;
|
|
2324
|
-
const dbPath = resolveDbPath();
|
|
2325
|
-
const dir = dirname(dbPath);
|
|
2326
|
-
if (dbPath !== ":memory:" && !existsSync(dir)) {
|
|
2327
|
-
mkdirSync(dir, { recursive: true });
|
|
2328
|
-
}
|
|
2329
|
-
db = new Database(dbPath);
|
|
2330
|
-
db.exec("PRAGMA journal_mode = WAL");
|
|
2331
|
-
db.exec("PRAGMA foreign_keys = ON");
|
|
2332
|
-
db.exec("PRAGMA busy_timeout = 5000");
|
|
2333
|
-
db.exec(`
|
|
2334
|
-
CREATE TABLE IF NOT EXISTS _migrations (
|
|
2335
|
-
id INTEGER PRIMARY KEY,
|
|
2336
|
-
applied_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
2337
|
-
);
|
|
2338
|
-
`);
|
|
2339
|
-
applyMigrations(db);
|
|
2340
|
-
return db;
|
|
2341
|
-
}
|
|
2342
|
-
function resolvePartialId(table, partialId) {
|
|
2343
|
-
const database = getDatabase();
|
|
2344
|
-
const rows = database.query(`SELECT id FROM ${table} WHERE id LIKE ? || '%'`).all(partialId);
|
|
2345
|
-
if (rows.length === 1)
|
|
2346
|
-
return rows[0].id;
|
|
2347
|
-
return null;
|
|
2348
|
-
}
|
|
2429
|
+
];
|
|
2430
|
+
});
|
|
2349
2431
|
|
|
2350
|
-
// src/db/
|
|
2351
|
-
|
|
2352
|
-
|
|
2353
|
-
|
|
2354
|
-
|
|
2355
|
-
|
|
2356
|
-
|
|
2357
|
-
|
|
2358
|
-
|
|
2359
|
-
|
|
2360
|
-
}
|
|
2361
|
-
return shortUuid();
|
|
2362
|
-
}
|
|
2363
|
-
function createScenario(input) {
|
|
2432
|
+
// src/db/runs.ts
|
|
2433
|
+
var exports_runs = {};
|
|
2434
|
+
__export(exports_runs, {
|
|
2435
|
+
updateRun: () => updateRun,
|
|
2436
|
+
listRuns: () => listRuns,
|
|
2437
|
+
getRun: () => getRun,
|
|
2438
|
+
deleteRun: () => deleteRun,
|
|
2439
|
+
createRun: () => createRun
|
|
2440
|
+
});
|
|
2441
|
+
function createRun(input) {
|
|
2364
2442
|
const db2 = getDatabase();
|
|
2365
2443
|
const id = uuid();
|
|
2366
|
-
const short_id = nextShortId(input.projectId);
|
|
2367
2444
|
const timestamp = now();
|
|
2368
2445
|
db2.query(`
|
|
2369
|
-
INSERT INTO
|
|
2370
|
-
VALUES (?, ?,
|
|
2371
|
-
`).run(id,
|
|
2372
|
-
return
|
|
2446
|
+
INSERT INTO runs (id, project_id, status, url, model, headed, parallel, total, passed, failed, started_at, finished_at, metadata)
|
|
2447
|
+
VALUES (?, ?, 'pending', ?, ?, ?, ?, 0, 0, 0, ?, NULL, ?)
|
|
2448
|
+
`).run(id, input.projectId ?? null, input.url, input.model, input.headed ? 1 : 0, input.parallel ?? 1, timestamp, input.model ? JSON.stringify({}) : null);
|
|
2449
|
+
return getRun(id);
|
|
2373
2450
|
}
|
|
2374
|
-
function
|
|
2451
|
+
function getRun(id) {
|
|
2375
2452
|
const db2 = getDatabase();
|
|
2376
|
-
let row = db2.query("SELECT * FROM
|
|
2377
|
-
if (row)
|
|
2378
|
-
return scenarioFromRow(row);
|
|
2379
|
-
row = db2.query("SELECT * FROM scenarios WHERE short_id = ?").get(id);
|
|
2453
|
+
let row = db2.query("SELECT * FROM runs WHERE id = ?").get(id);
|
|
2380
2454
|
if (row)
|
|
2381
|
-
return
|
|
2382
|
-
const fullId = resolvePartialId("
|
|
2455
|
+
return runFromRow(row);
|
|
2456
|
+
const fullId = resolvePartialId("runs", id);
|
|
2383
2457
|
if (fullId) {
|
|
2384
|
-
row = db2.query("SELECT * FROM
|
|
2458
|
+
row = db2.query("SELECT * FROM runs WHERE id = ?").get(fullId);
|
|
2385
2459
|
if (row)
|
|
2386
|
-
return
|
|
2460
|
+
return runFromRow(row);
|
|
2387
2461
|
}
|
|
2388
2462
|
return null;
|
|
2389
2463
|
}
|
|
2390
|
-
function
|
|
2391
|
-
const db2 = getDatabase();
|
|
2392
|
-
const row = db2.query("SELECT * FROM scenarios WHERE short_id = ?").get(shortId);
|
|
2393
|
-
return row ? scenarioFromRow(row) : null;
|
|
2394
|
-
}
|
|
2395
|
-
function listScenarios(filter) {
|
|
2464
|
+
function listRuns(filter) {
|
|
2396
2465
|
const db2 = getDatabase();
|
|
2397
2466
|
const conditions = [];
|
|
2398
2467
|
const params = [];
|
|
@@ -2400,26 +2469,15 @@ function listScenarios(filter) {
|
|
|
2400
2469
|
conditions.push("project_id = ?");
|
|
2401
2470
|
params.push(filter.projectId);
|
|
2402
2471
|
}
|
|
2403
|
-
if (filter?.
|
|
2404
|
-
|
|
2405
|
-
|
|
2406
|
-
params.push(`%"${tag}"%`);
|
|
2407
|
-
}
|
|
2408
|
-
}
|
|
2409
|
-
if (filter?.priority) {
|
|
2410
|
-
conditions.push("priority = ?");
|
|
2411
|
-
params.push(filter.priority);
|
|
2412
|
-
}
|
|
2413
|
-
if (filter?.search) {
|
|
2414
|
-
conditions.push("(name LIKE ? OR description LIKE ?)");
|
|
2415
|
-
const term = `%${filter.search}%`;
|
|
2416
|
-
params.push(term, term);
|
|
2472
|
+
if (filter?.status) {
|
|
2473
|
+
conditions.push("status = ?");
|
|
2474
|
+
params.push(filter.status);
|
|
2417
2475
|
}
|
|
2418
|
-
let sql = "SELECT * FROM
|
|
2476
|
+
let sql = "SELECT * FROM runs";
|
|
2419
2477
|
if (conditions.length > 0) {
|
|
2420
2478
|
sql += " WHERE " + conditions.join(" AND ");
|
|
2421
2479
|
}
|
|
2422
|
-
sql += " ORDER BY
|
|
2480
|
+
sql += " ORDER BY started_at DESC";
|
|
2423
2481
|
if (filter?.limit) {
|
|
2424
2482
|
sql += " LIMIT ?";
|
|
2425
2483
|
params.push(filter.limit);
|
|
@@ -2429,11 +2487,190 @@ function listScenarios(filter) {
|
|
|
2429
2487
|
params.push(filter.offset);
|
|
2430
2488
|
}
|
|
2431
2489
|
const rows = db2.query(sql).all(...params);
|
|
2432
|
-
return rows.map(
|
|
2490
|
+
return rows.map(runFromRow);
|
|
2433
2491
|
}
|
|
2434
|
-
function
|
|
2492
|
+
function updateRun(id, updates) {
|
|
2435
2493
|
const db2 = getDatabase();
|
|
2436
|
-
const existing =
|
|
2494
|
+
const existing = getRun(id);
|
|
2495
|
+
if (!existing) {
|
|
2496
|
+
throw new Error(`Run not found: ${id}`);
|
|
2497
|
+
}
|
|
2498
|
+
const sets = [];
|
|
2499
|
+
const params = [];
|
|
2500
|
+
if (updates.status !== undefined) {
|
|
2501
|
+
sets.push("status = ?");
|
|
2502
|
+
params.push(updates.status);
|
|
2503
|
+
}
|
|
2504
|
+
if (updates.url !== undefined) {
|
|
2505
|
+
sets.push("url = ?");
|
|
2506
|
+
params.push(updates.url);
|
|
2507
|
+
}
|
|
2508
|
+
if (updates.model !== undefined) {
|
|
2509
|
+
sets.push("model = ?");
|
|
2510
|
+
params.push(updates.model);
|
|
2511
|
+
}
|
|
2512
|
+
if (updates.headed !== undefined) {
|
|
2513
|
+
sets.push("headed = ?");
|
|
2514
|
+
params.push(updates.headed);
|
|
2515
|
+
}
|
|
2516
|
+
if (updates.parallel !== undefined) {
|
|
2517
|
+
sets.push("parallel = ?");
|
|
2518
|
+
params.push(updates.parallel);
|
|
2519
|
+
}
|
|
2520
|
+
if (updates.total !== undefined) {
|
|
2521
|
+
sets.push("total = ?");
|
|
2522
|
+
params.push(updates.total);
|
|
2523
|
+
}
|
|
2524
|
+
if (updates.passed !== undefined) {
|
|
2525
|
+
sets.push("passed = ?");
|
|
2526
|
+
params.push(updates.passed);
|
|
2527
|
+
}
|
|
2528
|
+
if (updates.failed !== undefined) {
|
|
2529
|
+
sets.push("failed = ?");
|
|
2530
|
+
params.push(updates.failed);
|
|
2531
|
+
}
|
|
2532
|
+
if (updates.started_at !== undefined) {
|
|
2533
|
+
sets.push("started_at = ?");
|
|
2534
|
+
params.push(updates.started_at);
|
|
2535
|
+
}
|
|
2536
|
+
if (updates.finished_at !== undefined) {
|
|
2537
|
+
sets.push("finished_at = ?");
|
|
2538
|
+
params.push(updates.finished_at);
|
|
2539
|
+
}
|
|
2540
|
+
if (updates.metadata !== undefined) {
|
|
2541
|
+
sets.push("metadata = ?");
|
|
2542
|
+
params.push(updates.metadata);
|
|
2543
|
+
}
|
|
2544
|
+
if (sets.length === 0) {
|
|
2545
|
+
return existing;
|
|
2546
|
+
}
|
|
2547
|
+
params.push(existing.id);
|
|
2548
|
+
db2.query(`UPDATE runs SET ${sets.join(", ")} WHERE id = ?`).run(...params);
|
|
2549
|
+
return getRun(existing.id);
|
|
2550
|
+
}
|
|
2551
|
+
function deleteRun(id) {
|
|
2552
|
+
const db2 = getDatabase();
|
|
2553
|
+
const run = getRun(id);
|
|
2554
|
+
if (!run)
|
|
2555
|
+
return false;
|
|
2556
|
+
const result = db2.query("DELETE FROM runs WHERE id = ?").run(run.id);
|
|
2557
|
+
return result.changes > 0;
|
|
2558
|
+
}
|
|
2559
|
+
var init_runs = __esm(() => {
|
|
2560
|
+
init_types();
|
|
2561
|
+
init_database();
|
|
2562
|
+
});
|
|
2563
|
+
|
|
2564
|
+
// node_modules/commander/esm.mjs
|
|
2565
|
+
var import__ = __toESM(require_commander(), 1);
|
|
2566
|
+
var {
|
|
2567
|
+
program,
|
|
2568
|
+
createCommand,
|
|
2569
|
+
createArgument,
|
|
2570
|
+
createOption,
|
|
2571
|
+
CommanderError,
|
|
2572
|
+
InvalidArgumentError,
|
|
2573
|
+
InvalidOptionArgumentError,
|
|
2574
|
+
Command,
|
|
2575
|
+
Argument,
|
|
2576
|
+
Option,
|
|
2577
|
+
Help
|
|
2578
|
+
} = import__.default;
|
|
2579
|
+
|
|
2580
|
+
// src/cli/index.tsx
|
|
2581
|
+
import chalk4 from "chalk";
|
|
2582
|
+
import { readFileSync as readFileSync4, readdirSync, writeFileSync as writeFileSync3 } from "fs";
|
|
2583
|
+
import { join as join6, resolve } from "path";
|
|
2584
|
+
|
|
2585
|
+
// src/db/scenarios.ts
|
|
2586
|
+
init_types();
|
|
2587
|
+
init_database();
|
|
2588
|
+
function nextShortId(projectId) {
|
|
2589
|
+
const db2 = getDatabase();
|
|
2590
|
+
if (projectId) {
|
|
2591
|
+
const project = db2.query("SELECT scenario_prefix, scenario_counter FROM projects WHERE id = ?").get(projectId);
|
|
2592
|
+
if (project) {
|
|
2593
|
+
const next = project.scenario_counter + 1;
|
|
2594
|
+
db2.query("UPDATE projects SET scenario_counter = ? WHERE id = ?").run(next, projectId);
|
|
2595
|
+
return `${project.scenario_prefix}-${next}`;
|
|
2596
|
+
}
|
|
2597
|
+
}
|
|
2598
|
+
return shortUuid();
|
|
2599
|
+
}
|
|
2600
|
+
function createScenario(input) {
|
|
2601
|
+
const db2 = getDatabase();
|
|
2602
|
+
const id = uuid();
|
|
2603
|
+
const short_id = nextShortId(input.projectId);
|
|
2604
|
+
const timestamp = now();
|
|
2605
|
+
db2.query(`
|
|
2606
|
+
INSERT INTO scenarios (id, short_id, project_id, name, description, steps, tags, priority, model, timeout_ms, target_path, requires_auth, auth_config, metadata, version, created_at, updated_at)
|
|
2607
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1, ?, ?)
|
|
2608
|
+
`).run(id, short_id, input.projectId ?? null, input.name, input.description, JSON.stringify(input.steps ?? []), JSON.stringify(input.tags ?? []), input.priority ?? "medium", input.model ?? null, input.timeoutMs ?? null, input.targetPath ?? null, input.requiresAuth ? 1 : 0, input.authConfig ? JSON.stringify(input.authConfig) : null, input.metadata ? JSON.stringify(input.metadata) : null, timestamp, timestamp);
|
|
2609
|
+
return getScenario(id);
|
|
2610
|
+
}
|
|
2611
|
+
function getScenario(id) {
|
|
2612
|
+
const db2 = getDatabase();
|
|
2613
|
+
let row = db2.query("SELECT * FROM scenarios WHERE id = ?").get(id);
|
|
2614
|
+
if (row)
|
|
2615
|
+
return scenarioFromRow(row);
|
|
2616
|
+
row = db2.query("SELECT * FROM scenarios WHERE short_id = ?").get(id);
|
|
2617
|
+
if (row)
|
|
2618
|
+
return scenarioFromRow(row);
|
|
2619
|
+
const fullId = resolvePartialId("scenarios", id);
|
|
2620
|
+
if (fullId) {
|
|
2621
|
+
row = db2.query("SELECT * FROM scenarios WHERE id = ?").get(fullId);
|
|
2622
|
+
if (row)
|
|
2623
|
+
return scenarioFromRow(row);
|
|
2624
|
+
}
|
|
2625
|
+
return null;
|
|
2626
|
+
}
|
|
2627
|
+
function getScenarioByShortId(shortId) {
|
|
2628
|
+
const db2 = getDatabase();
|
|
2629
|
+
const row = db2.query("SELECT * FROM scenarios WHERE short_id = ?").get(shortId);
|
|
2630
|
+
return row ? scenarioFromRow(row) : null;
|
|
2631
|
+
}
|
|
2632
|
+
function listScenarios(filter) {
|
|
2633
|
+
const db2 = getDatabase();
|
|
2634
|
+
const conditions = [];
|
|
2635
|
+
const params = [];
|
|
2636
|
+
if (filter?.projectId) {
|
|
2637
|
+
conditions.push("project_id = ?");
|
|
2638
|
+
params.push(filter.projectId);
|
|
2639
|
+
}
|
|
2640
|
+
if (filter?.tags && filter.tags.length > 0) {
|
|
2641
|
+
for (const tag of filter.tags) {
|
|
2642
|
+
conditions.push("tags LIKE ?");
|
|
2643
|
+
params.push(`%"${tag}"%`);
|
|
2644
|
+
}
|
|
2645
|
+
}
|
|
2646
|
+
if (filter?.priority) {
|
|
2647
|
+
conditions.push("priority = ?");
|
|
2648
|
+
params.push(filter.priority);
|
|
2649
|
+
}
|
|
2650
|
+
if (filter?.search) {
|
|
2651
|
+
conditions.push("(name LIKE ? OR description LIKE ?)");
|
|
2652
|
+
const term = `%${filter.search}%`;
|
|
2653
|
+
params.push(term, term);
|
|
2654
|
+
}
|
|
2655
|
+
let sql = "SELECT * FROM scenarios";
|
|
2656
|
+
if (conditions.length > 0) {
|
|
2657
|
+
sql += " WHERE " + conditions.join(" AND ");
|
|
2658
|
+
}
|
|
2659
|
+
sql += " ORDER BY created_at DESC";
|
|
2660
|
+
if (filter?.limit) {
|
|
2661
|
+
sql += " LIMIT ?";
|
|
2662
|
+
params.push(filter.limit);
|
|
2663
|
+
}
|
|
2664
|
+
if (filter?.offset) {
|
|
2665
|
+
sql += " OFFSET ?";
|
|
2666
|
+
params.push(filter.offset);
|
|
2667
|
+
}
|
|
2668
|
+
const rows = db2.query(sql).all(...params);
|
|
2669
|
+
return rows.map(scenarioFromRow);
|
|
2670
|
+
}
|
|
2671
|
+
function updateScenario(id, input, version) {
|
|
2672
|
+
const db2 = getDatabase();
|
|
2673
|
+
const existing = getScenario(id);
|
|
2437
2674
|
if (!existing) {
|
|
2438
2675
|
throw new Error(`Scenario not found: ${id}`);
|
|
2439
2676
|
}
|
|
@@ -2510,63 +2747,45 @@ function deleteScenario(id) {
|
|
|
2510
2747
|
return result.changes > 0;
|
|
2511
2748
|
}
|
|
2512
2749
|
|
|
2513
|
-
// src/
|
|
2514
|
-
|
|
2750
|
+
// src/cli/index.tsx
|
|
2751
|
+
init_runs();
|
|
2752
|
+
|
|
2753
|
+
// src/db/results.ts
|
|
2754
|
+
init_types();
|
|
2755
|
+
init_database();
|
|
2756
|
+
function createResult(input) {
|
|
2515
2757
|
const db2 = getDatabase();
|
|
2516
2758
|
const id = uuid();
|
|
2517
2759
|
const timestamp = now();
|
|
2518
2760
|
db2.query(`
|
|
2519
|
-
INSERT INTO
|
|
2520
|
-
VALUES (?, ?, '
|
|
2521
|
-
`).run(id, input.
|
|
2522
|
-
return
|
|
2761
|
+
INSERT INTO results (id, run_id, scenario_id, status, reasoning, error, steps_completed, steps_total, duration_ms, model, tokens_used, cost_cents, metadata, created_at)
|
|
2762
|
+
VALUES (?, ?, ?, 'skipped', NULL, NULL, 0, ?, 0, ?, 0, 0, '{}', ?)
|
|
2763
|
+
`).run(id, input.runId, input.scenarioId, input.stepsTotal, input.model, timestamp);
|
|
2764
|
+
return getResult(id);
|
|
2523
2765
|
}
|
|
2524
|
-
function
|
|
2766
|
+
function getResult(id) {
|
|
2525
2767
|
const db2 = getDatabase();
|
|
2526
|
-
let row = db2.query("SELECT * FROM
|
|
2768
|
+
let row = db2.query("SELECT * FROM results WHERE id = ?").get(id);
|
|
2527
2769
|
if (row)
|
|
2528
|
-
return
|
|
2529
|
-
const fullId = resolvePartialId("
|
|
2770
|
+
return resultFromRow(row);
|
|
2771
|
+
const fullId = resolvePartialId("results", id);
|
|
2530
2772
|
if (fullId) {
|
|
2531
|
-
row = db2.query("SELECT * FROM
|
|
2773
|
+
row = db2.query("SELECT * FROM results WHERE id = ?").get(fullId);
|
|
2532
2774
|
if (row)
|
|
2533
|
-
return
|
|
2775
|
+
return resultFromRow(row);
|
|
2534
2776
|
}
|
|
2535
2777
|
return null;
|
|
2536
2778
|
}
|
|
2537
|
-
function
|
|
2779
|
+
function listResults(runId) {
|
|
2538
2780
|
const db2 = getDatabase();
|
|
2539
|
-
const
|
|
2540
|
-
|
|
2541
|
-
if (filter?.projectId) {
|
|
2542
|
-
conditions.push("project_id = ?");
|
|
2543
|
-
params.push(filter.projectId);
|
|
2544
|
-
}
|
|
2545
|
-
if (filter?.status) {
|
|
2546
|
-
conditions.push("status = ?");
|
|
2547
|
-
params.push(filter.status);
|
|
2548
|
-
}
|
|
2549
|
-
let sql = "SELECT * FROM runs";
|
|
2550
|
-
if (conditions.length > 0) {
|
|
2551
|
-
sql += " WHERE " + conditions.join(" AND ");
|
|
2552
|
-
}
|
|
2553
|
-
sql += " ORDER BY started_at DESC";
|
|
2554
|
-
if (filter?.limit) {
|
|
2555
|
-
sql += " LIMIT ?";
|
|
2556
|
-
params.push(filter.limit);
|
|
2557
|
-
}
|
|
2558
|
-
if (filter?.offset) {
|
|
2559
|
-
sql += " OFFSET ?";
|
|
2560
|
-
params.push(filter.offset);
|
|
2561
|
-
}
|
|
2562
|
-
const rows = db2.query(sql).all(...params);
|
|
2563
|
-
return rows.map(runFromRow);
|
|
2781
|
+
const rows = db2.query("SELECT * FROM results WHERE run_id = ? ORDER BY created_at ASC").all(runId);
|
|
2782
|
+
return rows.map(resultFromRow);
|
|
2564
2783
|
}
|
|
2565
|
-
function
|
|
2784
|
+
function updateResult(id, updates) {
|
|
2566
2785
|
const db2 = getDatabase();
|
|
2567
|
-
const existing =
|
|
2786
|
+
const existing = getResult(id);
|
|
2568
2787
|
if (!existing) {
|
|
2569
|
-
throw new Error(`
|
|
2788
|
+
throw new Error(`Result not found: ${id}`);
|
|
2570
2789
|
}
|
|
2571
2790
|
const sets = [];
|
|
2572
2791
|
const params = [];
|
|
@@ -2574,110 +2793,21 @@ function updateRun(id, updates) {
|
|
|
2574
2793
|
sets.push("status = ?");
|
|
2575
2794
|
params.push(updates.status);
|
|
2576
2795
|
}
|
|
2577
|
-
if (updates.
|
|
2578
|
-
sets.push("
|
|
2579
|
-
params.push(updates.
|
|
2580
|
-
}
|
|
2581
|
-
if (updates.model !== undefined) {
|
|
2582
|
-
sets.push("model = ?");
|
|
2583
|
-
params.push(updates.model);
|
|
2796
|
+
if (updates.reasoning !== undefined) {
|
|
2797
|
+
sets.push("reasoning = ?");
|
|
2798
|
+
params.push(updates.reasoning);
|
|
2584
2799
|
}
|
|
2585
|
-
if (updates.
|
|
2586
|
-
sets.push("
|
|
2587
|
-
params.push(updates.
|
|
2800
|
+
if (updates.error !== undefined) {
|
|
2801
|
+
sets.push("error = ?");
|
|
2802
|
+
params.push(updates.error);
|
|
2588
2803
|
}
|
|
2589
|
-
if (updates.
|
|
2590
|
-
sets.push("
|
|
2591
|
-
params.push(updates.
|
|
2804
|
+
if (updates.stepsCompleted !== undefined) {
|
|
2805
|
+
sets.push("steps_completed = ?");
|
|
2806
|
+
params.push(updates.stepsCompleted);
|
|
2592
2807
|
}
|
|
2593
|
-
if (updates.
|
|
2594
|
-
sets.push("
|
|
2595
|
-
params.push(updates.
|
|
2596
|
-
}
|
|
2597
|
-
if (updates.passed !== undefined) {
|
|
2598
|
-
sets.push("passed = ?");
|
|
2599
|
-
params.push(updates.passed);
|
|
2600
|
-
}
|
|
2601
|
-
if (updates.failed !== undefined) {
|
|
2602
|
-
sets.push("failed = ?");
|
|
2603
|
-
params.push(updates.failed);
|
|
2604
|
-
}
|
|
2605
|
-
if (updates.started_at !== undefined) {
|
|
2606
|
-
sets.push("started_at = ?");
|
|
2607
|
-
params.push(updates.started_at);
|
|
2608
|
-
}
|
|
2609
|
-
if (updates.finished_at !== undefined) {
|
|
2610
|
-
sets.push("finished_at = ?");
|
|
2611
|
-
params.push(updates.finished_at);
|
|
2612
|
-
}
|
|
2613
|
-
if (updates.metadata !== undefined) {
|
|
2614
|
-
sets.push("metadata = ?");
|
|
2615
|
-
params.push(updates.metadata);
|
|
2616
|
-
}
|
|
2617
|
-
if (sets.length === 0) {
|
|
2618
|
-
return existing;
|
|
2619
|
-
}
|
|
2620
|
-
params.push(existing.id);
|
|
2621
|
-
db2.query(`UPDATE runs SET ${sets.join(", ")} WHERE id = ?`).run(...params);
|
|
2622
|
-
return getRun(existing.id);
|
|
2623
|
-
}
|
|
2624
|
-
|
|
2625
|
-
// src/db/results.ts
|
|
2626
|
-
function createResult(input) {
|
|
2627
|
-
const db2 = getDatabase();
|
|
2628
|
-
const id = uuid();
|
|
2629
|
-
const timestamp = now();
|
|
2630
|
-
db2.query(`
|
|
2631
|
-
INSERT INTO results (id, run_id, scenario_id, status, reasoning, error, steps_completed, steps_total, duration_ms, model, tokens_used, cost_cents, metadata, created_at)
|
|
2632
|
-
VALUES (?, ?, ?, 'skipped', NULL, NULL, 0, ?, 0, ?, 0, 0, '{}', ?)
|
|
2633
|
-
`).run(id, input.runId, input.scenarioId, input.stepsTotal, input.model, timestamp);
|
|
2634
|
-
return getResult(id);
|
|
2635
|
-
}
|
|
2636
|
-
function getResult(id) {
|
|
2637
|
-
const db2 = getDatabase();
|
|
2638
|
-
let row = db2.query("SELECT * FROM results WHERE id = ?").get(id);
|
|
2639
|
-
if (row)
|
|
2640
|
-
return resultFromRow(row);
|
|
2641
|
-
const fullId = resolvePartialId("results", id);
|
|
2642
|
-
if (fullId) {
|
|
2643
|
-
row = db2.query("SELECT * FROM results WHERE id = ?").get(fullId);
|
|
2644
|
-
if (row)
|
|
2645
|
-
return resultFromRow(row);
|
|
2646
|
-
}
|
|
2647
|
-
return null;
|
|
2648
|
-
}
|
|
2649
|
-
function listResults(runId) {
|
|
2650
|
-
const db2 = getDatabase();
|
|
2651
|
-
const rows = db2.query("SELECT * FROM results WHERE run_id = ? ORDER BY created_at ASC").all(runId);
|
|
2652
|
-
return rows.map(resultFromRow);
|
|
2653
|
-
}
|
|
2654
|
-
function updateResult(id, updates) {
|
|
2655
|
-
const db2 = getDatabase();
|
|
2656
|
-
const existing = getResult(id);
|
|
2657
|
-
if (!existing) {
|
|
2658
|
-
throw new Error(`Result not found: ${id}`);
|
|
2659
|
-
}
|
|
2660
|
-
const sets = [];
|
|
2661
|
-
const params = [];
|
|
2662
|
-
if (updates.status !== undefined) {
|
|
2663
|
-
sets.push("status = ?");
|
|
2664
|
-
params.push(updates.status);
|
|
2665
|
-
}
|
|
2666
|
-
if (updates.reasoning !== undefined) {
|
|
2667
|
-
sets.push("reasoning = ?");
|
|
2668
|
-
params.push(updates.reasoning);
|
|
2669
|
-
}
|
|
2670
|
-
if (updates.error !== undefined) {
|
|
2671
|
-
sets.push("error = ?");
|
|
2672
|
-
params.push(updates.error);
|
|
2673
|
-
}
|
|
2674
|
-
if (updates.stepsCompleted !== undefined) {
|
|
2675
|
-
sets.push("steps_completed = ?");
|
|
2676
|
-
params.push(updates.stepsCompleted);
|
|
2677
|
-
}
|
|
2678
|
-
if (updates.durationMs !== undefined) {
|
|
2679
|
-
sets.push("duration_ms = ?");
|
|
2680
|
-
params.push(updates.durationMs);
|
|
2808
|
+
if (updates.durationMs !== undefined) {
|
|
2809
|
+
sets.push("duration_ms = ?");
|
|
2810
|
+
params.push(updates.durationMs);
|
|
2681
2811
|
}
|
|
2682
2812
|
if (updates.tokensUsed !== undefined) {
|
|
2683
2813
|
sets.push("tokens_used = ?");
|
|
@@ -2699,14 +2829,16 @@ function getResultsByRun(runId) {
|
|
|
2699
2829
|
}
|
|
2700
2830
|
|
|
2701
2831
|
// src/db/screenshots.ts
|
|
2832
|
+
init_types();
|
|
2833
|
+
init_database();
|
|
2702
2834
|
function createScreenshot(input) {
|
|
2703
2835
|
const db2 = getDatabase();
|
|
2704
2836
|
const id = uuid();
|
|
2705
2837
|
const timestamp = now();
|
|
2706
2838
|
db2.query(`
|
|
2707
|
-
INSERT INTO screenshots (id, result_id, step_number, action, file_path, width, height, timestamp)
|
|
2708
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
2709
|
-
`).run(id, input.resultId, input.stepNumber, input.action, input.filePath, input.width, input.height, timestamp);
|
|
2839
|
+
INSERT INTO screenshots (id, result_id, step_number, action, file_path, width, height, timestamp, description, page_url, thumbnail_path)
|
|
2840
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
2841
|
+
`).run(id, input.resultId, input.stepNumber, input.action, input.filePath, input.width, input.height, timestamp, input.description ?? null, input.pageUrl ?? null, input.thumbnailPath ?? null);
|
|
2710
2842
|
return getScreenshot(id);
|
|
2711
2843
|
}
|
|
2712
2844
|
function getScreenshot(id) {
|
|
@@ -2720,7 +2852,11 @@ function listScreenshots(resultId) {
|
|
|
2720
2852
|
return rows.map(screenshotFromRow);
|
|
2721
2853
|
}
|
|
2722
2854
|
|
|
2855
|
+
// src/lib/runner.ts
|
|
2856
|
+
init_runs();
|
|
2857
|
+
|
|
2723
2858
|
// src/lib/browser.ts
|
|
2859
|
+
init_types();
|
|
2724
2860
|
import { chromium } from "playwright";
|
|
2725
2861
|
import { execSync } from "child_process";
|
|
2726
2862
|
var DEFAULT_VIEWPORT = { width: 1280, height: 720 };
|
|
@@ -2775,7 +2911,7 @@ async function installBrowser() {
|
|
|
2775
2911
|
}
|
|
2776
2912
|
|
|
2777
2913
|
// src/lib/screenshotter.ts
|
|
2778
|
-
import { mkdirSync as mkdirSync2, existsSync as existsSync2 } from "fs";
|
|
2914
|
+
import { mkdirSync as mkdirSync2, existsSync as existsSync2, writeFileSync } from "fs";
|
|
2779
2915
|
import { join as join2 } from "path";
|
|
2780
2916
|
import { homedir as homedir2 } from "os";
|
|
2781
2917
|
function slugify(text) {
|
|
@@ -2784,16 +2920,51 @@ function slugify(text) {
|
|
|
2784
2920
|
function generateFilename(stepNumber, action) {
|
|
2785
2921
|
const padded = String(stepNumber).padStart(3, "0");
|
|
2786
2922
|
const slug = slugify(action);
|
|
2787
|
-
return `${padded}
|
|
2923
|
+
return `${padded}_${slug}.png`;
|
|
2924
|
+
}
|
|
2925
|
+
function formatDate(date) {
|
|
2926
|
+
return date.toISOString().slice(0, 10);
|
|
2788
2927
|
}
|
|
2789
|
-
function
|
|
2790
|
-
return
|
|
2928
|
+
function formatTime(date) {
|
|
2929
|
+
return date.toISOString().slice(11, 19).replace(/:/g, "-");
|
|
2930
|
+
}
|
|
2931
|
+
function getScreenshotDir(baseDir, runId, scenarioSlug, projectName, timestamp) {
|
|
2932
|
+
const now2 = timestamp ?? new Date;
|
|
2933
|
+
const project = projectName ?? "default";
|
|
2934
|
+
const dateDir = formatDate(now2);
|
|
2935
|
+
const timeDir = `${formatTime(now2)}_${runId.slice(0, 8)}`;
|
|
2936
|
+
return join2(baseDir, project, dateDir, timeDir, scenarioSlug);
|
|
2791
2937
|
}
|
|
2792
2938
|
function ensureDir(dirPath) {
|
|
2793
2939
|
if (!existsSync2(dirPath)) {
|
|
2794
2940
|
mkdirSync2(dirPath, { recursive: true });
|
|
2795
2941
|
}
|
|
2796
2942
|
}
|
|
2943
|
+
function writeMetaSidecar(screenshotPath, meta) {
|
|
2944
|
+
const metaPath = screenshotPath.replace(/\.png$/, ".meta.json").replace(/\.jpeg$/, ".meta.json");
|
|
2945
|
+
try {
|
|
2946
|
+
writeFileSync(metaPath, JSON.stringify(meta, null, 2), "utf-8");
|
|
2947
|
+
} catch {}
|
|
2948
|
+
}
|
|
2949
|
+
async function generateThumbnail(page, screenshotDir, filename) {
|
|
2950
|
+
try {
|
|
2951
|
+
const thumbDir = join2(screenshotDir, "_thumbnail");
|
|
2952
|
+
ensureDir(thumbDir);
|
|
2953
|
+
const thumbFilename = filename.replace(/\.(png|jpeg)$/, ".thumb.$1");
|
|
2954
|
+
const thumbPath = join2(thumbDir, thumbFilename);
|
|
2955
|
+
const viewport = page.viewportSize();
|
|
2956
|
+
if (viewport) {
|
|
2957
|
+
await page.screenshot({
|
|
2958
|
+
path: thumbPath,
|
|
2959
|
+
type: "png",
|
|
2960
|
+
clip: { x: 0, y: 0, width: Math.min(viewport.width, 1280), height: Math.min(viewport.height, 720) }
|
|
2961
|
+
});
|
|
2962
|
+
}
|
|
2963
|
+
return thumbPath;
|
|
2964
|
+
} catch {
|
|
2965
|
+
return null;
|
|
2966
|
+
}
|
|
2967
|
+
}
|
|
2797
2968
|
var DEFAULT_BASE_DIR = join2(homedir2(), ".testers", "screenshots");
|
|
2798
2969
|
|
|
2799
2970
|
class Screenshotter {
|
|
@@ -2801,15 +2972,20 @@ class Screenshotter {
|
|
|
2801
2972
|
format;
|
|
2802
2973
|
quality;
|
|
2803
2974
|
fullPage;
|
|
2975
|
+
projectName;
|
|
2976
|
+
runTimestamp;
|
|
2804
2977
|
constructor(options = {}) {
|
|
2805
2978
|
this.baseDir = options.baseDir ?? DEFAULT_BASE_DIR;
|
|
2806
2979
|
this.format = options.format ?? "png";
|
|
2807
2980
|
this.quality = options.quality ?? 90;
|
|
2808
2981
|
this.fullPage = options.fullPage ?? false;
|
|
2982
|
+
this.projectName = options.projectName ?? "default";
|
|
2983
|
+
this.runTimestamp = new Date;
|
|
2809
2984
|
}
|
|
2810
2985
|
async capture(page, options) {
|
|
2811
|
-
const
|
|
2812
|
-
const
|
|
2986
|
+
const action = options.description ?? options.action;
|
|
2987
|
+
const dir = getScreenshotDir(this.baseDir, options.runId, options.scenarioSlug, this.projectName, this.runTimestamp);
|
|
2988
|
+
const filename = generateFilename(options.stepNumber, action);
|
|
2813
2989
|
const filePath = join2(dir, filename);
|
|
2814
2990
|
ensureDir(dir);
|
|
2815
2991
|
await page.screenshot({
|
|
@@ -2819,16 +2995,32 @@ class Screenshotter {
|
|
|
2819
2995
|
quality: this.format === "jpeg" ? this.quality : undefined
|
|
2820
2996
|
});
|
|
2821
2997
|
const viewport = page.viewportSize() ?? { width: 0, height: 0 };
|
|
2998
|
+
const pageUrl = page.url();
|
|
2999
|
+
const timestamp = new Date().toISOString();
|
|
3000
|
+
writeMetaSidecar(filePath, {
|
|
3001
|
+
stepNumber: options.stepNumber,
|
|
3002
|
+
action: options.action,
|
|
3003
|
+
description: options.description ?? null,
|
|
3004
|
+
pageUrl,
|
|
3005
|
+
viewport,
|
|
3006
|
+
timestamp,
|
|
3007
|
+
filePath
|
|
3008
|
+
});
|
|
3009
|
+
const thumbnailPath = await generateThumbnail(page, dir, filename);
|
|
2822
3010
|
return {
|
|
2823
3011
|
filePath,
|
|
2824
3012
|
width: viewport.width,
|
|
2825
3013
|
height: viewport.height,
|
|
2826
|
-
timestamp
|
|
3014
|
+
timestamp,
|
|
3015
|
+
description: options.description ?? null,
|
|
3016
|
+
pageUrl,
|
|
3017
|
+
thumbnailPath
|
|
2827
3018
|
};
|
|
2828
3019
|
}
|
|
2829
3020
|
async captureFullPage(page, options) {
|
|
2830
|
-
const
|
|
2831
|
-
const
|
|
3021
|
+
const action = options.description ?? options.action;
|
|
3022
|
+
const dir = getScreenshotDir(this.baseDir, options.runId, options.scenarioSlug, this.projectName, this.runTimestamp);
|
|
3023
|
+
const filename = generateFilename(options.stepNumber, action);
|
|
2832
3024
|
const filePath = join2(dir, filename);
|
|
2833
3025
|
ensureDir(dir);
|
|
2834
3026
|
await page.screenshot({
|
|
@@ -2838,16 +3030,32 @@ class Screenshotter {
|
|
|
2838
3030
|
quality: this.format === "jpeg" ? this.quality : undefined
|
|
2839
3031
|
});
|
|
2840
3032
|
const viewport = page.viewportSize() ?? { width: 0, height: 0 };
|
|
3033
|
+
const pageUrl = page.url();
|
|
3034
|
+
const timestamp = new Date().toISOString();
|
|
3035
|
+
writeMetaSidecar(filePath, {
|
|
3036
|
+
stepNumber: options.stepNumber,
|
|
3037
|
+
action: options.action,
|
|
3038
|
+
description: options.description ?? null,
|
|
3039
|
+
pageUrl,
|
|
3040
|
+
viewport,
|
|
3041
|
+
timestamp,
|
|
3042
|
+
filePath
|
|
3043
|
+
});
|
|
3044
|
+
const thumbnailPath = await generateThumbnail(page, dir, filename);
|
|
2841
3045
|
return {
|
|
2842
3046
|
filePath,
|
|
2843
3047
|
width: viewport.width,
|
|
2844
3048
|
height: viewport.height,
|
|
2845
|
-
timestamp
|
|
3049
|
+
timestamp,
|
|
3050
|
+
description: options.description ?? null,
|
|
3051
|
+
pageUrl,
|
|
3052
|
+
thumbnailPath
|
|
2846
3053
|
};
|
|
2847
3054
|
}
|
|
2848
3055
|
async captureElement(page, selector, options) {
|
|
2849
|
-
const
|
|
2850
|
-
const
|
|
3056
|
+
const action = options.description ?? options.action;
|
|
3057
|
+
const dir = getScreenshotDir(this.baseDir, options.runId, options.scenarioSlug, this.projectName, this.runTimestamp);
|
|
3058
|
+
const filename = generateFilename(options.stepNumber, action);
|
|
2851
3059
|
const filePath = join2(dir, filename);
|
|
2852
3060
|
ensureDir(dir);
|
|
2853
3061
|
await page.locator(selector).screenshot({
|
|
@@ -2856,16 +3064,31 @@ class Screenshotter {
|
|
|
2856
3064
|
quality: this.format === "jpeg" ? this.quality : undefined
|
|
2857
3065
|
});
|
|
2858
3066
|
const viewport = page.viewportSize() ?? { width: 0, height: 0 };
|
|
3067
|
+
const pageUrl = page.url();
|
|
3068
|
+
const timestamp = new Date().toISOString();
|
|
3069
|
+
writeMetaSidecar(filePath, {
|
|
3070
|
+
stepNumber: options.stepNumber,
|
|
3071
|
+
action: options.action,
|
|
3072
|
+
description: options.description ?? null,
|
|
3073
|
+
pageUrl,
|
|
3074
|
+
viewport,
|
|
3075
|
+
timestamp,
|
|
3076
|
+
filePath
|
|
3077
|
+
});
|
|
2859
3078
|
return {
|
|
2860
3079
|
filePath,
|
|
2861
3080
|
width: viewport.width,
|
|
2862
3081
|
height: viewport.height,
|
|
2863
|
-
timestamp
|
|
3082
|
+
timestamp,
|
|
3083
|
+
description: options.description ?? null,
|
|
3084
|
+
pageUrl,
|
|
3085
|
+
thumbnailPath: null
|
|
2864
3086
|
};
|
|
2865
3087
|
}
|
|
2866
3088
|
}
|
|
2867
3089
|
|
|
2868
3090
|
// src/lib/ai-client.ts
|
|
3091
|
+
init_types();
|
|
2869
3092
|
import Anthropic from "@anthropic-ai/sdk";
|
|
2870
3093
|
function resolveModel(nameOrPreset) {
|
|
2871
3094
|
if (nameOrPreset in MODEL_MAP) {
|
|
@@ -3036,6 +3259,127 @@ var BROWSER_TOOLS = [
|
|
|
3036
3259
|
required: ["text"]
|
|
3037
3260
|
}
|
|
3038
3261
|
},
|
|
3262
|
+
{
|
|
3263
|
+
name: "scroll",
|
|
3264
|
+
description: "Scroll the page up or down by a given amount of pixels.",
|
|
3265
|
+
input_schema: {
|
|
3266
|
+
type: "object",
|
|
3267
|
+
properties: {
|
|
3268
|
+
direction: {
|
|
3269
|
+
type: "string",
|
|
3270
|
+
enum: ["up", "down"],
|
|
3271
|
+
description: "Direction to scroll."
|
|
3272
|
+
},
|
|
3273
|
+
amount: {
|
|
3274
|
+
type: "number",
|
|
3275
|
+
description: "Number of pixels to scroll (default: 500)."
|
|
3276
|
+
}
|
|
3277
|
+
},
|
|
3278
|
+
required: ["direction"]
|
|
3279
|
+
}
|
|
3280
|
+
},
|
|
3281
|
+
{
|
|
3282
|
+
name: "get_page_html",
|
|
3283
|
+
description: "Get simplified HTML of the page body content, truncated to 8000 characters.",
|
|
3284
|
+
input_schema: {
|
|
3285
|
+
type: "object",
|
|
3286
|
+
properties: {},
|
|
3287
|
+
required: []
|
|
3288
|
+
}
|
|
3289
|
+
},
|
|
3290
|
+
{
|
|
3291
|
+
name: "get_elements",
|
|
3292
|
+
description: "List elements matching a CSS selector with their text, tag name, and key attributes (max 20 results).",
|
|
3293
|
+
input_schema: {
|
|
3294
|
+
type: "object",
|
|
3295
|
+
properties: {
|
|
3296
|
+
selector: {
|
|
3297
|
+
type: "string",
|
|
3298
|
+
description: "CSS selector to match elements."
|
|
3299
|
+
}
|
|
3300
|
+
},
|
|
3301
|
+
required: ["selector"]
|
|
3302
|
+
}
|
|
3303
|
+
},
|
|
3304
|
+
{
|
|
3305
|
+
name: "wait_for_navigation",
|
|
3306
|
+
description: "Wait for page navigation/load to complete (network idle).",
|
|
3307
|
+
input_schema: {
|
|
3308
|
+
type: "object",
|
|
3309
|
+
properties: {
|
|
3310
|
+
timeout: {
|
|
3311
|
+
type: "number",
|
|
3312
|
+
description: "Maximum time to wait in milliseconds (default: 10000)."
|
|
3313
|
+
}
|
|
3314
|
+
},
|
|
3315
|
+
required: []
|
|
3316
|
+
}
|
|
3317
|
+
},
|
|
3318
|
+
{
|
|
3319
|
+
name: "get_page_title",
|
|
3320
|
+
description: "Get the document title of the current page.",
|
|
3321
|
+
input_schema: {
|
|
3322
|
+
type: "object",
|
|
3323
|
+
properties: {},
|
|
3324
|
+
required: []
|
|
3325
|
+
}
|
|
3326
|
+
},
|
|
3327
|
+
{
|
|
3328
|
+
name: "count_elements",
|
|
3329
|
+
description: "Count the number of elements matching a CSS selector.",
|
|
3330
|
+
input_schema: {
|
|
3331
|
+
type: "object",
|
|
3332
|
+
properties: {
|
|
3333
|
+
selector: {
|
|
3334
|
+
type: "string",
|
|
3335
|
+
description: "CSS selector to count matching elements."
|
|
3336
|
+
}
|
|
3337
|
+
},
|
|
3338
|
+
required: ["selector"]
|
|
3339
|
+
}
|
|
3340
|
+
},
|
|
3341
|
+
{
|
|
3342
|
+
name: "hover",
|
|
3343
|
+
description: "Hover over an element matching the given CSS selector.",
|
|
3344
|
+
input_schema: {
|
|
3345
|
+
type: "object",
|
|
3346
|
+
properties: {
|
|
3347
|
+
selector: {
|
|
3348
|
+
type: "string",
|
|
3349
|
+
description: "CSS selector of the element to hover over."
|
|
3350
|
+
}
|
|
3351
|
+
},
|
|
3352
|
+
required: ["selector"]
|
|
3353
|
+
}
|
|
3354
|
+
},
|
|
3355
|
+
{
|
|
3356
|
+
name: "check",
|
|
3357
|
+
description: "Check a checkbox matching the given CSS selector.",
|
|
3358
|
+
input_schema: {
|
|
3359
|
+
type: "object",
|
|
3360
|
+
properties: {
|
|
3361
|
+
selector: {
|
|
3362
|
+
type: "string",
|
|
3363
|
+
description: "CSS selector of the checkbox to check."
|
|
3364
|
+
}
|
|
3365
|
+
},
|
|
3366
|
+
required: ["selector"]
|
|
3367
|
+
}
|
|
3368
|
+
},
|
|
3369
|
+
{
|
|
3370
|
+
name: "uncheck",
|
|
3371
|
+
description: "Uncheck a checkbox matching the given CSS selector.",
|
|
3372
|
+
input_schema: {
|
|
3373
|
+
type: "object",
|
|
3374
|
+
properties: {
|
|
3375
|
+
selector: {
|
|
3376
|
+
type: "string",
|
|
3377
|
+
description: "CSS selector of the checkbox to uncheck."
|
|
3378
|
+
}
|
|
3379
|
+
},
|
|
3380
|
+
required: ["selector"]
|
|
3381
|
+
}
|
|
3382
|
+
},
|
|
3039
3383
|
{
|
|
3040
3384
|
name: "report_result",
|
|
3041
3385
|
description: "Report the final test result. Call this when you have completed testing the scenario. This MUST be the last tool you call.",
|
|
@@ -3167,6 +3511,113 @@ async function executeTool(page, screenshotter, toolName, toolInput, context) {
|
|
|
3167
3511
|
return { result: "false" };
|
|
3168
3512
|
}
|
|
3169
3513
|
}
|
|
3514
|
+
case "scroll": {
|
|
3515
|
+
const direction = toolInput.direction;
|
|
3516
|
+
const amount = typeof toolInput.amount === "number" ? toolInput.amount : 500;
|
|
3517
|
+
const scrollY = direction === "down" ? amount : -amount;
|
|
3518
|
+
await page.evaluate((y) => window.scrollBy(0, y), scrollY);
|
|
3519
|
+
const screenshot = await screenshotter.capture(page, {
|
|
3520
|
+
runId: context.runId,
|
|
3521
|
+
scenarioSlug: context.scenarioSlug,
|
|
3522
|
+
stepNumber: context.stepNumber,
|
|
3523
|
+
action: "scroll"
|
|
3524
|
+
});
|
|
3525
|
+
return {
|
|
3526
|
+
result: `Scrolled ${direction} by ${amount}px`,
|
|
3527
|
+
screenshot
|
|
3528
|
+
};
|
|
3529
|
+
}
|
|
3530
|
+
case "get_page_html": {
|
|
3531
|
+
const html = await page.evaluate(() => document.body.innerHTML);
|
|
3532
|
+
const truncated = html.length > 8000 ? html.slice(0, 8000) + "..." : html;
|
|
3533
|
+
return {
|
|
3534
|
+
result: truncated
|
|
3535
|
+
};
|
|
3536
|
+
}
|
|
3537
|
+
case "get_elements": {
|
|
3538
|
+
const selector = toolInput.selector;
|
|
3539
|
+
const allElements = await page.locator(selector).all();
|
|
3540
|
+
const elements = allElements.slice(0, 20);
|
|
3541
|
+
const results = [];
|
|
3542
|
+
for (let i = 0;i < elements.length; i++) {
|
|
3543
|
+
const el = elements[i];
|
|
3544
|
+
const tagName = await el.evaluate((e) => e.tagName.toLowerCase());
|
|
3545
|
+
const textContent = await el.textContent() ?? "";
|
|
3546
|
+
const trimmedText = textContent.trim().slice(0, 100);
|
|
3547
|
+
const id = await el.getAttribute("id");
|
|
3548
|
+
const className = await el.getAttribute("class");
|
|
3549
|
+
const href = await el.getAttribute("href");
|
|
3550
|
+
const type = await el.getAttribute("type");
|
|
3551
|
+
const placeholder = await el.getAttribute("placeholder");
|
|
3552
|
+
const ariaLabel = await el.getAttribute("aria-label");
|
|
3553
|
+
const attrs = [];
|
|
3554
|
+
if (id)
|
|
3555
|
+
attrs.push(`id="${id}"`);
|
|
3556
|
+
if (className)
|
|
3557
|
+
attrs.push(`class="${className}"`);
|
|
3558
|
+
if (href)
|
|
3559
|
+
attrs.push(`href="${href}"`);
|
|
3560
|
+
if (type)
|
|
3561
|
+
attrs.push(`type="${type}"`);
|
|
3562
|
+
if (placeholder)
|
|
3563
|
+
attrs.push(`placeholder="${placeholder}"`);
|
|
3564
|
+
if (ariaLabel)
|
|
3565
|
+
attrs.push(`aria-label="${ariaLabel}"`);
|
|
3566
|
+
results.push(`[${i}] <${tagName}${attrs.length ? " " + attrs.join(" ") : ""}> ${trimmedText}`);
|
|
3567
|
+
}
|
|
3568
|
+
return {
|
|
3569
|
+
result: results.length > 0 ? results.join(`
|
|
3570
|
+
`) : `No elements found matching "${selector}"`
|
|
3571
|
+
};
|
|
3572
|
+
}
|
|
3573
|
+
case "wait_for_navigation": {
|
|
3574
|
+
const timeout = typeof toolInput.timeout === "number" ? toolInput.timeout : 1e4;
|
|
3575
|
+
await page.waitForLoadState("networkidle", { timeout });
|
|
3576
|
+
return {
|
|
3577
|
+
result: "Navigation/load completed"
|
|
3578
|
+
};
|
|
3579
|
+
}
|
|
3580
|
+
case "get_page_title": {
|
|
3581
|
+
const title = await page.title();
|
|
3582
|
+
return {
|
|
3583
|
+
result: title || "(no title)"
|
|
3584
|
+
};
|
|
3585
|
+
}
|
|
3586
|
+
case "count_elements": {
|
|
3587
|
+
const selector = toolInput.selector;
|
|
3588
|
+
const count = await page.locator(selector).count();
|
|
3589
|
+
return {
|
|
3590
|
+
result: `${count} element(s) matching "${selector}"`
|
|
3591
|
+
};
|
|
3592
|
+
}
|
|
3593
|
+
case "hover": {
|
|
3594
|
+
const selector = toolInput.selector;
|
|
3595
|
+
await page.hover(selector);
|
|
3596
|
+
const screenshot = await screenshotter.capture(page, {
|
|
3597
|
+
runId: context.runId,
|
|
3598
|
+
scenarioSlug: context.scenarioSlug,
|
|
3599
|
+
stepNumber: context.stepNumber,
|
|
3600
|
+
action: "hover"
|
|
3601
|
+
});
|
|
3602
|
+
return {
|
|
3603
|
+
result: `Hovered over: ${selector}`,
|
|
3604
|
+
screenshot
|
|
3605
|
+
};
|
|
3606
|
+
}
|
|
3607
|
+
case "check": {
|
|
3608
|
+
const selector = toolInput.selector;
|
|
3609
|
+
await page.check(selector);
|
|
3610
|
+
return {
|
|
3611
|
+
result: `Checked checkbox: ${selector}`
|
|
3612
|
+
};
|
|
3613
|
+
}
|
|
3614
|
+
case "uncheck": {
|
|
3615
|
+
const selector = toolInput.selector;
|
|
3616
|
+
await page.uncheck(selector);
|
|
3617
|
+
return {
|
|
3618
|
+
result: `Unchecked checkbox: ${selector}`
|
|
3619
|
+
};
|
|
3620
|
+
}
|
|
3170
3621
|
case "report_result": {
|
|
3171
3622
|
const status = toolInput.status;
|
|
3172
3623
|
const reasoning = toolInput.reasoning;
|
|
@@ -3193,13 +3644,26 @@ async function runAgentLoop(options) {
|
|
|
3193
3644
|
maxTurns = 30
|
|
3194
3645
|
} = options;
|
|
3195
3646
|
const systemPrompt = [
|
|
3196
|
-
"You are
|
|
3197
|
-
"
|
|
3198
|
-
"
|
|
3199
|
-
"
|
|
3200
|
-
"
|
|
3201
|
-
"
|
|
3202
|
-
|
|
3647
|
+
"You are an expert QA testing agent. Your job is to thoroughly test web application scenarios.",
|
|
3648
|
+
"You have browser tools to navigate, interact with, and inspect web pages.",
|
|
3649
|
+
"",
|
|
3650
|
+
"Strategy:",
|
|
3651
|
+
"1. First navigate to the target page and take a screenshot to understand the layout",
|
|
3652
|
+
"2. If you can't find an element, use get_elements or get_page_html to discover selectors",
|
|
3653
|
+
"3. Use scroll to discover content below the fold",
|
|
3654
|
+
"4. Use wait_for or wait_for_navigation after actions that trigger page loads",
|
|
3655
|
+
"5. Take screenshots after every meaningful state change",
|
|
3656
|
+
"6. Use assert_text and assert_visible to verify expected outcomes",
|
|
3657
|
+
"7. When done testing, call report_result with detailed pass/fail reasoning",
|
|
3658
|
+
"",
|
|
3659
|
+
"Tips:",
|
|
3660
|
+
"- Try multiple selector strategies: by text, by role, by class, by id",
|
|
3661
|
+
"- If a click triggers navigation, use wait_for_navigation after",
|
|
3662
|
+
"- For forms, fill all fields before submitting",
|
|
3663
|
+
"- Check for error messages after form submissions",
|
|
3664
|
+
"- Verify both positive and negative states"
|
|
3665
|
+
].join(`
|
|
3666
|
+
`);
|
|
3203
3667
|
const userParts = [
|
|
3204
3668
|
`**Scenario:** ${scenario.name}`,
|
|
3205
3669
|
`**Description:** ${scenario.description}`
|
|
@@ -3303,6 +3767,7 @@ function createClient(apiKey) {
|
|
|
3303
3767
|
}
|
|
3304
3768
|
|
|
3305
3769
|
// src/lib/config.ts
|
|
3770
|
+
init_types();
|
|
3306
3771
|
import { homedir as homedir3 } from "os";
|
|
3307
3772
|
import { join as join3 } from "path";
|
|
3308
3773
|
import { readFileSync, existsSync as existsSync3 } from "fs";
|
|
@@ -3402,7 +3867,10 @@ async function runSingleScenario(scenario, runId, options) {
|
|
|
3402
3867
|
action: ss.action,
|
|
3403
3868
|
filePath: ss.filePath,
|
|
3404
3869
|
width: ss.width,
|
|
3405
|
-
height: ss.height
|
|
3870
|
+
height: ss.height,
|
|
3871
|
+
description: ss.description,
|
|
3872
|
+
pageUrl: ss.pageUrl,
|
|
3873
|
+
thumbnailPath: ss.thumbnailPath
|
|
3406
3874
|
});
|
|
3407
3875
|
emit({ type: "screenshot:captured", screenshotPath: ss.filePath, scenarioId: scenario.id, runId });
|
|
3408
3876
|
}
|
|
@@ -3669,6 +4137,7 @@ import { Database as Database2 } from "bun:sqlite";
|
|
|
3669
4137
|
import { existsSync as existsSync4 } from "fs";
|
|
3670
4138
|
import { join as join4 } from "path";
|
|
3671
4139
|
import { homedir as homedir4 } from "os";
|
|
4140
|
+
init_types();
|
|
3672
4141
|
function resolveTodosDbPath() {
|
|
3673
4142
|
const envPath = process.env["TODOS_DB_PATH"];
|
|
3674
4143
|
if (envPath)
|
|
@@ -3765,17 +4234,1099 @@ function importFromTodos(options = {}) {
|
|
|
3765
4234
|
return { imported, skipped };
|
|
3766
4235
|
}
|
|
3767
4236
|
|
|
4237
|
+
// src/lib/init.ts
|
|
4238
|
+
import { existsSync as existsSync5, readFileSync as readFileSync2, writeFileSync as writeFileSync2, mkdirSync as mkdirSync3 } from "fs";
|
|
4239
|
+
import { join as join5, basename } from "path";
|
|
4240
|
+
import { homedir as homedir5 } from "os";
|
|
4241
|
+
|
|
4242
|
+
// src/db/projects.ts
|
|
4243
|
+
init_types();
|
|
4244
|
+
init_database();
|
|
4245
|
+
function createProject(input) {
|
|
4246
|
+
const db2 = getDatabase();
|
|
4247
|
+
const id = uuid();
|
|
4248
|
+
const timestamp = now();
|
|
4249
|
+
db2.query(`
|
|
4250
|
+
INSERT INTO projects (id, name, path, description, created_at, updated_at)
|
|
4251
|
+
VALUES (?, ?, ?, ?, ?, ?)
|
|
4252
|
+
`).run(id, input.name, input.path ?? null, input.description ?? null, timestamp, timestamp);
|
|
4253
|
+
return getProject(id);
|
|
4254
|
+
}
|
|
4255
|
+
function getProject(id) {
|
|
4256
|
+
const db2 = getDatabase();
|
|
4257
|
+
const row = db2.query("SELECT * FROM projects WHERE id = ?").get(id);
|
|
4258
|
+
return row ? projectFromRow(row) : null;
|
|
4259
|
+
}
|
|
4260
|
+
function listProjects() {
|
|
4261
|
+
const db2 = getDatabase();
|
|
4262
|
+
const rows = db2.query("SELECT * FROM projects ORDER BY created_at DESC").all();
|
|
4263
|
+
return rows.map(projectFromRow);
|
|
4264
|
+
}
|
|
4265
|
+
function ensureProject(name, path) {
|
|
4266
|
+
const db2 = getDatabase();
|
|
4267
|
+
const byPath = db2.query("SELECT * FROM projects WHERE path = ?").get(path);
|
|
4268
|
+
if (byPath)
|
|
4269
|
+
return projectFromRow(byPath);
|
|
4270
|
+
const byName = db2.query("SELECT * FROM projects WHERE name = ?").get(name);
|
|
4271
|
+
if (byName)
|
|
4272
|
+
return projectFromRow(byName);
|
|
4273
|
+
return createProject({ name, path });
|
|
4274
|
+
}
|
|
4275
|
+
|
|
4276
|
+
// src/lib/init.ts
|
|
4277
|
+
function detectFramework(dir) {
|
|
4278
|
+
const pkgPath = join5(dir, "package.json");
|
|
4279
|
+
if (!existsSync5(pkgPath))
|
|
4280
|
+
return null;
|
|
4281
|
+
let pkg;
|
|
4282
|
+
try {
|
|
4283
|
+
pkg = JSON.parse(readFileSync2(pkgPath, "utf-8"));
|
|
4284
|
+
} catch {
|
|
4285
|
+
return null;
|
|
4286
|
+
}
|
|
4287
|
+
const allDeps = { ...pkg.dependencies, ...pkg.devDependencies };
|
|
4288
|
+
const depNames = Object.keys(allDeps);
|
|
4289
|
+
const features = [];
|
|
4290
|
+
const hasAuth = depNames.some((d) => d === "next-auth" || d.startsWith("@auth/") || d === "passport" || d === "lucia");
|
|
4291
|
+
if (hasAuth)
|
|
4292
|
+
features.push("hasAuth");
|
|
4293
|
+
const hasForms = depNames.some((d) => d === "react-hook-form" || d === "formik" || d === "zod");
|
|
4294
|
+
if (hasForms)
|
|
4295
|
+
features.push("hasForms");
|
|
4296
|
+
if ("next" in allDeps) {
|
|
4297
|
+
return { name: "Next.js", defaultUrl: "http://localhost:3000", features };
|
|
4298
|
+
}
|
|
4299
|
+
if ("vite" in allDeps) {
|
|
4300
|
+
return { name: "Vite", defaultUrl: "http://localhost:5173", features };
|
|
4301
|
+
}
|
|
4302
|
+
if (depNames.some((d) => d.startsWith("@remix-run"))) {
|
|
4303
|
+
return { name: "Remix", defaultUrl: "http://localhost:3000", features };
|
|
4304
|
+
}
|
|
4305
|
+
if ("nuxt" in allDeps) {
|
|
4306
|
+
return { name: "Nuxt", defaultUrl: "http://localhost:3000", features };
|
|
4307
|
+
}
|
|
4308
|
+
if (depNames.some((d) => d.startsWith("svelte") || d === "@sveltejs/kit")) {
|
|
4309
|
+
return { name: "SvelteKit", defaultUrl: "http://localhost:5173", features };
|
|
4310
|
+
}
|
|
4311
|
+
if (depNames.some((d) => d.startsWith("@angular"))) {
|
|
4312
|
+
return { name: "Angular", defaultUrl: "http://localhost:4200", features };
|
|
4313
|
+
}
|
|
4314
|
+
if ("express" in allDeps) {
|
|
4315
|
+
return { name: "Express", defaultUrl: "http://localhost:3000", features };
|
|
4316
|
+
}
|
|
4317
|
+
return null;
|
|
4318
|
+
}
|
|
4319
|
+
function getStarterScenarios(framework, projectId) {
|
|
4320
|
+
const scenarios = [
|
|
4321
|
+
{
|
|
4322
|
+
name: "Landing page loads",
|
|
4323
|
+
description: "Navigate to the landing page and verify it loads correctly with no console errors. Check that the main heading, navigation, and primary CTA are visible.",
|
|
4324
|
+
tags: ["smoke"],
|
|
4325
|
+
priority: "high",
|
|
4326
|
+
projectId
|
|
4327
|
+
},
|
|
4328
|
+
{
|
|
4329
|
+
name: "Navigation works",
|
|
4330
|
+
description: "Click through main navigation links and verify each page loads without errors.",
|
|
4331
|
+
tags: ["smoke"],
|
|
4332
|
+
priority: "medium",
|
|
4333
|
+
projectId
|
|
4334
|
+
},
|
|
4335
|
+
{
|
|
4336
|
+
name: "No console errors",
|
|
4337
|
+
description: "Navigate through the main pages and check the browser console for any JavaScript errors or warnings.",
|
|
4338
|
+
tags: ["smoke"],
|
|
4339
|
+
priority: "high",
|
|
4340
|
+
projectId
|
|
4341
|
+
}
|
|
4342
|
+
];
|
|
4343
|
+
if (framework.features.includes("hasAuth")) {
|
|
4344
|
+
scenarios.push({
|
|
4345
|
+
name: "Login flow",
|
|
4346
|
+
description: "Navigate to the login page, enter valid credentials, and verify successful authentication and redirect.",
|
|
4347
|
+
tags: ["auth"],
|
|
4348
|
+
priority: "critical",
|
|
4349
|
+
projectId
|
|
4350
|
+
}, {
|
|
4351
|
+
name: "Signup flow",
|
|
4352
|
+
description: "Navigate to the signup page, fill in registration details, and verify account creation succeeds.",
|
|
4353
|
+
tags: ["auth"],
|
|
4354
|
+
priority: "medium",
|
|
4355
|
+
projectId
|
|
4356
|
+
});
|
|
4357
|
+
}
|
|
4358
|
+
if (framework.features.includes("hasForms")) {
|
|
4359
|
+
scenarios.push({
|
|
4360
|
+
name: "Form validation",
|
|
4361
|
+
description: "Submit forms with empty/invalid data and verify validation errors appear correctly.",
|
|
4362
|
+
tags: ["forms"],
|
|
4363
|
+
priority: "medium",
|
|
4364
|
+
projectId
|
|
4365
|
+
});
|
|
4366
|
+
}
|
|
4367
|
+
return scenarios;
|
|
4368
|
+
}
|
|
4369
|
+
function initProject(options) {
|
|
4370
|
+
const dir = options.dir ?? process.cwd();
|
|
4371
|
+
const name = options.name ?? basename(dir);
|
|
4372
|
+
const framework = detectFramework(dir);
|
|
4373
|
+
const url = options.url ?? framework?.defaultUrl ?? "http://localhost:3000";
|
|
4374
|
+
const projectPath = options.path ?? dir;
|
|
4375
|
+
const project = ensureProject(name, projectPath);
|
|
4376
|
+
const starterInputs = getStarterScenarios(framework ?? { name: "Unknown", features: [] }, project.id);
|
|
4377
|
+
const scenarios = starterInputs.map((input) => createScenario(input));
|
|
4378
|
+
const configDir = join5(homedir5(), ".testers");
|
|
4379
|
+
const configPath = join5(configDir, "config.json");
|
|
4380
|
+
if (!existsSync5(configDir)) {
|
|
4381
|
+
mkdirSync3(configDir, { recursive: true });
|
|
4382
|
+
}
|
|
4383
|
+
let config = {};
|
|
4384
|
+
if (existsSync5(configPath)) {
|
|
4385
|
+
try {
|
|
4386
|
+
config = JSON.parse(readFileSync2(configPath, "utf-8"));
|
|
4387
|
+
} catch {}
|
|
4388
|
+
}
|
|
4389
|
+
config.activeProject = project.id;
|
|
4390
|
+
writeFileSync2(configPath, JSON.stringify(config, null, 2), "utf-8");
|
|
4391
|
+
return { project, scenarios, framework, url };
|
|
4392
|
+
}
|
|
4393
|
+
|
|
4394
|
+
// src/lib/smoke.ts
|
|
4395
|
+
init_runs();
|
|
4396
|
+
var SMOKE_DESCRIPTION = `You are performing an autonomous smoke test of this web application. Your job is to explore as many pages as possible and find issues. Follow these instructions:
|
|
4397
|
+
|
|
4398
|
+
1. Start at the given URL and take a screenshot
|
|
4399
|
+
2. Find all visible navigation links and click through each one
|
|
4400
|
+
3. On each page: check for visible error messages, broken layouts, missing images
|
|
4401
|
+
4. Use get_page_html to check for error indicators in the HTML
|
|
4402
|
+
5. Try clicking the main interactive elements (buttons, links, forms)
|
|
4403
|
+
6. Keep track of every page you visit
|
|
4404
|
+
7. After exploring at least 5 different pages (or all available pages), report your findings
|
|
4405
|
+
|
|
4406
|
+
In your report_result, include:
|
|
4407
|
+
- Total pages visited
|
|
4408
|
+
- Any JavaScript errors you noticed
|
|
4409
|
+
- Any broken links (pages that show 404 or error)
|
|
4410
|
+
- Any visual issues (broken layouts, missing images, overlapping text)
|
|
4411
|
+
- Any forms that don't work
|
|
4412
|
+
- Rate each issue as critical/high/medium/low severity`;
|
|
4413
|
+
async function runSmoke(options) {
|
|
4414
|
+
const config = loadConfig();
|
|
4415
|
+
const model = resolveModel(options.model ?? config.defaultModel);
|
|
4416
|
+
const scenario = createScenario({
|
|
4417
|
+
name: "Smoke Test",
|
|
4418
|
+
description: SMOKE_DESCRIPTION,
|
|
4419
|
+
tags: ["smoke", "auto"],
|
|
4420
|
+
priority: "high",
|
|
4421
|
+
projectId: options.projectId
|
|
4422
|
+
});
|
|
4423
|
+
const run = createRun({
|
|
4424
|
+
url: options.url,
|
|
4425
|
+
model,
|
|
4426
|
+
headed: options.headed,
|
|
4427
|
+
parallel: 1,
|
|
4428
|
+
projectId: options.projectId
|
|
4429
|
+
});
|
|
4430
|
+
updateRun(run.id, { status: "running", total: 1 });
|
|
4431
|
+
let result;
|
|
4432
|
+
try {
|
|
4433
|
+
result = await runSingleScenario(scenario, run.id, {
|
|
4434
|
+
url: options.url,
|
|
4435
|
+
model: options.model,
|
|
4436
|
+
headed: options.headed,
|
|
4437
|
+
timeout: options.timeout,
|
|
4438
|
+
projectId: options.projectId,
|
|
4439
|
+
apiKey: options.apiKey
|
|
4440
|
+
});
|
|
4441
|
+
const finalStatus = result.status === "passed" ? "passed" : "failed";
|
|
4442
|
+
updateRun(run.id, {
|
|
4443
|
+
status: finalStatus,
|
|
4444
|
+
passed: result.status === "passed" ? 1 : 0,
|
|
4445
|
+
failed: result.status === "passed" ? 0 : 1,
|
|
4446
|
+
total: 1,
|
|
4447
|
+
finished_at: new Date().toISOString()
|
|
4448
|
+
});
|
|
4449
|
+
} catch (error) {
|
|
4450
|
+
updateRun(run.id, {
|
|
4451
|
+
status: "failed",
|
|
4452
|
+
failed: 1,
|
|
4453
|
+
total: 1,
|
|
4454
|
+
finished_at: new Date().toISOString()
|
|
4455
|
+
});
|
|
4456
|
+
throw error;
|
|
4457
|
+
} finally {
|
|
4458
|
+
deleteScenario(scenario.id);
|
|
4459
|
+
}
|
|
4460
|
+
const issues = parseSmokeIssues(result.reasoning ?? "");
|
|
4461
|
+
const pagesVisited = extractPagesVisited(result.reasoning ?? "");
|
|
4462
|
+
const { getRun: getRun2 } = await Promise.resolve().then(() => (init_runs(), exports_runs));
|
|
4463
|
+
const finalRun = getRun2(run.id);
|
|
4464
|
+
return {
|
|
4465
|
+
run: finalRun,
|
|
4466
|
+
result,
|
|
4467
|
+
pagesVisited,
|
|
4468
|
+
issuesFound: issues
|
|
4469
|
+
};
|
|
4470
|
+
}
|
|
4471
|
+
var SEVERITY_PATTERN = /\b(CRITICAL|HIGH|MEDIUM|LOW)\b[:\s-]*(.+)/gi;
|
|
4472
|
+
var PAGES_VISITED_PATTERN = /(\d+)\s*(?:pages?\s*visited|pages?\s*explored|pages?\s*checked|total\s*pages?)/i;
|
|
4473
|
+
var URL_PATTERN = /https?:\/\/[^\s,)]+/g;
|
|
4474
|
+
var ISSUE_TYPE_MAP = {
|
|
4475
|
+
javascript: "js-error",
|
|
4476
|
+
"js error": "js-error",
|
|
4477
|
+
"js-error": "js-error",
|
|
4478
|
+
"console error": "js-error",
|
|
4479
|
+
"404": "404",
|
|
4480
|
+
"not found": "404",
|
|
4481
|
+
"broken link": "broken-link",
|
|
4482
|
+
"dead link": "broken-link",
|
|
4483
|
+
"broken image": "broken-image",
|
|
4484
|
+
"missing image": "broken-image",
|
|
4485
|
+
visual: "visual",
|
|
4486
|
+
layout: "visual",
|
|
4487
|
+
overlap: "visual",
|
|
4488
|
+
"broken layout": "visual",
|
|
4489
|
+
performance: "performance",
|
|
4490
|
+
slow: "performance"
|
|
4491
|
+
};
|
|
4492
|
+
function inferIssueType(text) {
|
|
4493
|
+
const lower = text.toLowerCase();
|
|
4494
|
+
for (const [keyword, type] of Object.entries(ISSUE_TYPE_MAP)) {
|
|
4495
|
+
if (lower.includes(keyword))
|
|
4496
|
+
return type;
|
|
4497
|
+
}
|
|
4498
|
+
return "visual";
|
|
4499
|
+
}
|
|
4500
|
+
function extractUrl(text, fallback = "") {
|
|
4501
|
+
const match = text.match(URL_PATTERN);
|
|
4502
|
+
return match ? match[0] : fallback;
|
|
4503
|
+
}
|
|
4504
|
+
function parseSmokeIssues(reasoning) {
|
|
4505
|
+
const issues = [];
|
|
4506
|
+
const seen = new Set;
|
|
4507
|
+
let match;
|
|
4508
|
+
const severityRegex = new RegExp(SEVERITY_PATTERN.source, "gi");
|
|
4509
|
+
while ((match = severityRegex.exec(reasoning)) !== null) {
|
|
4510
|
+
const severity = match[1].toLowerCase();
|
|
4511
|
+
const description = match[2].trim();
|
|
4512
|
+
const key = `${severity}:${description.slice(0, 80)}`;
|
|
4513
|
+
if (seen.has(key))
|
|
4514
|
+
continue;
|
|
4515
|
+
seen.add(key);
|
|
4516
|
+
issues.push({
|
|
4517
|
+
type: inferIssueType(description),
|
|
4518
|
+
severity,
|
|
4519
|
+
description,
|
|
4520
|
+
url: extractUrl(description)
|
|
4521
|
+
});
|
|
4522
|
+
}
|
|
4523
|
+
const bulletLines = reasoning.split(`
|
|
4524
|
+
`).filter((line) => /^\s*[-*]\s/.test(line) && /\b(error|broken|missing|404|fail|issue|bug|problem)\b/i.test(line));
|
|
4525
|
+
for (const line of bulletLines) {
|
|
4526
|
+
const cleaned = line.replace(/^\s*[-*]\s*/, "").trim();
|
|
4527
|
+
const key = `bullet:${cleaned.slice(0, 80)}`;
|
|
4528
|
+
if (seen.has(key))
|
|
4529
|
+
continue;
|
|
4530
|
+
seen.add(key);
|
|
4531
|
+
let severity = "medium";
|
|
4532
|
+
if (/\bcritical\b/i.test(cleaned))
|
|
4533
|
+
severity = "critical";
|
|
4534
|
+
else if (/\bhigh\b/i.test(cleaned))
|
|
4535
|
+
severity = "high";
|
|
4536
|
+
else if (/\blow\b/i.test(cleaned))
|
|
4537
|
+
severity = "low";
|
|
4538
|
+
else if (/\b(error|fail|broken|crash)\b/i.test(cleaned))
|
|
4539
|
+
severity = "high";
|
|
4540
|
+
issues.push({
|
|
4541
|
+
type: inferIssueType(cleaned),
|
|
4542
|
+
severity,
|
|
4543
|
+
description: cleaned,
|
|
4544
|
+
url: extractUrl(cleaned)
|
|
4545
|
+
});
|
|
4546
|
+
}
|
|
4547
|
+
return issues;
|
|
4548
|
+
}
|
|
4549
|
+
function extractPagesVisited(reasoning) {
|
|
4550
|
+
const match = reasoning.match(PAGES_VISITED_PATTERN);
|
|
4551
|
+
if (match)
|
|
4552
|
+
return parseInt(match[1], 10);
|
|
4553
|
+
const urls = reasoning.match(URL_PATTERN);
|
|
4554
|
+
if (urls) {
|
|
4555
|
+
const unique = new Set(urls.map((u) => new URL(u).pathname));
|
|
4556
|
+
return unique.size;
|
|
4557
|
+
}
|
|
4558
|
+
return 0;
|
|
4559
|
+
}
|
|
4560
|
+
var SEVERITY_COLORS = {
|
|
4561
|
+
critical: (t) => `\x1B[41m\x1B[37m ${t} \x1B[0m`,
|
|
4562
|
+
high: (t) => `\x1B[31m${t}\x1B[0m`,
|
|
4563
|
+
medium: (t) => `\x1B[33m${t}\x1B[0m`,
|
|
4564
|
+
low: (t) => `\x1B[36m${t}\x1B[0m`
|
|
4565
|
+
};
|
|
4566
|
+
var SEVERITY_ORDER = ["critical", "high", "medium", "low"];
|
|
4567
|
+
function formatSmokeReport(result) {
|
|
4568
|
+
const lines = [];
|
|
4569
|
+
const url = result.run.url;
|
|
4570
|
+
lines.push("");
|
|
4571
|
+
lines.push(`\x1B[1m Smoke Test Report \x1B[2m- ${url}\x1B[0m`);
|
|
4572
|
+
lines.push(` ${"\u2500".repeat(60)}`);
|
|
4573
|
+
const issueCount = result.issuesFound.length;
|
|
4574
|
+
const criticalCount = result.issuesFound.filter((i) => i.severity === "critical").length;
|
|
4575
|
+
const highCount = result.issuesFound.filter((i) => i.severity === "high").length;
|
|
4576
|
+
lines.push("");
|
|
4577
|
+
lines.push(` Pages visited: \x1B[1m${result.pagesVisited}\x1B[0m`);
|
|
4578
|
+
lines.push(` Issues found: \x1B[1m${issueCount}\x1B[0m`);
|
|
4579
|
+
lines.push(` Duration: ${result.result.durationMs ? `${(result.result.durationMs / 1000).toFixed(1)}s` : "N/A"}`);
|
|
4580
|
+
lines.push(` Model: ${result.run.model}`);
|
|
4581
|
+
lines.push(` Tokens used: ${result.result.tokensUsed}`);
|
|
4582
|
+
if (issueCount > 0) {
|
|
4583
|
+
lines.push("");
|
|
4584
|
+
lines.push(`\x1B[1m Issues\x1B[0m`);
|
|
4585
|
+
lines.push("");
|
|
4586
|
+
for (const severity of SEVERITY_ORDER) {
|
|
4587
|
+
const group = result.issuesFound.filter((i) => i.severity === severity);
|
|
4588
|
+
if (group.length === 0)
|
|
4589
|
+
continue;
|
|
4590
|
+
const badge = SEVERITY_COLORS[severity](severity.toUpperCase());
|
|
4591
|
+
lines.push(` ${badge}`);
|
|
4592
|
+
for (const issue of group) {
|
|
4593
|
+
const urlSuffix = issue.url ? ` \x1B[2m(${issue.url})\x1B[0m` : "";
|
|
4594
|
+
lines.push(` - ${issue.description}${urlSuffix}`);
|
|
4595
|
+
}
|
|
4596
|
+
lines.push("");
|
|
4597
|
+
}
|
|
4598
|
+
}
|
|
4599
|
+
lines.push(` ${"\u2500".repeat(60)}`);
|
|
4600
|
+
const hasCritical = criticalCount > 0 || highCount > 0;
|
|
4601
|
+
if (hasCritical) {
|
|
4602
|
+
lines.push(` Verdict: \x1B[31m\x1B[1mFAIL\x1B[0m \x1B[2m(${criticalCount} critical, ${highCount} high severity issues)\x1B[0m`);
|
|
4603
|
+
} else if (issueCount > 0) {
|
|
4604
|
+
lines.push(` Verdict: \x1B[33m\x1B[1mWARN\x1B[0m \x1B[2m(${issueCount} issues found, none critical/high)\x1B[0m`);
|
|
4605
|
+
} else {
|
|
4606
|
+
lines.push(` Verdict: \x1B[32m\x1B[1mPASS\x1B[0m \x1B[2m(no issues found)\x1B[0m`);
|
|
4607
|
+
}
|
|
4608
|
+
lines.push("");
|
|
4609
|
+
return lines.join(`
|
|
4610
|
+
`);
|
|
4611
|
+
}
|
|
4612
|
+
|
|
4613
|
+
// src/lib/diff.ts
|
|
4614
|
+
init_runs();
|
|
4615
|
+
import chalk2 from "chalk";
|
|
4616
|
+
function diffRuns(runId1, runId2) {
|
|
4617
|
+
const run1 = getRun(runId1);
|
|
4618
|
+
if (!run1) {
|
|
4619
|
+
throw new Error(`Run not found: ${runId1}`);
|
|
4620
|
+
}
|
|
4621
|
+
const run2 = getRun(runId2);
|
|
4622
|
+
if (!run2) {
|
|
4623
|
+
throw new Error(`Run not found: ${runId2}`);
|
|
4624
|
+
}
|
|
4625
|
+
const results1 = getResultsByRun(run1.id);
|
|
4626
|
+
const results2 = getResultsByRun(run2.id);
|
|
4627
|
+
const map1 = new Map;
|
|
4628
|
+
for (const r of results1) {
|
|
4629
|
+
map1.set(r.scenarioId, r);
|
|
4630
|
+
}
|
|
4631
|
+
const map2 = new Map;
|
|
4632
|
+
for (const r of results2) {
|
|
4633
|
+
map2.set(r.scenarioId, r);
|
|
4634
|
+
}
|
|
4635
|
+
const allScenarioIds = new Set([...map1.keys(), ...map2.keys()]);
|
|
4636
|
+
const regressions = [];
|
|
4637
|
+
const fixes = [];
|
|
4638
|
+
const unchanged = [];
|
|
4639
|
+
const newScenarios = [];
|
|
4640
|
+
const removedScenarios = [];
|
|
4641
|
+
for (const scenarioId of allScenarioIds) {
|
|
4642
|
+
const r1 = map1.get(scenarioId) ?? null;
|
|
4643
|
+
const r2 = map2.get(scenarioId) ?? null;
|
|
4644
|
+
const scenario = getScenario(scenarioId);
|
|
4645
|
+
const diff = {
|
|
4646
|
+
scenarioId,
|
|
4647
|
+
scenarioName: scenario?.name ?? null,
|
|
4648
|
+
scenarioShortId: scenario?.shortId ?? null,
|
|
4649
|
+
status1: r1?.status ?? null,
|
|
4650
|
+
status2: r2?.status ?? null,
|
|
4651
|
+
duration1: r1?.durationMs ?? null,
|
|
4652
|
+
duration2: r2?.durationMs ?? null,
|
|
4653
|
+
tokens1: r1?.tokensUsed ?? null,
|
|
4654
|
+
tokens2: r2?.tokensUsed ?? null
|
|
4655
|
+
};
|
|
4656
|
+
if (!r1 && r2) {
|
|
4657
|
+
newScenarios.push(diff);
|
|
4658
|
+
} else if (r1 && !r2) {
|
|
4659
|
+
removedScenarios.push(diff);
|
|
4660
|
+
} else if (r1 && r2) {
|
|
4661
|
+
const wasPass = r1.status === "passed";
|
|
4662
|
+
const nowPass = r2.status === "passed";
|
|
4663
|
+
const wasFail = r1.status === "failed" || r1.status === "error";
|
|
4664
|
+
const nowFail = r2.status === "failed" || r2.status === "error";
|
|
4665
|
+
if (wasPass && nowFail) {
|
|
4666
|
+
regressions.push(diff);
|
|
4667
|
+
} else if (wasFail && nowPass) {
|
|
4668
|
+
fixes.push(diff);
|
|
4669
|
+
} else {
|
|
4670
|
+
unchanged.push(diff);
|
|
4671
|
+
}
|
|
4672
|
+
}
|
|
4673
|
+
}
|
|
4674
|
+
return { run1, run2, regressions, fixes, unchanged, newScenarios, removedScenarios };
|
|
4675
|
+
}
|
|
4676
|
+
function formatScenarioLabel(diff) {
|
|
4677
|
+
if (diff.scenarioShortId && diff.scenarioName) {
|
|
4678
|
+
return `${diff.scenarioShortId}: ${diff.scenarioName}`;
|
|
4679
|
+
}
|
|
4680
|
+
if (diff.scenarioName) {
|
|
4681
|
+
return diff.scenarioName;
|
|
4682
|
+
}
|
|
4683
|
+
return diff.scenarioId.slice(0, 8);
|
|
4684
|
+
}
|
|
4685
|
+
function formatDuration(ms) {
|
|
4686
|
+
if (ms === null)
|
|
4687
|
+
return "-";
|
|
4688
|
+
if (ms < 1000)
|
|
4689
|
+
return `${ms}ms`;
|
|
4690
|
+
return `${(ms / 1000).toFixed(1)}s`;
|
|
4691
|
+
}
|
|
4692
|
+
function formatDurationComparison(d1, d2) {
|
|
4693
|
+
const s1 = formatDuration(d1);
|
|
4694
|
+
const s2 = formatDuration(d2);
|
|
4695
|
+
if (d1 !== null && d2 !== null) {
|
|
4696
|
+
const delta = d2 - d1;
|
|
4697
|
+
const sign = delta > 0 ? "+" : "";
|
|
4698
|
+
return `${s1} -> ${s2} (${sign}${formatDuration(delta)})`;
|
|
4699
|
+
}
|
|
4700
|
+
return `${s1} -> ${s2}`;
|
|
4701
|
+
}
|
|
4702
|
+
function formatDiffTerminal(diff) {
|
|
4703
|
+
const lines = [];
|
|
4704
|
+
lines.push("");
|
|
4705
|
+
lines.push(chalk2.bold(" Run Comparison"));
|
|
4706
|
+
lines.push(` Run 1: ${chalk2.dim(diff.run1.id.slice(0, 8))} (${diff.run1.status}) \u2014 ${diff.run1.startedAt}`);
|
|
4707
|
+
lines.push(` Run 2: ${chalk2.dim(diff.run2.id.slice(0, 8))} (${diff.run2.status}) \u2014 ${diff.run2.startedAt}`);
|
|
4708
|
+
lines.push("");
|
|
4709
|
+
if (diff.regressions.length > 0) {
|
|
4710
|
+
lines.push(chalk2.red.bold(` Regressions (${diff.regressions.length}):`));
|
|
4711
|
+
for (const d of diff.regressions) {
|
|
4712
|
+
const label = formatScenarioLabel(d);
|
|
4713
|
+
const dur = formatDurationComparison(d.duration1, d.duration2);
|
|
4714
|
+
lines.push(chalk2.red(` \u2B07 ${label} ${d.status1} -> ${d.status2} ${chalk2.dim(dur)}`));
|
|
4715
|
+
}
|
|
4716
|
+
lines.push("");
|
|
4717
|
+
}
|
|
4718
|
+
if (diff.fixes.length > 0) {
|
|
4719
|
+
lines.push(chalk2.green.bold(` Fixes (${diff.fixes.length}):`));
|
|
4720
|
+
for (const d of diff.fixes) {
|
|
4721
|
+
const label = formatScenarioLabel(d);
|
|
4722
|
+
const dur = formatDurationComparison(d.duration1, d.duration2);
|
|
4723
|
+
lines.push(chalk2.green(` \u2B06 ${label} ${d.status1} -> ${d.status2} ${chalk2.dim(dur)}`));
|
|
4724
|
+
}
|
|
4725
|
+
lines.push("");
|
|
4726
|
+
}
|
|
4727
|
+
if (diff.unchanged.length > 0) {
|
|
4728
|
+
lines.push(chalk2.dim(` Unchanged (${diff.unchanged.length}):`));
|
|
4729
|
+
for (const d of diff.unchanged) {
|
|
4730
|
+
const label = formatScenarioLabel(d);
|
|
4731
|
+
const dur = formatDurationComparison(d.duration1, d.duration2);
|
|
4732
|
+
lines.push(chalk2.dim(` = ${label} ${d.status2} ${dur}`));
|
|
4733
|
+
}
|
|
4734
|
+
lines.push("");
|
|
4735
|
+
}
|
|
4736
|
+
if (diff.newScenarios.length > 0) {
|
|
4737
|
+
lines.push(chalk2.cyan(` New in run 2 (${diff.newScenarios.length}):`));
|
|
4738
|
+
for (const d of diff.newScenarios) {
|
|
4739
|
+
const label = formatScenarioLabel(d);
|
|
4740
|
+
lines.push(chalk2.cyan(` + ${label} ${d.status2}`));
|
|
4741
|
+
}
|
|
4742
|
+
lines.push("");
|
|
4743
|
+
}
|
|
4744
|
+
if (diff.removedScenarios.length > 0) {
|
|
4745
|
+
lines.push(chalk2.yellow(` Removed from run 2 (${diff.removedScenarios.length}):`));
|
|
4746
|
+
for (const d of diff.removedScenarios) {
|
|
4747
|
+
const label = formatScenarioLabel(d);
|
|
4748
|
+
lines.push(chalk2.yellow(` - ${label} was ${d.status1}`));
|
|
4749
|
+
}
|
|
4750
|
+
lines.push("");
|
|
4751
|
+
}
|
|
4752
|
+
lines.push(chalk2.bold(` Summary: ${diff.regressions.length} regressions, ${diff.fixes.length} fixes, ${diff.unchanged.length} unchanged`));
|
|
4753
|
+
lines.push("");
|
|
4754
|
+
return lines.join(`
|
|
4755
|
+
`);
|
|
4756
|
+
}
|
|
4757
|
+
function formatDiffJSON(diff) {
|
|
4758
|
+
return JSON.stringify(diff, null, 2);
|
|
4759
|
+
}
|
|
4760
|
+
|
|
4761
|
+
// src/lib/report.ts
|
|
4762
|
+
init_runs();
|
|
4763
|
+
import { readFileSync as readFileSync3, existsSync as existsSync6 } from "fs";
|
|
4764
|
+
function imageToBase64(filePath) {
|
|
4765
|
+
if (!filePath || !existsSync6(filePath))
|
|
4766
|
+
return "";
|
|
4767
|
+
try {
|
|
4768
|
+
const buffer = readFileSync3(filePath);
|
|
4769
|
+
const base64 = buffer.toString("base64");
|
|
4770
|
+
return `data:image/png;base64,${base64}`;
|
|
4771
|
+
} catch {
|
|
4772
|
+
return "";
|
|
4773
|
+
}
|
|
4774
|
+
}
|
|
4775
|
+
function escapeHtml(text) {
|
|
4776
|
+
return text.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, """).replace(/'/g, "'");
|
|
4777
|
+
}
|
|
4778
|
+
function formatDuration2(ms) {
|
|
4779
|
+
if (ms < 1000)
|
|
4780
|
+
return `${ms}ms`;
|
|
4781
|
+
if (ms < 60000)
|
|
4782
|
+
return `${(ms / 1000).toFixed(1)}s`;
|
|
4783
|
+
const mins = Math.floor(ms / 60000);
|
|
4784
|
+
const secs = (ms % 60000 / 1000).toFixed(0);
|
|
4785
|
+
return `${mins}m ${secs}s`;
|
|
4786
|
+
}
|
|
4787
|
+
function formatCost(cents) {
|
|
4788
|
+
if (cents < 1)
|
|
4789
|
+
return `$${(cents / 100).toFixed(4)}`;
|
|
4790
|
+
return `$${(cents / 100).toFixed(2)}`;
|
|
4791
|
+
}
|
|
4792
|
+
function statusBadge(status) {
|
|
4793
|
+
const colors = {
|
|
4794
|
+
passed: { bg: "#22c55e", text: "#000" },
|
|
4795
|
+
failed: { bg: "#ef4444", text: "#fff" },
|
|
4796
|
+
error: { bg: "#eab308", text: "#000" },
|
|
4797
|
+
skipped: { bg: "#6b7280", text: "#fff" }
|
|
4798
|
+
};
|
|
4799
|
+
const c = colors[status] ?? { bg: "#6b7280", text: "#fff" };
|
|
4800
|
+
const label = status.toUpperCase();
|
|
4801
|
+
return `<span style="display:inline-block;padding:2px 10px;border-radius:4px;font-size:12px;font-weight:700;background:${c.bg};color:${c.text};letter-spacing:0.5px;">${label}</span>`;
|
|
4802
|
+
}
|
|
4803
|
+
function renderScreenshots(screenshots) {
|
|
4804
|
+
if (screenshots.length === 0)
|
|
4805
|
+
return "";
|
|
4806
|
+
let html = `<div style="display:flex;flex-wrap:wrap;gap:12px;margin-top:12px;">`;
|
|
4807
|
+
for (let i = 0;i < screenshots.length; i++) {
|
|
4808
|
+
const ss = screenshots[i];
|
|
4809
|
+
const dataUri = imageToBase64(ss.filePath);
|
|
4810
|
+
const checkId = `ss-${ss.id}`;
|
|
4811
|
+
if (dataUri) {
|
|
4812
|
+
html += `
|
|
4813
|
+
<div style="flex:0 0 auto;">
|
|
4814
|
+
<input type="checkbox" id="${checkId}" style="display:none;" />
|
|
4815
|
+
<label for="${checkId}" style="cursor:pointer;">
|
|
4816
|
+
<img src="${dataUri}" alt="Step ${ss.stepNumber}: ${escapeHtml(ss.action)}"
|
|
4817
|
+
style="max-width:200px;max-height:150px;border-radius:6px;border:1px solid #262626;display:block;" />
|
|
4818
|
+
</label>
|
|
4819
|
+
<div style="position:fixed;top:0;left:0;width:100%;height:100%;background:rgba(0,0,0,0.9);z-index:1000;display:none;align-items:center;justify-content:center;">
|
|
4820
|
+
<label for="${checkId}" style="position:absolute;top:0;left:0;width:100%;height:100%;cursor:pointer;"></label>
|
|
4821
|
+
<img src="${dataUri}" alt="Step ${ss.stepNumber}: ${escapeHtml(ss.action)}"
|
|
4822
|
+
style="max-width:600px;max-height:90vh;border-radius:8px;position:relative;z-index:1001;" />
|
|
4823
|
+
</div>
|
|
4824
|
+
<div style="font-size:11px;color:#888;margin-top:4px;max-width:200px;overflow:hidden;text-overflow:ellipsis;white-space:nowrap;">
|
|
4825
|
+
${ss.stepNumber}. ${escapeHtml(ss.action)}
|
|
4826
|
+
</div>
|
|
4827
|
+
</div>`;
|
|
4828
|
+
} else {
|
|
4829
|
+
html += `
|
|
4830
|
+
<div style="flex:0 0 auto;width:200px;height:150px;background:#1a1a1a;border:1px dashed #333;border-radius:6px;display:flex;align-items:center;justify-content:center;color:#555;font-size:12px;">
|
|
4831
|
+
Screenshot not found
|
|
4832
|
+
<div style="font-size:11px;color:#888;margin-top:4px;">${ss.stepNumber}. ${escapeHtml(ss.action)}</div>
|
|
4833
|
+
</div>`;
|
|
4834
|
+
}
|
|
4835
|
+
}
|
|
4836
|
+
html += `</div>`;
|
|
4837
|
+
return html;
|
|
4838
|
+
}
|
|
4839
|
+
function generateHtmlReport(runId) {
|
|
4840
|
+
const run = getRun(runId);
|
|
4841
|
+
if (!run)
|
|
4842
|
+
throw new Error(`Run not found: ${runId}`);
|
|
4843
|
+
const results = getResultsByRun(run.id);
|
|
4844
|
+
const resultData = [];
|
|
4845
|
+
for (const result of results) {
|
|
4846
|
+
const screenshots = listScreenshots(result.id);
|
|
4847
|
+
const scenario = getScenario(result.scenarioId);
|
|
4848
|
+
resultData.push({
|
|
4849
|
+
result,
|
|
4850
|
+
scenarioName: scenario?.name ?? "Unknown Scenario",
|
|
4851
|
+
scenarioShortId: scenario?.shortId ?? result.scenarioId.slice(0, 8),
|
|
4852
|
+
screenshots
|
|
4853
|
+
});
|
|
4854
|
+
}
|
|
4855
|
+
const passedCount = results.filter((r) => r.status === "passed").length;
|
|
4856
|
+
const failedCount = results.filter((r) => r.status === "failed").length;
|
|
4857
|
+
const errorCount = results.filter((r) => r.status === "error").length;
|
|
4858
|
+
const totalCount = results.length;
|
|
4859
|
+
const totalTokens = results.reduce((sum, r) => sum + r.tokensUsed, 0);
|
|
4860
|
+
const totalCostCents = results.reduce((sum, r) => sum + r.costCents, 0);
|
|
4861
|
+
const totalDurationMs = run.finishedAt && run.startedAt ? new Date(run.finishedAt).getTime() - new Date(run.startedAt).getTime() : results.reduce((sum, r) => sum + r.durationMs, 0);
|
|
4862
|
+
const generatedAt = new Date().toISOString();
|
|
4863
|
+
let resultCards = "";
|
|
4864
|
+
for (const { result, scenarioName, scenarioShortId, screenshots } of resultData) {
|
|
4865
|
+
resultCards += `
|
|
4866
|
+
<div style="background:#141414;border:1px solid #262626;border-radius:8px;padding:20px;margin-bottom:16px;">
|
|
4867
|
+
<div style="display:flex;align-items:center;gap:12px;margin-bottom:12px;">
|
|
4868
|
+
${statusBadge(result.status)}
|
|
4869
|
+
<span style="font-size:16px;font-weight:600;color:#e5e5e5;">${escapeHtml(scenarioName)}</span>
|
|
4870
|
+
<span style="font-size:12px;color:#666;font-family:monospace;">${escapeHtml(scenarioShortId)}</span>
|
|
4871
|
+
</div>
|
|
4872
|
+
|
|
4873
|
+
${result.reasoning ? `<div style="color:#a3a3a3;font-size:14px;line-height:1.6;margin-bottom:12px;padding:12px;background:#0d0d0d;border-radius:6px;border-left:3px solid #333;">${escapeHtml(result.reasoning)}</div>` : ""}
|
|
4874
|
+
|
|
4875
|
+
${result.error ? `<div style="color:#ef4444;font-size:13px;margin-bottom:12px;padding:12px;background:#1a0a0a;border-radius:6px;border-left:3px solid #ef4444;font-family:monospace;">${escapeHtml(result.error)}</div>` : ""}
|
|
4876
|
+
|
|
4877
|
+
<div style="display:flex;gap:24px;font-size:13px;color:#888;">
|
|
4878
|
+
<span>Duration: <span style="color:#d4d4d4;">${formatDuration2(result.durationMs)}</span></span>
|
|
4879
|
+
<span>Steps: <span style="color:#d4d4d4;">${result.stepsCompleted}/${result.stepsTotal}</span></span>
|
|
4880
|
+
<span>Tokens: <span style="color:#d4d4d4;">${result.tokensUsed.toLocaleString()}</span></span>
|
|
4881
|
+
<span>Cost: <span style="color:#d4d4d4;">${formatCost(result.costCents)}</span></span>
|
|
4882
|
+
<span>Model: <span style="color:#d4d4d4;">${escapeHtml(result.model)}</span></span>
|
|
4883
|
+
</div>
|
|
4884
|
+
|
|
4885
|
+
${renderScreenshots(screenshots)}
|
|
4886
|
+
</div>`;
|
|
4887
|
+
}
|
|
4888
|
+
return `<!DOCTYPE html>
|
|
4889
|
+
<html lang="en">
|
|
4890
|
+
<head>
|
|
4891
|
+
<meta charset="UTF-8" />
|
|
4892
|
+
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
|
4893
|
+
<title>Test Report - ${escapeHtml(run.id.slice(0, 8))}</title>
|
|
4894
|
+
<style>
|
|
4895
|
+
* { margin: 0; padding: 0; box-sizing: border-box; }
|
|
4896
|
+
body { background: #0a0a0a; color: #e5e5e5; font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif; padding: 40px 20px; }
|
|
4897
|
+
.container { max-width: 960px; margin: 0 auto; }
|
|
4898
|
+
input[type="checkbox"]:checked ~ div:last-of-type { display: flex !important; }
|
|
4899
|
+
</style>
|
|
4900
|
+
</head>
|
|
4901
|
+
<body>
|
|
4902
|
+
<div class="container">
|
|
4903
|
+
<!-- Header -->
|
|
4904
|
+
<div style="margin-bottom:32px;">
|
|
4905
|
+
<h1 style="font-size:28px;font-weight:700;margin-bottom:8px;color:#fff;">Test Report</h1>
|
|
4906
|
+
<div style="display:flex;flex-wrap:wrap;gap:24px;font-size:14px;color:#888;">
|
|
4907
|
+
<span>Run: <span style="color:#d4d4d4;font-family:monospace;">${escapeHtml(run.id.slice(0, 8))}</span></span>
|
|
4908
|
+
<span>URL: <a href="${escapeHtml(run.url)}" style="color:#60a5fa;text-decoration:none;">${escapeHtml(run.url)}</a></span>
|
|
4909
|
+
<span>Model: <span style="color:#d4d4d4;">${escapeHtml(run.model)}</span></span>
|
|
4910
|
+
<span>Date: <span style="color:#d4d4d4;">${escapeHtml(run.startedAt)}</span></span>
|
|
4911
|
+
<span>Duration: <span style="color:#d4d4d4;">${formatDuration2(totalDurationMs)}</span></span>
|
|
4912
|
+
<span>Status: ${statusBadge(run.status)}</span>
|
|
4913
|
+
</div>
|
|
4914
|
+
</div>
|
|
4915
|
+
|
|
4916
|
+
<!-- Summary Bar -->
|
|
4917
|
+
<div style="display:flex;gap:16px;margin-bottom:32px;">
|
|
4918
|
+
<div style="flex:1;background:#141414;border:1px solid #262626;border-radius:8px;padding:16px;text-align:center;">
|
|
4919
|
+
<div style="font-size:28px;font-weight:700;color:#e5e5e5;">${totalCount}</div>
|
|
4920
|
+
<div style="font-size:12px;color:#888;margin-top:4px;">TOTAL</div>
|
|
4921
|
+
</div>
|
|
4922
|
+
<div style="flex:1;background:#141414;border:1px solid #262626;border-radius:8px;padding:16px;text-align:center;">
|
|
4923
|
+
<div style="font-size:28px;font-weight:700;color:#22c55e;">${passedCount}</div>
|
|
4924
|
+
<div style="font-size:12px;color:#888;margin-top:4px;">PASSED</div>
|
|
4925
|
+
</div>
|
|
4926
|
+
<div style="flex:1;background:#141414;border:1px solid #262626;border-radius:8px;padding:16px;text-align:center;">
|
|
4927
|
+
<div style="font-size:28px;font-weight:700;color:#ef4444;">${failedCount}</div>
|
|
4928
|
+
<div style="font-size:12px;color:#888;margin-top:4px;">FAILED</div>
|
|
4929
|
+
</div>
|
|
4930
|
+
${errorCount > 0 ? `
|
|
4931
|
+
<div style="flex:1;background:#141414;border:1px solid #262626;border-radius:8px;padding:16px;text-align:center;">
|
|
4932
|
+
<div style="font-size:28px;font-weight:700;color:#eab308;">${errorCount}</div>
|
|
4933
|
+
<div style="font-size:12px;color:#888;margin-top:4px;">ERRORS</div>
|
|
4934
|
+
</div>` : ""}
|
|
4935
|
+
</div>
|
|
4936
|
+
|
|
4937
|
+
<!-- Results -->
|
|
4938
|
+
${resultCards}
|
|
4939
|
+
|
|
4940
|
+
<!-- Footer -->
|
|
4941
|
+
<div style="margin-top:32px;padding-top:20px;border-top:1px solid #262626;display:flex;justify-content:space-between;font-size:13px;color:#666;">
|
|
4942
|
+
<div>
|
|
4943
|
+
Total tokens: ${totalTokens.toLocaleString()} | Total cost: ${formatCost(totalCostCents)}
|
|
4944
|
+
</div>
|
|
4945
|
+
<div>
|
|
4946
|
+
Generated: ${escapeHtml(generatedAt)}
|
|
4947
|
+
</div>
|
|
4948
|
+
</div>
|
|
4949
|
+
</div>
|
|
4950
|
+
</body>
|
|
4951
|
+
</html>`;
|
|
4952
|
+
}
|
|
4953
|
+
function generateLatestReport() {
|
|
4954
|
+
const runs = listRuns({ limit: 1 });
|
|
4955
|
+
if (runs.length === 0)
|
|
4956
|
+
throw new Error("No runs found");
|
|
4957
|
+
return generateHtmlReport(runs[0].id);
|
|
4958
|
+
}
|
|
4959
|
+
|
|
4960
|
+
// src/lib/costs.ts
|
|
4961
|
+
init_database();
|
|
4962
|
+
import chalk3 from "chalk";
|
|
4963
|
+
function getDateFilter(period) {
|
|
4964
|
+
switch (period) {
|
|
4965
|
+
case "day":
|
|
4966
|
+
return "AND r.created_at >= date('now', 'start of day')";
|
|
4967
|
+
case "week":
|
|
4968
|
+
return "AND r.created_at >= date('now', '-7 days')";
|
|
4969
|
+
case "month":
|
|
4970
|
+
return "AND r.created_at >= date('now', '-30 days')";
|
|
4971
|
+
case "all":
|
|
4972
|
+
return "";
|
|
4973
|
+
}
|
|
4974
|
+
}
|
|
4975
|
+
function getPeriodDays(period) {
|
|
4976
|
+
switch (period) {
|
|
4977
|
+
case "day":
|
|
4978
|
+
return 1;
|
|
4979
|
+
case "week":
|
|
4980
|
+
return 7;
|
|
4981
|
+
case "month":
|
|
4982
|
+
return 30;
|
|
4983
|
+
case "all":
|
|
4984
|
+
return 30;
|
|
4985
|
+
}
|
|
4986
|
+
}
|
|
4987
|
+
function getCostSummary(options) {
|
|
4988
|
+
const db2 = getDatabase();
|
|
4989
|
+
const period = options?.period ?? "month";
|
|
4990
|
+
const projectId = options?.projectId;
|
|
4991
|
+
const dateFilter = getDateFilter(period);
|
|
4992
|
+
const projectFilter = projectId ? "AND ru.project_id = ?" : "";
|
|
4993
|
+
const projectParams = projectId ? [projectId] : [];
|
|
4994
|
+
const totalsRow = db2.query(`SELECT
|
|
4995
|
+
COALESCE(SUM(r.cost_cents), 0) as total_cost,
|
|
4996
|
+
COALESCE(SUM(r.tokens_used), 0) as total_tokens,
|
|
4997
|
+
COUNT(DISTINCT r.run_id) as run_count
|
|
4998
|
+
FROM results r
|
|
4999
|
+
JOIN runs ru ON r.run_id = ru.id
|
|
5000
|
+
WHERE 1=1 ${dateFilter} ${projectFilter}`).get(...projectParams);
|
|
5001
|
+
const modelRows = db2.query(`SELECT
|
|
5002
|
+
r.model,
|
|
5003
|
+
COALESCE(SUM(r.cost_cents), 0) as cost_cents,
|
|
5004
|
+
COALESCE(SUM(r.tokens_used), 0) as tokens,
|
|
5005
|
+
COUNT(DISTINCT r.run_id) as runs
|
|
5006
|
+
FROM results r
|
|
5007
|
+
JOIN runs ru ON r.run_id = ru.id
|
|
5008
|
+
WHERE 1=1 ${dateFilter} ${projectFilter}
|
|
5009
|
+
GROUP BY r.model
|
|
5010
|
+
ORDER BY cost_cents DESC`).all(...projectParams);
|
|
5011
|
+
const byModel = {};
|
|
5012
|
+
for (const row of modelRows) {
|
|
5013
|
+
byModel[row.model] = {
|
|
5014
|
+
costCents: row.cost_cents,
|
|
5015
|
+
tokens: row.tokens,
|
|
5016
|
+
runs: row.runs
|
|
5017
|
+
};
|
|
5018
|
+
}
|
|
5019
|
+
const scenarioRows = db2.query(`SELECT
|
|
5020
|
+
r.scenario_id,
|
|
5021
|
+
COALESCE(s.name, r.scenario_id) as name,
|
|
5022
|
+
COALESCE(SUM(r.cost_cents), 0) as cost_cents,
|
|
5023
|
+
COALESCE(SUM(r.tokens_used), 0) as tokens,
|
|
5024
|
+
COUNT(DISTINCT r.run_id) as runs
|
|
5025
|
+
FROM results r
|
|
5026
|
+
JOIN runs ru ON r.run_id = ru.id
|
|
5027
|
+
LEFT JOIN scenarios s ON r.scenario_id = s.id
|
|
5028
|
+
WHERE 1=1 ${dateFilter} ${projectFilter}
|
|
5029
|
+
GROUP BY r.scenario_id
|
|
5030
|
+
ORDER BY cost_cents DESC
|
|
5031
|
+
LIMIT 10`).all(...projectParams);
|
|
5032
|
+
const byScenario = scenarioRows.map((row) => ({
|
|
5033
|
+
scenarioId: row.scenario_id,
|
|
5034
|
+
name: row.name,
|
|
5035
|
+
costCents: row.cost_cents,
|
|
5036
|
+
tokens: row.tokens,
|
|
5037
|
+
runs: row.runs
|
|
5038
|
+
}));
|
|
5039
|
+
const runCount = totalsRow.run_count;
|
|
5040
|
+
const avgCostPerRun = runCount > 0 ? totalsRow.total_cost / runCount : 0;
|
|
5041
|
+
const periodDays = getPeriodDays(period);
|
|
5042
|
+
const estimatedMonthlyCents = periodDays > 0 ? totalsRow.total_cost / periodDays * 30 : 0;
|
|
5043
|
+
return {
|
|
5044
|
+
period,
|
|
5045
|
+
totalCostCents: totalsRow.total_cost,
|
|
5046
|
+
totalTokens: totalsRow.total_tokens,
|
|
5047
|
+
runCount,
|
|
5048
|
+
byModel,
|
|
5049
|
+
byScenario,
|
|
5050
|
+
avgCostPerRun,
|
|
5051
|
+
estimatedMonthlyCents
|
|
5052
|
+
};
|
|
5053
|
+
}
|
|
5054
|
+
function formatDollars(cents) {
|
|
5055
|
+
return `$${(cents / 100).toFixed(2)}`;
|
|
5056
|
+
}
|
|
5057
|
+
function formatTokens(tokens) {
|
|
5058
|
+
if (tokens >= 1e6)
|
|
5059
|
+
return `${(tokens / 1e6).toFixed(1)}M`;
|
|
5060
|
+
if (tokens >= 1000)
|
|
5061
|
+
return `${(tokens / 1000).toFixed(1)}K`;
|
|
5062
|
+
return String(tokens);
|
|
5063
|
+
}
|
|
5064
|
+
function formatCostsTerminal(summary) {
|
|
5065
|
+
const lines = [];
|
|
5066
|
+
lines.push("");
|
|
5067
|
+
lines.push(chalk3.bold(` Cost Summary (${summary.period})`));
|
|
5068
|
+
lines.push("");
|
|
5069
|
+
lines.push(` Total: ${chalk3.yellow(formatDollars(summary.totalCostCents))} (${formatTokens(summary.totalTokens)} tokens across ${summary.runCount} runs)`);
|
|
5070
|
+
lines.push(` Avg/run: ${chalk3.yellow(formatDollars(summary.avgCostPerRun))}`);
|
|
5071
|
+
lines.push(` Est/month: ${chalk3.yellow(formatDollars(summary.estimatedMonthlyCents))}`);
|
|
5072
|
+
const modelEntries = Object.entries(summary.byModel);
|
|
5073
|
+
if (modelEntries.length > 0) {
|
|
5074
|
+
lines.push("");
|
|
5075
|
+
lines.push(chalk3.bold(" By Model"));
|
|
5076
|
+
lines.push(` ${"Model".padEnd(40)} ${"Cost".padEnd(12)} ${"Tokens".padEnd(12)} Runs`);
|
|
5077
|
+
lines.push(` ${"\u2500".repeat(40)} ${"\u2500".repeat(12)} ${"\u2500".repeat(12)} ${"\u2500".repeat(6)}`);
|
|
5078
|
+
for (const [model, data] of modelEntries) {
|
|
5079
|
+
lines.push(` ${model.padEnd(40)} ${formatDollars(data.costCents).padEnd(12)} ${formatTokens(data.tokens).padEnd(12)} ${data.runs}`);
|
|
5080
|
+
}
|
|
5081
|
+
}
|
|
5082
|
+
if (summary.byScenario.length > 0) {
|
|
5083
|
+
lines.push("");
|
|
5084
|
+
lines.push(chalk3.bold(" Top Scenarios by Cost"));
|
|
5085
|
+
lines.push(` ${"Scenario".padEnd(40)} ${"Cost".padEnd(12)} ${"Tokens".padEnd(12)} Runs`);
|
|
5086
|
+
lines.push(` ${"\u2500".repeat(40)} ${"\u2500".repeat(12)} ${"\u2500".repeat(12)} ${"\u2500".repeat(6)}`);
|
|
5087
|
+
for (const s of summary.byScenario) {
|
|
5088
|
+
const label = s.name.length > 38 ? s.name.slice(0, 35) + "..." : s.name;
|
|
5089
|
+
lines.push(` ${label.padEnd(40)} ${formatDollars(s.costCents).padEnd(12)} ${formatTokens(s.tokens).padEnd(12)} ${s.runs}`);
|
|
5090
|
+
}
|
|
5091
|
+
}
|
|
5092
|
+
lines.push("");
|
|
5093
|
+
return lines.join(`
|
|
5094
|
+
`);
|
|
5095
|
+
}
|
|
5096
|
+
function formatCostsJSON(summary) {
|
|
5097
|
+
return JSON.stringify(summary, null, 2);
|
|
5098
|
+
}
|
|
5099
|
+
|
|
5100
|
+
// src/db/schedules.ts
|
|
5101
|
+
init_database();
|
|
5102
|
+
init_types();
|
|
5103
|
+
init_database();
|
|
5104
|
+
function createSchedule(input) {
|
|
5105
|
+
const db2 = getDatabase();
|
|
5106
|
+
const id = uuid();
|
|
5107
|
+
const timestamp = now();
|
|
5108
|
+
db2.query(`
|
|
5109
|
+
INSERT INTO schedules (id, project_id, name, cron_expression, url, scenario_filter, model, headed, parallel, timeout_ms, enabled, created_at, updated_at)
|
|
5110
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1, ?, ?)
|
|
5111
|
+
`).run(id, input.projectId ?? null, input.name, input.cronExpression, input.url, JSON.stringify(input.scenarioFilter ?? {}), input.model ?? null, input.headed ? 1 : 0, input.parallel ?? 1, input.timeoutMs ?? null, timestamp, timestamp);
|
|
5112
|
+
return getSchedule(id);
|
|
5113
|
+
}
|
|
5114
|
+
function getSchedule(id) {
|
|
5115
|
+
const db2 = getDatabase();
|
|
5116
|
+
let row = db2.query("SELECT * FROM schedules WHERE id = ?").get(id);
|
|
5117
|
+
if (row)
|
|
5118
|
+
return scheduleFromRow(row);
|
|
5119
|
+
const fullId = resolvePartialId("schedules", id);
|
|
5120
|
+
if (fullId) {
|
|
5121
|
+
row = db2.query("SELECT * FROM schedules WHERE id = ?").get(fullId);
|
|
5122
|
+
if (row)
|
|
5123
|
+
return scheduleFromRow(row);
|
|
5124
|
+
}
|
|
5125
|
+
return null;
|
|
5126
|
+
}
|
|
5127
|
+
function listSchedules(filter) {
|
|
5128
|
+
const db2 = getDatabase();
|
|
5129
|
+
const conditions = [];
|
|
5130
|
+
const params = [];
|
|
5131
|
+
if (filter?.projectId) {
|
|
5132
|
+
conditions.push("project_id = ?");
|
|
5133
|
+
params.push(filter.projectId);
|
|
5134
|
+
}
|
|
5135
|
+
if (filter?.enabled !== undefined) {
|
|
5136
|
+
conditions.push("enabled = ?");
|
|
5137
|
+
params.push(filter.enabled ? 1 : 0);
|
|
5138
|
+
}
|
|
5139
|
+
let sql = "SELECT * FROM schedules";
|
|
5140
|
+
if (conditions.length > 0) {
|
|
5141
|
+
sql += " WHERE " + conditions.join(" AND ");
|
|
5142
|
+
}
|
|
5143
|
+
sql += " ORDER BY created_at DESC";
|
|
5144
|
+
if (filter?.limit) {
|
|
5145
|
+
sql += " LIMIT ?";
|
|
5146
|
+
params.push(filter.limit);
|
|
5147
|
+
}
|
|
5148
|
+
if (filter?.offset) {
|
|
5149
|
+
sql += " OFFSET ?";
|
|
5150
|
+
params.push(filter.offset);
|
|
5151
|
+
}
|
|
5152
|
+
const rows = db2.query(sql).all(...params);
|
|
5153
|
+
return rows.map(scheduleFromRow);
|
|
5154
|
+
}
|
|
5155
|
+
function updateSchedule(id, input) {
|
|
5156
|
+
const db2 = getDatabase();
|
|
5157
|
+
const existing = getSchedule(id);
|
|
5158
|
+
if (!existing) {
|
|
5159
|
+
throw new ScheduleNotFoundError(id);
|
|
5160
|
+
}
|
|
5161
|
+
const sets = [];
|
|
5162
|
+
const params = [];
|
|
5163
|
+
if (input.name !== undefined) {
|
|
5164
|
+
sets.push("name = ?");
|
|
5165
|
+
params.push(input.name);
|
|
5166
|
+
}
|
|
5167
|
+
if (input.cronExpression !== undefined) {
|
|
5168
|
+
sets.push("cron_expression = ?");
|
|
5169
|
+
params.push(input.cronExpression);
|
|
5170
|
+
}
|
|
5171
|
+
if (input.url !== undefined) {
|
|
5172
|
+
sets.push("url = ?");
|
|
5173
|
+
params.push(input.url);
|
|
5174
|
+
}
|
|
5175
|
+
if (input.scenarioFilter !== undefined) {
|
|
5176
|
+
sets.push("scenario_filter = ?");
|
|
5177
|
+
params.push(JSON.stringify(input.scenarioFilter));
|
|
5178
|
+
}
|
|
5179
|
+
if (input.model !== undefined) {
|
|
5180
|
+
sets.push("model = ?");
|
|
5181
|
+
params.push(input.model);
|
|
5182
|
+
}
|
|
5183
|
+
if (input.headed !== undefined) {
|
|
5184
|
+
sets.push("headed = ?");
|
|
5185
|
+
params.push(input.headed ? 1 : 0);
|
|
5186
|
+
}
|
|
5187
|
+
if (input.parallel !== undefined) {
|
|
5188
|
+
sets.push("parallel = ?");
|
|
5189
|
+
params.push(input.parallel);
|
|
5190
|
+
}
|
|
5191
|
+
if (input.timeoutMs !== undefined) {
|
|
5192
|
+
sets.push("timeout_ms = ?");
|
|
5193
|
+
params.push(input.timeoutMs);
|
|
5194
|
+
}
|
|
5195
|
+
if (input.enabled !== undefined) {
|
|
5196
|
+
sets.push("enabled = ?");
|
|
5197
|
+
params.push(input.enabled ? 1 : 0);
|
|
5198
|
+
}
|
|
5199
|
+
if (sets.length === 0) {
|
|
5200
|
+
return existing;
|
|
5201
|
+
}
|
|
5202
|
+
sets.push("updated_at = ?");
|
|
5203
|
+
params.push(now());
|
|
5204
|
+
params.push(existing.id);
|
|
5205
|
+
db2.query(`UPDATE schedules SET ${sets.join(", ")} WHERE id = ?`).run(...params);
|
|
5206
|
+
return getSchedule(existing.id);
|
|
5207
|
+
}
|
|
5208
|
+
function deleteSchedule(id) {
|
|
5209
|
+
const db2 = getDatabase();
|
|
5210
|
+
const schedule = getSchedule(id);
|
|
5211
|
+
if (!schedule)
|
|
5212
|
+
return false;
|
|
5213
|
+
const result = db2.query("DELETE FROM schedules WHERE id = ?").run(schedule.id);
|
|
5214
|
+
return result.changes > 0;
|
|
5215
|
+
}
|
|
5216
|
+
|
|
5217
|
+
// src/lib/templates.ts
|
|
5218
|
+
var SCENARIO_TEMPLATES = {
|
|
5219
|
+
auth: [
|
|
5220
|
+
{ name: "Login with valid credentials", description: "Navigate to the login page, enter valid credentials, submit the form, and verify redirect to authenticated area. Check that user menu/avatar is visible.", tags: ["auth", "smoke"], priority: "critical", requiresAuth: false, steps: ["Navigate to login page", "Enter email and password", "Submit login form", "Verify redirect to dashboard/home", "Verify user menu or avatar is visible"] },
|
|
5221
|
+
{ name: "Signup flow", description: "Navigate to signup page, fill all required fields with valid data, submit, and verify account creation succeeds.", tags: ["auth"], priority: "high", steps: ["Navigate to signup page", "Fill all required fields", "Submit registration form", "Verify success message or redirect"] },
|
|
5222
|
+
{ name: "Logout flow", description: "While authenticated, find and click the logout button/link, verify redirect to public page.", tags: ["auth"], priority: "medium", requiresAuth: true, steps: ["Click user menu or profile", "Click logout", "Verify redirect to login or home page"] }
|
|
5223
|
+
],
|
|
5224
|
+
crud: [
|
|
5225
|
+
{ name: "Create new item", description: "Navigate to the create form, fill all fields, submit, and verify the new item appears in the list.", tags: ["crud"], priority: "high", steps: ["Navigate to the list/index page", "Click create/add button", "Fill all required fields", "Submit the form", "Verify new item appears in list"] },
|
|
5226
|
+
{ name: "Read/view item details", description: "Click on an existing item to view its details page. Verify all fields are displayed correctly.", tags: ["crud"], priority: "medium", steps: ["Navigate to list page", "Click on an item", "Verify detail page shows all fields"] },
|
|
5227
|
+
{ name: "Update existing item", description: "Edit an existing item, change some fields, save, and verify changes persisted.", tags: ["crud"], priority: "high", steps: ["Navigate to item detail", "Click edit button", "Modify fields", "Save changes", "Verify updated values"] },
|
|
5228
|
+
{ name: "Delete item", description: "Delete an existing item and verify it's removed from the list.", tags: ["crud"], priority: "medium", steps: ["Navigate to list page", "Click delete on an item", "Confirm deletion", "Verify item removed from list"] }
|
|
5229
|
+
],
|
|
5230
|
+
forms: [
|
|
5231
|
+
{ name: "Form validation - empty submission", description: "Submit a form with all fields empty and verify validation errors appear for required fields.", tags: ["forms", "validation"], priority: "high", steps: ["Navigate to form page", "Click submit without filling fields", "Verify validation errors appear for each required field"] },
|
|
5232
|
+
{ name: "Form validation - invalid data", description: "Submit a form with invalid data (bad email, short password, etc) and verify appropriate error messages.", tags: ["forms", "validation"], priority: "high", steps: ["Navigate to form page", "Enter invalid email format", "Enter too-short password", "Submit form", "Verify specific validation error messages"] },
|
|
5233
|
+
{ name: "Form successful submission", description: "Fill form with valid data, submit, and verify success state (redirect, success message, or data saved).", tags: ["forms"], priority: "high", steps: ["Navigate to form page", "Fill all fields with valid data", "Submit form", "Verify success state"] }
|
|
5234
|
+
],
|
|
5235
|
+
nav: [
|
|
5236
|
+
{ name: "Main navigation links work", description: "Click through each main navigation link and verify each page loads correctly without errors.", tags: ["navigation", "smoke"], priority: "high", steps: ["Click each nav link", "Verify page loads", "Verify no error states", "Verify breadcrumbs if present"] },
|
|
5237
|
+
{ name: "Mobile navigation", description: "At mobile viewport, verify hamburger menu opens, navigation links are accessible, and pages load correctly.", tags: ["navigation", "responsive"], priority: "medium", steps: ["Resize to mobile viewport", "Click hamburger/menu icon", "Verify nav links appear", "Click a nav link", "Verify page loads"] }
|
|
5238
|
+
],
|
|
5239
|
+
a11y: [
|
|
5240
|
+
{ name: "Keyboard navigation", description: "Navigate the page using only keyboard (Tab, Enter, Escape). Verify all interactive elements are reachable and focusable.", tags: ["a11y", "keyboard"], priority: "high", steps: ["Press Tab to move through elements", "Verify focus indicators are visible", "Press Enter on buttons/links", "Verify actions trigger correctly", "Press Escape to close modals/dropdowns"] },
|
|
5241
|
+
{ name: "Image alt text", description: "Check that all images have meaningful alt text attributes.", tags: ["a11y"], priority: "medium", steps: ["Find all images on the page", "Check each image has an alt attribute", "Verify alt text is descriptive, not empty or generic"] }
|
|
5242
|
+
]
|
|
5243
|
+
};
|
|
5244
|
+
function getTemplate(name) {
|
|
5245
|
+
return SCENARIO_TEMPLATES[name] ?? null;
|
|
5246
|
+
}
|
|
5247
|
+
function listTemplateNames() {
|
|
5248
|
+
return Object.keys(SCENARIO_TEMPLATES);
|
|
5249
|
+
}
|
|
5250
|
+
|
|
5251
|
+
// src/db/auth-presets.ts
|
|
5252
|
+
init_database();
|
|
5253
|
+
function fromRow(row) {
|
|
5254
|
+
return {
|
|
5255
|
+
id: row.id,
|
|
5256
|
+
name: row.name,
|
|
5257
|
+
email: row.email,
|
|
5258
|
+
password: row.password,
|
|
5259
|
+
loginPath: row.login_path,
|
|
5260
|
+
metadata: JSON.parse(row.metadata),
|
|
5261
|
+
createdAt: row.created_at
|
|
5262
|
+
};
|
|
5263
|
+
}
|
|
5264
|
+
function createAuthPreset(input) {
|
|
5265
|
+
const db2 = getDatabase();
|
|
5266
|
+
const id = uuid();
|
|
5267
|
+
const timestamp = now();
|
|
5268
|
+
db2.query(`
|
|
5269
|
+
INSERT INTO auth_presets (id, name, email, password, login_path, metadata, created_at)
|
|
5270
|
+
VALUES (?, ?, ?, ?, ?, '{}', ?)
|
|
5271
|
+
`).run(id, input.name, input.email, input.password, input.loginPath ?? "/login", timestamp);
|
|
5272
|
+
return getAuthPreset(input.name);
|
|
5273
|
+
}
|
|
5274
|
+
function getAuthPreset(name) {
|
|
5275
|
+
const db2 = getDatabase();
|
|
5276
|
+
const row = db2.query("SELECT * FROM auth_presets WHERE name = ?").get(name);
|
|
5277
|
+
return row ? fromRow(row) : null;
|
|
5278
|
+
}
|
|
5279
|
+
function listAuthPresets() {
|
|
5280
|
+
const db2 = getDatabase();
|
|
5281
|
+
const rows = db2.query("SELECT * FROM auth_presets ORDER BY created_at DESC").all();
|
|
5282
|
+
return rows.map(fromRow);
|
|
5283
|
+
}
|
|
5284
|
+
function deleteAuthPreset(name) {
|
|
5285
|
+
const db2 = getDatabase();
|
|
5286
|
+
const result = db2.query("DELETE FROM auth_presets WHERE name = ?").run(name);
|
|
5287
|
+
return result.changes > 0;
|
|
5288
|
+
}
|
|
5289
|
+
|
|
3768
5290
|
// src/cli/index.tsx
|
|
5291
|
+
import { existsSync as existsSync7, mkdirSync as mkdirSync4 } from "fs";
|
|
3769
5292
|
var program2 = new Command;
|
|
3770
5293
|
program2.name("testers").version("0.0.1").description("AI-powered browser testing CLI");
|
|
5294
|
+
var CONFIG_DIR2 = join6(process.env["HOME"] ?? "~", ".testers");
|
|
5295
|
+
var CONFIG_PATH2 = join6(CONFIG_DIR2, "config.json");
|
|
5296
|
+
function getActiveProject() {
|
|
5297
|
+
try {
|
|
5298
|
+
if (existsSync7(CONFIG_PATH2)) {
|
|
5299
|
+
const raw = JSON.parse(readFileSync4(CONFIG_PATH2, "utf-8"));
|
|
5300
|
+
return raw.activeProject ?? undefined;
|
|
5301
|
+
}
|
|
5302
|
+
} catch {}
|
|
5303
|
+
return;
|
|
5304
|
+
}
|
|
5305
|
+
function resolveProject(optProject) {
|
|
5306
|
+
return optProject ?? getActiveProject();
|
|
5307
|
+
}
|
|
3771
5308
|
program2.command("add <name>").description("Create a new test scenario").option("-d, --description <text>", "Scenario description", "").option("-s, --steps <step>", "Test step (repeatable)", (val, acc) => {
|
|
3772
5309
|
acc.push(val);
|
|
3773
5310
|
return acc;
|
|
3774
5311
|
}, []).option("-t, --tag <tag>", "Tag (repeatable)", (val, acc) => {
|
|
3775
5312
|
acc.push(val);
|
|
3776
5313
|
return acc;
|
|
3777
|
-
}, []).option("-p, --priority <level>", "Priority level", "medium").option("-m, --model <model>", "AI model to use").option("--path <path>", "Target path on the URL").option("--auth", "Requires authentication", false).option("--timeout <ms>", "Timeout in milliseconds").action((name, opts) => {
|
|
5314
|
+
}, []).option("-p, --priority <level>", "Priority level", "medium").option("-m, --model <model>", "AI model to use").option("--path <path>", "Target path on the URL").option("--auth", "Requires authentication", false).option("--timeout <ms>", "Timeout in milliseconds").option("--project <id>", "Project ID").option("--template <name>", "Seed scenarios from a template (auth, crud, forms, nav, a11y)").action((name, opts) => {
|
|
3778
5315
|
try {
|
|
5316
|
+
if (opts.template) {
|
|
5317
|
+
const template = getTemplate(opts.template);
|
|
5318
|
+
if (!template) {
|
|
5319
|
+
console.error(chalk4.red(`Unknown template: ${opts.template}. Available: ${listTemplateNames().join(", ")}`));
|
|
5320
|
+
process.exit(1);
|
|
5321
|
+
}
|
|
5322
|
+
const projectId2 = resolveProject(opts.project);
|
|
5323
|
+
for (const input of template) {
|
|
5324
|
+
const s = createScenario({ ...input, projectId: projectId2 });
|
|
5325
|
+
console.log(chalk4.green(` Created ${s.shortId}: ${s.name}`));
|
|
5326
|
+
}
|
|
5327
|
+
return;
|
|
5328
|
+
}
|
|
5329
|
+
const projectId = resolveProject(opts.project);
|
|
3779
5330
|
const scenario = createScenario({
|
|
3780
5331
|
name,
|
|
3781
5332
|
description: opts.description || name,
|
|
@@ -3785,11 +5336,12 @@ program2.command("add <name>").description("Create a new test scenario").option(
|
|
|
3785
5336
|
model: opts.model,
|
|
3786
5337
|
targetPath: opts.path,
|
|
3787
5338
|
requiresAuth: opts.auth,
|
|
3788
|
-
timeoutMs: opts.timeout ? parseInt(opts.timeout, 10) : undefined
|
|
5339
|
+
timeoutMs: opts.timeout ? parseInt(opts.timeout, 10) : undefined,
|
|
5340
|
+
projectId
|
|
3789
5341
|
});
|
|
3790
|
-
console.log(
|
|
5342
|
+
console.log(chalk4.green(`Created scenario ${chalk4.bold(scenario.shortId)}: ${scenario.name}`));
|
|
3791
5343
|
} catch (error) {
|
|
3792
|
-
console.error(
|
|
5344
|
+
console.error(chalk4.red(`Error: ${error instanceof Error ? error.message : String(error)}`));
|
|
3793
5345
|
process.exit(1);
|
|
3794
5346
|
}
|
|
3795
5347
|
});
|
|
@@ -3803,7 +5355,7 @@ program2.command("list").description("List test scenarios").option("-t, --tag <t
|
|
|
3803
5355
|
});
|
|
3804
5356
|
console.log(formatScenarioList(scenarios));
|
|
3805
5357
|
} catch (error) {
|
|
3806
|
-
console.error(
|
|
5358
|
+
console.error(chalk4.red(`Error: ${error instanceof Error ? error.message : String(error)}`));
|
|
3807
5359
|
process.exit(1);
|
|
3808
5360
|
}
|
|
3809
5361
|
});
|
|
@@ -3811,33 +5363,33 @@ program2.command("show <id>").description("Show scenario details").action((id) =
|
|
|
3811
5363
|
try {
|
|
3812
5364
|
const scenario = getScenario(id) ?? getScenarioByShortId(id);
|
|
3813
5365
|
if (!scenario) {
|
|
3814
|
-
console.error(
|
|
5366
|
+
console.error(chalk4.red(`Scenario not found: ${id}`));
|
|
3815
5367
|
process.exit(1);
|
|
3816
5368
|
}
|
|
3817
5369
|
console.log("");
|
|
3818
|
-
console.log(
|
|
5370
|
+
console.log(chalk4.bold(` Scenario ${scenario.shortId}`));
|
|
3819
5371
|
console.log(` Name: ${scenario.name}`);
|
|
3820
|
-
console.log(` ID: ${
|
|
5372
|
+
console.log(` ID: ${chalk4.dim(scenario.id)}`);
|
|
3821
5373
|
console.log(` Description: ${scenario.description}`);
|
|
3822
5374
|
console.log(` Priority: ${scenario.priority}`);
|
|
3823
|
-
console.log(` Model: ${scenario.model ??
|
|
3824
|
-
console.log(` Tags: ${scenario.tags.length > 0 ? scenario.tags.join(", ") :
|
|
3825
|
-
console.log(` Path: ${scenario.targetPath ??
|
|
5375
|
+
console.log(` Model: ${scenario.model ?? chalk4.dim("default")}`);
|
|
5376
|
+
console.log(` Tags: ${scenario.tags.length > 0 ? scenario.tags.join(", ") : chalk4.dim("none")}`);
|
|
5377
|
+
console.log(` Path: ${scenario.targetPath ?? chalk4.dim("none")}`);
|
|
3826
5378
|
console.log(` Auth: ${scenario.requiresAuth ? "yes" : "no"}`);
|
|
3827
|
-
console.log(` Timeout: ${scenario.timeoutMs ? `${scenario.timeoutMs}ms` :
|
|
5379
|
+
console.log(` Timeout: ${scenario.timeoutMs ? `${scenario.timeoutMs}ms` : chalk4.dim("default")}`);
|
|
3828
5380
|
console.log(` Version: ${scenario.version}`);
|
|
3829
5381
|
console.log(` Created: ${scenario.createdAt}`);
|
|
3830
5382
|
console.log(` Updated: ${scenario.updatedAt}`);
|
|
3831
5383
|
if (scenario.steps.length > 0) {
|
|
3832
5384
|
console.log("");
|
|
3833
|
-
console.log(
|
|
5385
|
+
console.log(chalk4.bold(" Steps:"));
|
|
3834
5386
|
for (let i = 0;i < scenario.steps.length; i++) {
|
|
3835
5387
|
console.log(` ${i + 1}. ${scenario.steps[i]}`);
|
|
3836
5388
|
}
|
|
3837
5389
|
}
|
|
3838
5390
|
console.log("");
|
|
3839
5391
|
} catch (error) {
|
|
3840
|
-
console.error(
|
|
5392
|
+
console.error(chalk4.red(`Error: ${error instanceof Error ? error.message : String(error)}`));
|
|
3841
5393
|
process.exit(1);
|
|
3842
5394
|
}
|
|
3843
5395
|
});
|
|
@@ -3851,7 +5403,7 @@ program2.command("update <id>").description("Update a scenario").option("-n, --n
|
|
|
3851
5403
|
try {
|
|
3852
5404
|
const scenario = getScenario(id) ?? getScenarioByShortId(id);
|
|
3853
5405
|
if (!scenario) {
|
|
3854
|
-
console.error(
|
|
5406
|
+
console.error(chalk4.red(`Scenario not found: ${id}`));
|
|
3855
5407
|
process.exit(1);
|
|
3856
5408
|
}
|
|
3857
5409
|
const updated = updateScenario(scenario.id, {
|
|
@@ -3862,9 +5414,9 @@ program2.command("update <id>").description("Update a scenario").option("-n, --n
|
|
|
3862
5414
|
priority: opts.priority,
|
|
3863
5415
|
model: opts.model
|
|
3864
5416
|
}, scenario.version);
|
|
3865
|
-
console.log(
|
|
5417
|
+
console.log(chalk4.green(`Updated scenario ${chalk4.bold(updated.shortId)}: ${updated.name}`));
|
|
3866
5418
|
} catch (error) {
|
|
3867
|
-
console.error(
|
|
5419
|
+
console.error(chalk4.red(`Error: ${error instanceof Error ? error.message : String(error)}`));
|
|
3868
5420
|
process.exit(1);
|
|
3869
5421
|
}
|
|
3870
5422
|
});
|
|
@@ -3872,18 +5424,18 @@ program2.command("delete <id>").description("Delete a scenario").action((id) =>
|
|
|
3872
5424
|
try {
|
|
3873
5425
|
const scenario = getScenario(id) ?? getScenarioByShortId(id);
|
|
3874
5426
|
if (!scenario) {
|
|
3875
|
-
console.error(
|
|
5427
|
+
console.error(chalk4.red(`Scenario not found: ${id}`));
|
|
3876
5428
|
process.exit(1);
|
|
3877
5429
|
}
|
|
3878
5430
|
const deleted = deleteScenario(scenario.id);
|
|
3879
5431
|
if (deleted) {
|
|
3880
|
-
console.log(
|
|
5432
|
+
console.log(chalk4.green(`Deleted scenario ${scenario.shortId}: ${scenario.name}`));
|
|
3881
5433
|
} else {
|
|
3882
|
-
console.error(
|
|
5434
|
+
console.error(chalk4.red(`Failed to delete scenario: ${id}`));
|
|
3883
5435
|
process.exit(1);
|
|
3884
5436
|
}
|
|
3885
5437
|
} catch (error) {
|
|
3886
|
-
console.error(
|
|
5438
|
+
console.error(chalk4.red(`Error: ${error instanceof Error ? error.message : String(error)}`));
|
|
3887
5439
|
process.exit(1);
|
|
3888
5440
|
}
|
|
3889
5441
|
});
|
|
@@ -3892,12 +5444,13 @@ program2.command("run <url> [description]").description("Run test scenarios agai
|
|
|
3892
5444
|
return acc;
|
|
3893
5445
|
}, []).option("-s, --scenario <id>", "Run specific scenario ID").option("-p, --priority <level>", "Filter by priority").option("--headed", "Run browser in headed mode", false).option("-m, --model <model>", "AI model to use").option("--parallel <n>", "Number of parallel browsers", "1").option("--json", "Output results as JSON", false).option("-o, --output <filepath>", "Write JSON results to file").option("--timeout <ms>", "Timeout in milliseconds").option("--from-todos", "Import scenarios from todos before running", false).option("--project <id>", "Project ID").action(async (url, description, opts) => {
|
|
3894
5446
|
try {
|
|
5447
|
+
const projectId = resolveProject(opts.project);
|
|
3895
5448
|
if (description) {
|
|
3896
5449
|
const scenario = createScenario({
|
|
3897
5450
|
name: description,
|
|
3898
5451
|
description,
|
|
3899
5452
|
tags: ["ad-hoc"],
|
|
3900
|
-
projectId
|
|
5453
|
+
projectId
|
|
3901
5454
|
});
|
|
3902
5455
|
const { run: run2, results: results2 } = await runByFilter({
|
|
3903
5456
|
url,
|
|
@@ -3906,13 +5459,13 @@ program2.command("run <url> [description]").description("Run test scenarios agai
|
|
|
3906
5459
|
headed: opts.headed,
|
|
3907
5460
|
parallel: parseInt(opts.parallel, 10),
|
|
3908
5461
|
timeout: opts.timeout ? parseInt(opts.timeout, 10) : undefined,
|
|
3909
|
-
projectId
|
|
5462
|
+
projectId
|
|
3910
5463
|
});
|
|
3911
5464
|
if (opts.json || opts.output) {
|
|
3912
5465
|
const jsonOutput = formatJSON(run2, results2);
|
|
3913
5466
|
if (opts.output) {
|
|
3914
|
-
|
|
3915
|
-
console.log(
|
|
5467
|
+
writeFileSync3(opts.output, jsonOutput, "utf-8");
|
|
5468
|
+
console.log(chalk4.green(`Results written to ${opts.output}`));
|
|
3916
5469
|
}
|
|
3917
5470
|
if (opts.json) {
|
|
3918
5471
|
console.log(jsonOutput);
|
|
@@ -3923,8 +5476,8 @@ program2.command("run <url> [description]").description("Run test scenarios agai
|
|
|
3923
5476
|
process.exit(getExitCode(run2));
|
|
3924
5477
|
}
|
|
3925
5478
|
if (opts.fromTodos) {
|
|
3926
|
-
const result = importFromTodos({ projectId
|
|
3927
|
-
console.log(
|
|
5479
|
+
const result = importFromTodos({ projectId });
|
|
5480
|
+
console.log(chalk4.blue(`Imported ${result.imported} scenarios from todos (${result.skipped} skipped)`));
|
|
3928
5481
|
}
|
|
3929
5482
|
const { run, results } = await runByFilter({
|
|
3930
5483
|
url,
|
|
@@ -3935,13 +5488,13 @@ program2.command("run <url> [description]").description("Run test scenarios agai
|
|
|
3935
5488
|
headed: opts.headed,
|
|
3936
5489
|
parallel: parseInt(opts.parallel, 10),
|
|
3937
5490
|
timeout: opts.timeout ? parseInt(opts.timeout, 10) : undefined,
|
|
3938
|
-
projectId
|
|
5491
|
+
projectId
|
|
3939
5492
|
});
|
|
3940
5493
|
if (opts.json || opts.output) {
|
|
3941
5494
|
const jsonOutput = formatJSON(run, results);
|
|
3942
5495
|
if (opts.output) {
|
|
3943
|
-
|
|
3944
|
-
console.log(
|
|
5496
|
+
writeFileSync3(opts.output, jsonOutput, "utf-8");
|
|
5497
|
+
console.log(chalk4.green(`Results written to ${opts.output}`));
|
|
3945
5498
|
}
|
|
3946
5499
|
if (opts.json) {
|
|
3947
5500
|
console.log(jsonOutput);
|
|
@@ -3951,7 +5504,7 @@ program2.command("run <url> [description]").description("Run test scenarios agai
|
|
|
3951
5504
|
}
|
|
3952
5505
|
process.exit(getExitCode(run));
|
|
3953
5506
|
} catch (error) {
|
|
3954
|
-
console.error(
|
|
5507
|
+
console.error(chalk4.red(`Error: ${error instanceof Error ? error.message : String(error)}`));
|
|
3955
5508
|
process.exit(1);
|
|
3956
5509
|
}
|
|
3957
5510
|
});
|
|
@@ -3963,7 +5516,7 @@ program2.command("runs").description("List past test runs").option("--status <st
|
|
|
3963
5516
|
});
|
|
3964
5517
|
console.log(formatRunList(runs));
|
|
3965
5518
|
} catch (error) {
|
|
3966
|
-
console.error(
|
|
5519
|
+
console.error(chalk4.red(`Error: ${error instanceof Error ? error.message : String(error)}`));
|
|
3967
5520
|
process.exit(1);
|
|
3968
5521
|
}
|
|
3969
5522
|
});
|
|
@@ -3971,13 +5524,13 @@ program2.command("results <run-id>").description("Show results for a test run").
|
|
|
3971
5524
|
try {
|
|
3972
5525
|
const run = getRun(runId);
|
|
3973
5526
|
if (!run) {
|
|
3974
|
-
console.error(
|
|
5527
|
+
console.error(chalk4.red(`Run not found: ${runId}`));
|
|
3975
5528
|
process.exit(1);
|
|
3976
5529
|
}
|
|
3977
5530
|
const results = getResultsByRun(run.id);
|
|
3978
5531
|
console.log(formatTerminal(run, results));
|
|
3979
5532
|
} catch (error) {
|
|
3980
|
-
console.error(
|
|
5533
|
+
console.error(chalk4.red(`Error: ${error instanceof Error ? error.message : String(error)}`));
|
|
3981
5534
|
process.exit(1);
|
|
3982
5535
|
}
|
|
3983
5536
|
});
|
|
@@ -3988,23 +5541,23 @@ program2.command("screenshots <id>").description("List screenshots for a run or
|
|
|
3988
5541
|
const results = getResultsByRun(run.id);
|
|
3989
5542
|
let total = 0;
|
|
3990
5543
|
console.log("");
|
|
3991
|
-
console.log(
|
|
5544
|
+
console.log(chalk4.bold(` Screenshots for run ${run.id.slice(0, 8)}`));
|
|
3992
5545
|
console.log("");
|
|
3993
5546
|
for (const result of results) {
|
|
3994
5547
|
const screenshots2 = listScreenshots(result.id);
|
|
3995
5548
|
if (screenshots2.length > 0) {
|
|
3996
5549
|
const scenario = getScenario(result.scenarioId);
|
|
3997
5550
|
const label = scenario ? `${scenario.shortId}: ${scenario.name}` : result.scenarioId.slice(0, 8);
|
|
3998
|
-
console.log(
|
|
5551
|
+
console.log(chalk4.bold(` ${label}`));
|
|
3999
5552
|
for (const ss of screenshots2) {
|
|
4000
|
-
console.log(` ${
|
|
5553
|
+
console.log(` ${chalk4.dim(String(ss.stepNumber).padStart(3, "0"))} ${ss.action} \u2014 ${chalk4.dim(ss.filePath)}`);
|
|
4001
5554
|
total++;
|
|
4002
5555
|
}
|
|
4003
5556
|
console.log("");
|
|
4004
5557
|
}
|
|
4005
5558
|
}
|
|
4006
5559
|
if (total === 0) {
|
|
4007
|
-
console.log(
|
|
5560
|
+
console.log(chalk4.dim(" No screenshots found."));
|
|
4008
5561
|
console.log("");
|
|
4009
5562
|
}
|
|
4010
5563
|
return;
|
|
@@ -4012,18 +5565,18 @@ program2.command("screenshots <id>").description("List screenshots for a run or
|
|
|
4012
5565
|
const screenshots = listScreenshots(id);
|
|
4013
5566
|
if (screenshots.length > 0) {
|
|
4014
5567
|
console.log("");
|
|
4015
|
-
console.log(
|
|
5568
|
+
console.log(chalk4.bold(` Screenshots for result ${id.slice(0, 8)}`));
|
|
4016
5569
|
console.log("");
|
|
4017
5570
|
for (const ss of screenshots) {
|
|
4018
|
-
console.log(` ${
|
|
5571
|
+
console.log(` ${chalk4.dim(String(ss.stepNumber).padStart(3, "0"))} ${ss.action} \u2014 ${chalk4.dim(ss.filePath)}`);
|
|
4019
5572
|
}
|
|
4020
5573
|
console.log("");
|
|
4021
5574
|
return;
|
|
4022
5575
|
}
|
|
4023
|
-
console.error(
|
|
5576
|
+
console.error(chalk4.red(`No screenshots found for: ${id}`));
|
|
4024
5577
|
process.exit(1);
|
|
4025
5578
|
} catch (error) {
|
|
4026
|
-
console.error(
|
|
5579
|
+
console.error(chalk4.red(`Error: ${error instanceof Error ? error.message : String(error)}`));
|
|
4027
5580
|
process.exit(1);
|
|
4028
5581
|
}
|
|
4029
5582
|
});
|
|
@@ -4032,12 +5585,12 @@ program2.command("import <dir>").description("Import markdown test files as scen
|
|
|
4032
5585
|
const absDir = resolve(dir);
|
|
4033
5586
|
const files = readdirSync(absDir).filter((f) => f.endsWith(".md"));
|
|
4034
5587
|
if (files.length === 0) {
|
|
4035
|
-
console.log(
|
|
5588
|
+
console.log(chalk4.dim("No .md files found in directory."));
|
|
4036
5589
|
return;
|
|
4037
5590
|
}
|
|
4038
5591
|
let imported = 0;
|
|
4039
5592
|
for (const file of files) {
|
|
4040
|
-
const content =
|
|
5593
|
+
const content = readFileSync4(join6(absDir, file), "utf-8");
|
|
4041
5594
|
const lines = content.split(`
|
|
4042
5595
|
`);
|
|
4043
5596
|
let name = file.replace(/\.md$/, "");
|
|
@@ -4062,13 +5615,13 @@ program2.command("import <dir>").description("Import markdown test files as scen
|
|
|
4062
5615
|
description: descriptionLines.join(" ") || name,
|
|
4063
5616
|
steps
|
|
4064
5617
|
});
|
|
4065
|
-
console.log(
|
|
5618
|
+
console.log(chalk4.green(` Imported ${chalk4.bold(scenario.shortId)}: ${scenario.name}`));
|
|
4066
5619
|
imported++;
|
|
4067
5620
|
}
|
|
4068
5621
|
console.log("");
|
|
4069
|
-
console.log(
|
|
5622
|
+
console.log(chalk4.green(`Imported ${imported} scenario(s) from ${absDir}`));
|
|
4070
5623
|
} catch (error) {
|
|
4071
|
-
console.error(
|
|
5624
|
+
console.error(chalk4.red(`Error: ${error instanceof Error ? error.message : String(error)}`));
|
|
4072
5625
|
process.exit(1);
|
|
4073
5626
|
}
|
|
4074
5627
|
});
|
|
@@ -4077,7 +5630,7 @@ program2.command("config").description("Show current configuration").action(() =
|
|
|
4077
5630
|
const config = loadConfig();
|
|
4078
5631
|
console.log(JSON.stringify(config, null, 2));
|
|
4079
5632
|
} catch (error) {
|
|
4080
|
-
console.error(
|
|
5633
|
+
console.error(chalk4.red(`Error: ${error instanceof Error ? error.message : String(error)}`));
|
|
4081
5634
|
process.exit(1);
|
|
4082
5635
|
}
|
|
4083
5636
|
});
|
|
@@ -4085,27 +5638,545 @@ program2.command("status").description("Show database and auth status").action((
|
|
|
4085
5638
|
try {
|
|
4086
5639
|
const config = loadConfig();
|
|
4087
5640
|
const hasApiKey = !!config.anthropicApiKey || !!process.env["ANTHROPIC_API_KEY"];
|
|
4088
|
-
const dbPath =
|
|
5641
|
+
const dbPath = join6(process.env["HOME"] ?? "~", ".testers", "testers.db");
|
|
4089
5642
|
console.log("");
|
|
4090
|
-
console.log(
|
|
5643
|
+
console.log(chalk4.bold(" Open Testers Status"));
|
|
4091
5644
|
console.log("");
|
|
4092
|
-
console.log(` ANTHROPIC_API_KEY: ${hasApiKey ?
|
|
5645
|
+
console.log(` ANTHROPIC_API_KEY: ${hasApiKey ? chalk4.green("set") : chalk4.red("not set")}`);
|
|
4093
5646
|
console.log(` Database: ${dbPath}`);
|
|
4094
5647
|
console.log(` Default model: ${config.defaultModel}`);
|
|
4095
5648
|
console.log(` Screenshots dir: ${config.screenshots.dir}`);
|
|
4096
5649
|
console.log("");
|
|
4097
5650
|
} catch (error) {
|
|
4098
|
-
console.error(
|
|
5651
|
+
console.error(chalk4.red(`Error: ${error instanceof Error ? error.message : String(error)}`));
|
|
4099
5652
|
process.exit(1);
|
|
4100
5653
|
}
|
|
4101
5654
|
});
|
|
4102
5655
|
program2.command("install-browser").description("Install Playwright Chromium browser").action(async () => {
|
|
4103
5656
|
try {
|
|
4104
|
-
console.log(
|
|
5657
|
+
console.log(chalk4.blue("Installing Playwright Chromium..."));
|
|
4105
5658
|
await installBrowser();
|
|
4106
|
-
console.log(
|
|
5659
|
+
console.log(chalk4.green("Browser installed successfully."));
|
|
5660
|
+
} catch (error) {
|
|
5661
|
+
console.error(chalk4.red(`Error: ${error instanceof Error ? error.message : String(error)}`));
|
|
5662
|
+
process.exit(1);
|
|
5663
|
+
}
|
|
5664
|
+
});
|
|
5665
|
+
var projectCmd = program2.command("project").description("Manage test projects");
|
|
5666
|
+
projectCmd.command("create <name>").description("Create a new project").option("--path <path>", "Project path").option("-d, --description <text>", "Project description").option("--prefix <prefix>", "Scenario prefix", "TST").action((name, opts) => {
|
|
5667
|
+
try {
|
|
5668
|
+
const project = createProject({
|
|
5669
|
+
name,
|
|
5670
|
+
path: opts.path,
|
|
5671
|
+
description: opts.description
|
|
5672
|
+
});
|
|
5673
|
+
console.log(chalk4.green(`Created project ${chalk4.bold(project.name)} (${project.id})`));
|
|
5674
|
+
} catch (error) {
|
|
5675
|
+
console.error(chalk4.red(`Error: ${error instanceof Error ? error.message : String(error)}`));
|
|
5676
|
+
process.exit(1);
|
|
5677
|
+
}
|
|
5678
|
+
});
|
|
5679
|
+
projectCmd.command("list").description("List all projects").action(() => {
|
|
5680
|
+
try {
|
|
5681
|
+
const projects = listProjects();
|
|
5682
|
+
if (projects.length === 0) {
|
|
5683
|
+
console.log(chalk4.dim("No projects found."));
|
|
5684
|
+
return;
|
|
5685
|
+
}
|
|
5686
|
+
console.log("");
|
|
5687
|
+
console.log(chalk4.bold(" Projects"));
|
|
5688
|
+
console.log("");
|
|
5689
|
+
console.log(` ${"ID".padEnd(38)} ${"Name".padEnd(24)} ${"Path".padEnd(30)} Created`);
|
|
5690
|
+
console.log(` ${"\u2500".repeat(38)} ${"\u2500".repeat(24)} ${"\u2500".repeat(30)} ${"\u2500".repeat(20)}`);
|
|
5691
|
+
for (const p of projects) {
|
|
5692
|
+
console.log(` ${p.id.padEnd(38)} ${p.name.padEnd(24)} ${(p.path ?? chalk4.dim("\u2014")).toString().padEnd(30)} ${p.createdAt}`);
|
|
5693
|
+
}
|
|
5694
|
+
console.log("");
|
|
5695
|
+
} catch (error) {
|
|
5696
|
+
console.error(chalk4.red(`Error: ${error instanceof Error ? error.message : String(error)}`));
|
|
5697
|
+
process.exit(1);
|
|
5698
|
+
}
|
|
5699
|
+
});
|
|
5700
|
+
projectCmd.command("show <id>").description("Show project details").action((id) => {
|
|
5701
|
+
try {
|
|
5702
|
+
const project = getProject(id);
|
|
5703
|
+
if (!project) {
|
|
5704
|
+
console.error(chalk4.red(`Project not found: ${id}`));
|
|
5705
|
+
process.exit(1);
|
|
5706
|
+
}
|
|
5707
|
+
console.log("");
|
|
5708
|
+
console.log(chalk4.bold(` Project: ${project.name}`));
|
|
5709
|
+
console.log(` ID: ${project.id}`);
|
|
5710
|
+
console.log(` Path: ${project.path ?? chalk4.dim("none")}`);
|
|
5711
|
+
console.log(` Description: ${project.description ?? chalk4.dim("none")}`);
|
|
5712
|
+
console.log(` Created: ${project.createdAt}`);
|
|
5713
|
+
console.log(` Updated: ${project.updatedAt}`);
|
|
5714
|
+
console.log("");
|
|
5715
|
+
} catch (error) {
|
|
5716
|
+
console.error(chalk4.red(`Error: ${error instanceof Error ? error.message : String(error)}`));
|
|
5717
|
+
process.exit(1);
|
|
5718
|
+
}
|
|
5719
|
+
});
|
|
5720
|
+
projectCmd.command("use <name>").description("Set active project (find or create)").action((name) => {
|
|
5721
|
+
try {
|
|
5722
|
+
const project = ensureProject(name, process.cwd());
|
|
5723
|
+
if (!existsSync7(CONFIG_DIR2)) {
|
|
5724
|
+
mkdirSync4(CONFIG_DIR2, { recursive: true });
|
|
5725
|
+
}
|
|
5726
|
+
let config = {};
|
|
5727
|
+
if (existsSync7(CONFIG_PATH2)) {
|
|
5728
|
+
try {
|
|
5729
|
+
config = JSON.parse(readFileSync4(CONFIG_PATH2, "utf-8"));
|
|
5730
|
+
} catch {}
|
|
5731
|
+
}
|
|
5732
|
+
config.activeProject = project.id;
|
|
5733
|
+
writeFileSync3(CONFIG_PATH2, JSON.stringify(config, null, 2), "utf-8");
|
|
5734
|
+
console.log(chalk4.green(`Active project set to ${chalk4.bold(project.name)} (${project.id})`));
|
|
5735
|
+
} catch (error) {
|
|
5736
|
+
console.error(chalk4.red(`Error: ${error instanceof Error ? error.message : String(error)}`));
|
|
5737
|
+
process.exit(1);
|
|
5738
|
+
}
|
|
5739
|
+
});
|
|
5740
|
+
var scheduleCmd = program2.command("schedule").description("Manage recurring test schedules");
|
|
5741
|
+
scheduleCmd.command("create <name>").description("Create a new schedule").requiredOption("--cron <expression>", "Cron expression").requiredOption("--url <url>", "Target URL").option("-t, --tag <tag>", "Tag filter (repeatable)", (val, acc) => {
|
|
5742
|
+
acc.push(val);
|
|
5743
|
+
return acc;
|
|
5744
|
+
}, []).option("-p, --priority <level>", "Priority filter").option("-m, --model <model>", "AI model to use").option("--parallel <n>", "Parallel browsers", "1").option("--headed", "Run in headed mode", false).option("--timeout <ms>", "Timeout in milliseconds").option("--project <id>", "Project ID").action((name, opts) => {
|
|
5745
|
+
try {
|
|
5746
|
+
const projectId = resolveProject(opts.project);
|
|
5747
|
+
const schedule = createSchedule({
|
|
5748
|
+
name,
|
|
5749
|
+
cronExpression: opts.cron,
|
|
5750
|
+
url: opts.url,
|
|
5751
|
+
scenarioFilter: {
|
|
5752
|
+
tags: opts.tag.length > 0 ? opts.tag : undefined,
|
|
5753
|
+
priority: opts.priority
|
|
5754
|
+
},
|
|
5755
|
+
model: opts.model,
|
|
5756
|
+
headed: opts.headed,
|
|
5757
|
+
parallel: parseInt(opts.parallel, 10),
|
|
5758
|
+
timeoutMs: opts.timeout ? parseInt(opts.timeout, 10) : undefined,
|
|
5759
|
+
projectId
|
|
5760
|
+
});
|
|
5761
|
+
console.log(chalk4.green(`Created schedule ${chalk4.bold(schedule.name)} (${schedule.id})`));
|
|
5762
|
+
if (schedule.nextRunAt) {
|
|
5763
|
+
console.log(chalk4.dim(` Next run at: ${schedule.nextRunAt}`));
|
|
5764
|
+
}
|
|
5765
|
+
} catch (error) {
|
|
5766
|
+
console.error(chalk4.red(`Error: ${error instanceof Error ? error.message : String(error)}`));
|
|
5767
|
+
process.exit(1);
|
|
5768
|
+
}
|
|
5769
|
+
});
|
|
5770
|
+
scheduleCmd.command("list").description("List schedules").option("--project <id>", "Filter by project ID").option("--enabled", "Show only enabled schedules").action((opts) => {
|
|
5771
|
+
try {
|
|
5772
|
+
const projectId = resolveProject(opts.project);
|
|
5773
|
+
const schedules = listSchedules({
|
|
5774
|
+
projectId,
|
|
5775
|
+
enabled: opts.enabled ? true : undefined
|
|
5776
|
+
});
|
|
5777
|
+
if (schedules.length === 0) {
|
|
5778
|
+
console.log(chalk4.dim("No schedules found."));
|
|
5779
|
+
return;
|
|
5780
|
+
}
|
|
5781
|
+
console.log("");
|
|
5782
|
+
console.log(chalk4.bold(" Schedules"));
|
|
5783
|
+
console.log("");
|
|
5784
|
+
console.log(` ${"Name".padEnd(20)} ${"Cron".padEnd(18)} ${"URL".padEnd(30)} ${"Enabled".padEnd(9)} ${"Next Run".padEnd(22)} Last Run`);
|
|
5785
|
+
console.log(` ${"\u2500".repeat(20)} ${"\u2500".repeat(18)} ${"\u2500".repeat(30)} ${"\u2500".repeat(9)} ${"\u2500".repeat(22)} ${"\u2500".repeat(22)}`);
|
|
5786
|
+
for (const s of schedules) {
|
|
5787
|
+
const enabled = s.enabled ? chalk4.green("yes") : chalk4.red("no");
|
|
5788
|
+
const nextRun = s.nextRunAt ?? chalk4.dim("\u2014");
|
|
5789
|
+
const lastRun = s.lastRunAt ?? chalk4.dim("\u2014");
|
|
5790
|
+
console.log(` ${s.name.padEnd(20)} ${s.cronExpression.padEnd(18)} ${s.url.padEnd(30)} ${enabled.toString().padEnd(9)} ${nextRun.toString().padEnd(22)} ${lastRun}`);
|
|
5791
|
+
}
|
|
5792
|
+
console.log("");
|
|
5793
|
+
} catch (error) {
|
|
5794
|
+
console.error(chalk4.red(`Error: ${error instanceof Error ? error.message : String(error)}`));
|
|
5795
|
+
process.exit(1);
|
|
5796
|
+
}
|
|
5797
|
+
});
|
|
5798
|
+
scheduleCmd.command("show <id>").description("Show schedule details").action((id) => {
|
|
5799
|
+
try {
|
|
5800
|
+
const schedule = getSchedule(id);
|
|
5801
|
+
if (!schedule) {
|
|
5802
|
+
console.error(chalk4.red(`Schedule not found: ${id}`));
|
|
5803
|
+
process.exit(1);
|
|
5804
|
+
}
|
|
5805
|
+
console.log("");
|
|
5806
|
+
console.log(chalk4.bold(` Schedule: ${schedule.name}`));
|
|
5807
|
+
console.log(` ID: ${schedule.id}`);
|
|
5808
|
+
console.log(` Cron: ${schedule.cronExpression}`);
|
|
5809
|
+
console.log(` URL: ${schedule.url}`);
|
|
5810
|
+
console.log(` Enabled: ${schedule.enabled ? chalk4.green("yes") : chalk4.red("no")}`);
|
|
5811
|
+
console.log(` Model: ${schedule.model ?? chalk4.dim("default")}`);
|
|
5812
|
+
console.log(` Headed: ${schedule.headed ? "yes" : "no"}`);
|
|
5813
|
+
console.log(` Parallel: ${schedule.parallel}`);
|
|
5814
|
+
console.log(` Timeout: ${schedule.timeoutMs ? `${schedule.timeoutMs}ms` : chalk4.dim("default")}`);
|
|
5815
|
+
console.log(` Project: ${schedule.projectId ?? chalk4.dim("none")}`);
|
|
5816
|
+
console.log(` Filter: ${JSON.stringify(schedule.scenarioFilter)}`);
|
|
5817
|
+
console.log(` Next run: ${schedule.nextRunAt ?? chalk4.dim("not scheduled")}`);
|
|
5818
|
+
console.log(` Last run: ${schedule.lastRunAt ?? chalk4.dim("never")}`);
|
|
5819
|
+
console.log(` Last run ID: ${schedule.lastRunId ?? chalk4.dim("none")}`);
|
|
5820
|
+
console.log(` Created: ${schedule.createdAt}`);
|
|
5821
|
+
console.log(` Updated: ${schedule.updatedAt}`);
|
|
5822
|
+
console.log("");
|
|
5823
|
+
} catch (error) {
|
|
5824
|
+
console.error(chalk4.red(`Error: ${error instanceof Error ? error.message : String(error)}`));
|
|
5825
|
+
process.exit(1);
|
|
5826
|
+
}
|
|
5827
|
+
});
|
|
5828
|
+
scheduleCmd.command("enable <id>").description("Enable a schedule").action((id) => {
|
|
5829
|
+
try {
|
|
5830
|
+
const schedule = updateSchedule(id, { enabled: true });
|
|
5831
|
+
console.log(chalk4.green(`Enabled schedule ${chalk4.bold(schedule.name)}`));
|
|
5832
|
+
} catch (error) {
|
|
5833
|
+
console.error(chalk4.red(`Error: ${error instanceof Error ? error.message : String(error)}`));
|
|
5834
|
+
process.exit(1);
|
|
5835
|
+
}
|
|
5836
|
+
});
|
|
5837
|
+
scheduleCmd.command("disable <id>").description("Disable a schedule").action((id) => {
|
|
5838
|
+
try {
|
|
5839
|
+
const schedule = updateSchedule(id, { enabled: false });
|
|
5840
|
+
console.log(chalk4.green(`Disabled schedule ${chalk4.bold(schedule.name)}`));
|
|
5841
|
+
} catch (error) {
|
|
5842
|
+
console.error(chalk4.red(`Error: ${error instanceof Error ? error.message : String(error)}`));
|
|
5843
|
+
process.exit(1);
|
|
5844
|
+
}
|
|
5845
|
+
});
|
|
5846
|
+
scheduleCmd.command("delete <id>").description("Delete a schedule").action((id) => {
|
|
5847
|
+
try {
|
|
5848
|
+
const deleted = deleteSchedule(id);
|
|
5849
|
+
if (deleted) {
|
|
5850
|
+
console.log(chalk4.green(`Deleted schedule: ${id}`));
|
|
5851
|
+
} else {
|
|
5852
|
+
console.error(chalk4.red(`Schedule not found: ${id}`));
|
|
5853
|
+
process.exit(1);
|
|
5854
|
+
}
|
|
5855
|
+
} catch (error) {
|
|
5856
|
+
console.error(chalk4.red(`Error: ${error instanceof Error ? error.message : String(error)}`));
|
|
5857
|
+
process.exit(1);
|
|
5858
|
+
}
|
|
5859
|
+
});
|
|
5860
|
+
scheduleCmd.command("run <id>").description("Manually trigger a schedule").option("--json", "Output results as JSON", false).action(async (id, opts) => {
|
|
5861
|
+
try {
|
|
5862
|
+
const schedule = getSchedule(id);
|
|
5863
|
+
if (!schedule) {
|
|
5864
|
+
console.error(chalk4.red(`Schedule not found: ${id}`));
|
|
5865
|
+
process.exit(1);
|
|
5866
|
+
return;
|
|
5867
|
+
}
|
|
5868
|
+
console.log(chalk4.blue(`Running schedule ${chalk4.bold(schedule.name)} against ${schedule.url}...`));
|
|
5869
|
+
const { run, results } = await runByFilter({
|
|
5870
|
+
url: schedule.url,
|
|
5871
|
+
tags: schedule.scenarioFilter.tags,
|
|
5872
|
+
priority: schedule.scenarioFilter.priority,
|
|
5873
|
+
scenarioIds: schedule.scenarioFilter.scenarioIds,
|
|
5874
|
+
model: schedule.model ?? undefined,
|
|
5875
|
+
headed: schedule.headed,
|
|
5876
|
+
parallel: schedule.parallel,
|
|
5877
|
+
timeout: schedule.timeoutMs ?? undefined,
|
|
5878
|
+
projectId: schedule.projectId ?? undefined
|
|
5879
|
+
});
|
|
5880
|
+
if (opts.json) {
|
|
5881
|
+
console.log(formatJSON(run, results));
|
|
5882
|
+
} else {
|
|
5883
|
+
console.log(formatTerminal(run, results));
|
|
5884
|
+
}
|
|
5885
|
+
process.exit(getExitCode(run));
|
|
5886
|
+
} catch (error) {
|
|
5887
|
+
console.error(chalk4.red(`Error: ${error instanceof Error ? error.message : String(error)}`));
|
|
5888
|
+
process.exit(1);
|
|
5889
|
+
}
|
|
5890
|
+
});
|
|
5891
|
+
program2.command("daemon").description("Start the scheduler daemon").option("--interval <seconds>", "Check interval in seconds", "60").action(async (opts) => {
|
|
5892
|
+
try {
|
|
5893
|
+
const intervalMs = parseInt(opts.interval, 10) * 1000;
|
|
5894
|
+
console.log(chalk4.blue("Scheduler daemon started. Press Ctrl+C to stop."));
|
|
5895
|
+
console.log(chalk4.dim(` Check interval: ${opts.interval}s`));
|
|
5896
|
+
let running = true;
|
|
5897
|
+
const checkAndRun = async () => {
|
|
5898
|
+
while (running) {
|
|
5899
|
+
try {
|
|
5900
|
+
const schedules = listSchedules({ enabled: true });
|
|
5901
|
+
const now2 = new Date().toISOString();
|
|
5902
|
+
for (const schedule of schedules) {
|
|
5903
|
+
if (schedule.nextRunAt && schedule.nextRunAt <= now2) {
|
|
5904
|
+
console.log(chalk4.blue(`[${new Date().toISOString()}] Triggering schedule: ${schedule.name}`));
|
|
5905
|
+
try {
|
|
5906
|
+
const { run } = await runByFilter({
|
|
5907
|
+
url: schedule.url,
|
|
5908
|
+
tags: schedule.scenarioFilter.tags,
|
|
5909
|
+
priority: schedule.scenarioFilter.priority,
|
|
5910
|
+
scenarioIds: schedule.scenarioFilter.scenarioIds,
|
|
5911
|
+
model: schedule.model ?? undefined,
|
|
5912
|
+
headed: schedule.headed,
|
|
5913
|
+
parallel: schedule.parallel,
|
|
5914
|
+
timeout: schedule.timeoutMs ?? undefined,
|
|
5915
|
+
projectId: schedule.projectId ?? undefined
|
|
5916
|
+
});
|
|
5917
|
+
const statusColor = run.status === "passed" ? chalk4.green : chalk4.red;
|
|
5918
|
+
console.log(` ${statusColor(run.status)} \u2014 ${run.passed}/${run.total} passed`);
|
|
5919
|
+
updateSchedule(schedule.id, {});
|
|
5920
|
+
} catch (err) {
|
|
5921
|
+
console.error(chalk4.red(` Error running schedule ${schedule.name}: ${err instanceof Error ? err.message : String(err)}`));
|
|
5922
|
+
}
|
|
5923
|
+
}
|
|
5924
|
+
}
|
|
5925
|
+
} catch (err) {
|
|
5926
|
+
console.error(chalk4.red(`Daemon error: ${err instanceof Error ? err.message : String(err)}`));
|
|
5927
|
+
}
|
|
5928
|
+
await new Promise((resolve2) => setTimeout(resolve2, intervalMs));
|
|
5929
|
+
}
|
|
5930
|
+
};
|
|
5931
|
+
process.on("SIGINT", () => {
|
|
5932
|
+
console.log(chalk4.yellow(`
|
|
5933
|
+
Shutting down scheduler daemon...`));
|
|
5934
|
+
running = false;
|
|
5935
|
+
process.exit(0);
|
|
5936
|
+
});
|
|
5937
|
+
process.on("SIGTERM", () => {
|
|
5938
|
+
console.log(chalk4.yellow(`
|
|
5939
|
+
Shutting down scheduler daemon...`));
|
|
5940
|
+
running = false;
|
|
5941
|
+
process.exit(0);
|
|
5942
|
+
});
|
|
5943
|
+
await checkAndRun();
|
|
5944
|
+
} catch (error) {
|
|
5945
|
+
console.error(chalk4.red(`Error: ${error instanceof Error ? error.message : String(error)}`));
|
|
5946
|
+
process.exit(1);
|
|
5947
|
+
}
|
|
5948
|
+
});
|
|
5949
|
+
program2.command("init").description("Initialize a new testing project").option("-n, --name <name>", "Project name").option("-u, --url <url>", "Base URL").option("-p, --path <path>", "Project path").action((opts) => {
|
|
5950
|
+
try {
|
|
5951
|
+
const { project, scenarios, framework } = initProject({
|
|
5952
|
+
name: opts.name,
|
|
5953
|
+
url: opts.url,
|
|
5954
|
+
path: opts.path
|
|
5955
|
+
});
|
|
5956
|
+
console.log("");
|
|
5957
|
+
console.log(chalk4.bold(" Project initialized!"));
|
|
5958
|
+
console.log("");
|
|
5959
|
+
if (framework) {
|
|
5960
|
+
console.log(` Framework: ${chalk4.cyan(framework.name)}`);
|
|
5961
|
+
if (framework.features.length > 0) {
|
|
5962
|
+
console.log(` Features: ${chalk4.dim(framework.features.join(", "))}`);
|
|
5963
|
+
}
|
|
5964
|
+
} else {
|
|
5965
|
+
console.log(` Framework: ${chalk4.dim("not detected")}`);
|
|
5966
|
+
}
|
|
5967
|
+
console.log(` Project: ${chalk4.green(project.name)} ${chalk4.dim(`(${project.id})`)}`);
|
|
5968
|
+
console.log(` Scenarios: ${chalk4.green(String(scenarios.length))} starter scenarios created`);
|
|
5969
|
+
console.log("");
|
|
5970
|
+
for (const s of scenarios) {
|
|
5971
|
+
console.log(` ${chalk4.dim(s.shortId)} ${s.name} ${chalk4.dim(`[${s.tags.join(", ")}]`)}`);
|
|
5972
|
+
}
|
|
5973
|
+
console.log("");
|
|
5974
|
+
console.log(chalk4.bold(" Next steps:"));
|
|
5975
|
+
console.log(` 1. Start your dev server`);
|
|
5976
|
+
console.log(` 2. Run ${chalk4.cyan("testers run <url>")} to execute tests`);
|
|
5977
|
+
console.log(` 3. Add more scenarios with ${chalk4.cyan("testers add <name>")}`);
|
|
5978
|
+
console.log("");
|
|
5979
|
+
} catch (error) {
|
|
5980
|
+
console.error(chalk4.red(`Error: ${error instanceof Error ? error.message : String(error)}`));
|
|
5981
|
+
process.exit(1);
|
|
5982
|
+
}
|
|
5983
|
+
});
|
|
5984
|
+
program2.command("replay <run-id>").description("Re-run all scenarios from a previous run").option("-u, --url <url>", "Override URL").option("-m, --model <model>", "Override model").option("--headed", "Run headed", false).option("--json", "JSON output", false).option("--parallel <n>", "Parallel count", "1").action(async (runId, opts) => {
|
|
5985
|
+
try {
|
|
5986
|
+
const originalRun = getRun(runId);
|
|
5987
|
+
if (!originalRun) {
|
|
5988
|
+
console.error(chalk4.red(`Run not found: ${runId}`));
|
|
5989
|
+
process.exit(1);
|
|
5990
|
+
}
|
|
5991
|
+
const originalResults = getResultsByRun(originalRun.id);
|
|
5992
|
+
const scenarioIds = originalResults.map((r) => r.scenarioId);
|
|
5993
|
+
if (scenarioIds.length === 0) {
|
|
5994
|
+
console.log(chalk4.dim("No scenarios to replay."));
|
|
5995
|
+
return;
|
|
5996
|
+
}
|
|
5997
|
+
console.log(chalk4.blue(`Replaying ${scenarioIds.length} scenarios from run ${originalRun.id.slice(0, 8)}...`));
|
|
5998
|
+
const { run, results } = await runByFilter({
|
|
5999
|
+
url: opts.url ?? originalRun.url,
|
|
6000
|
+
scenarioIds,
|
|
6001
|
+
model: opts.model,
|
|
6002
|
+
headed: opts.headed,
|
|
6003
|
+
parallel: parseInt(opts.parallel, 10)
|
|
6004
|
+
});
|
|
6005
|
+
if (opts.json) {
|
|
6006
|
+
console.log(formatJSON(run, results));
|
|
6007
|
+
} else {
|
|
6008
|
+
console.log(formatTerminal(run, results));
|
|
6009
|
+
}
|
|
6010
|
+
process.exit(getExitCode(run));
|
|
6011
|
+
} catch (error) {
|
|
6012
|
+
console.error(chalk4.red(`Error: ${error instanceof Error ? error.message : String(error)}`));
|
|
6013
|
+
process.exit(1);
|
|
6014
|
+
}
|
|
6015
|
+
});
|
|
6016
|
+
program2.command("retry <run-id>").description("Re-run only failed scenarios from a previous run").option("-u, --url <url>", "Override URL").option("-m, --model <model>", "Override model").option("--headed", "Run headed", false).option("--json", "JSON output", false).option("--parallel <n>", "Parallel count", "1").action(async (runId, opts) => {
|
|
6017
|
+
try {
|
|
6018
|
+
const originalRun = getRun(runId);
|
|
6019
|
+
if (!originalRun) {
|
|
6020
|
+
console.error(chalk4.red(`Run not found: ${runId}`));
|
|
6021
|
+
process.exit(1);
|
|
6022
|
+
}
|
|
6023
|
+
const originalResults = getResultsByRun(originalRun.id);
|
|
6024
|
+
const failedScenarioIds = originalResults.filter((r) => r.status === "failed" || r.status === "error").map((r) => r.scenarioId);
|
|
6025
|
+
if (failedScenarioIds.length === 0) {
|
|
6026
|
+
console.log(chalk4.green("No failed scenarios to retry. All passed!"));
|
|
6027
|
+
return;
|
|
6028
|
+
}
|
|
6029
|
+
console.log(chalk4.blue(`Retrying ${failedScenarioIds.length} failed scenarios from run ${originalRun.id.slice(0, 8)}...`));
|
|
6030
|
+
const { run, results } = await runByFilter({
|
|
6031
|
+
url: opts.url ?? originalRun.url,
|
|
6032
|
+
scenarioIds: failedScenarioIds,
|
|
6033
|
+
model: opts.model,
|
|
6034
|
+
headed: opts.headed,
|
|
6035
|
+
parallel: parseInt(opts.parallel, 10)
|
|
6036
|
+
});
|
|
6037
|
+
if (!opts.json) {
|
|
6038
|
+
console.log("");
|
|
6039
|
+
console.log(chalk4.bold(" Comparison with original run:"));
|
|
6040
|
+
for (const result of results) {
|
|
6041
|
+
const original = originalResults.find((r) => r.scenarioId === result.scenarioId);
|
|
6042
|
+
if (original) {
|
|
6043
|
+
const changed = original.status !== result.status;
|
|
6044
|
+
const arrow = changed ? chalk4.yellow(`${original.status} \u2192 ${result.status}`) : chalk4.dim(`${result.status} (unchanged)`);
|
|
6045
|
+
const icon = result.status === "passed" ? chalk4.green("\u2713") : chalk4.red("\u2717");
|
|
6046
|
+
console.log(` ${icon} ${result.scenarioId.slice(0, 8)}: ${arrow}`);
|
|
6047
|
+
}
|
|
6048
|
+
}
|
|
6049
|
+
console.log("");
|
|
6050
|
+
}
|
|
6051
|
+
if (opts.json) {
|
|
6052
|
+
console.log(formatJSON(run, results));
|
|
6053
|
+
} else {
|
|
6054
|
+
console.log(formatTerminal(run, results));
|
|
6055
|
+
}
|
|
6056
|
+
process.exit(getExitCode(run));
|
|
6057
|
+
} catch (error) {
|
|
6058
|
+
console.error(chalk4.red(`Error: ${error instanceof Error ? error.message : String(error)}`));
|
|
6059
|
+
process.exit(1);
|
|
6060
|
+
}
|
|
6061
|
+
});
|
|
6062
|
+
program2.command("smoke <url>").description("Run autonomous smoke test").option("-m, --model <model>", "AI model").option("--headed", "Watch browser", false).option("--timeout <ms>", "Timeout in milliseconds").option("--json", "JSON output", false).option("--project <id>", "Project ID").action(async (url, opts) => {
|
|
6063
|
+
try {
|
|
6064
|
+
const projectId = resolveProject(opts.project);
|
|
6065
|
+
console.log(chalk4.blue(`Running smoke test against ${chalk4.bold(url)}...`));
|
|
6066
|
+
console.log("");
|
|
6067
|
+
const smokeResult = await runSmoke({
|
|
6068
|
+
url,
|
|
6069
|
+
model: opts.model,
|
|
6070
|
+
headed: opts.headed,
|
|
6071
|
+
timeout: opts.timeout ? parseInt(opts.timeout, 10) : undefined,
|
|
6072
|
+
projectId
|
|
6073
|
+
});
|
|
6074
|
+
if (opts.json) {
|
|
6075
|
+
console.log(JSON.stringify({
|
|
6076
|
+
run: smokeResult.run,
|
|
6077
|
+
result: smokeResult.result,
|
|
6078
|
+
pagesVisited: smokeResult.pagesVisited,
|
|
6079
|
+
issues: smokeResult.issuesFound
|
|
6080
|
+
}, null, 2));
|
|
6081
|
+
} else {
|
|
6082
|
+
console.log(formatSmokeReport(smokeResult));
|
|
6083
|
+
}
|
|
6084
|
+
const hasCritical = smokeResult.issuesFound.some((i) => i.severity === "critical" || i.severity === "high");
|
|
6085
|
+
process.exit(hasCritical ? 1 : 0);
|
|
6086
|
+
} catch (error) {
|
|
6087
|
+
console.error(chalk4.red(`Error: ${error instanceof Error ? error.message : String(error)}`));
|
|
6088
|
+
process.exit(1);
|
|
6089
|
+
}
|
|
6090
|
+
});
|
|
6091
|
+
program2.command("diff <run1> <run2>").description("Compare two test runs").option("--json", "JSON output", false).action((run1, run2, opts) => {
|
|
6092
|
+
try {
|
|
6093
|
+
const diff = diffRuns(run1, run2);
|
|
6094
|
+
if (opts.json) {
|
|
6095
|
+
console.log(formatDiffJSON(diff));
|
|
6096
|
+
} else {
|
|
6097
|
+
console.log(formatDiffTerminal(diff));
|
|
6098
|
+
}
|
|
6099
|
+
process.exit(diff.regressions.length > 0 ? 1 : 0);
|
|
6100
|
+
} catch (error) {
|
|
6101
|
+
console.error(chalk4.red(`Error: ${error instanceof Error ? error.message : String(error)}`));
|
|
6102
|
+
process.exit(1);
|
|
6103
|
+
}
|
|
6104
|
+
});
|
|
6105
|
+
program2.command("report [run-id]").description("Generate HTML test report").option("--latest", "Use most recent run", false).option("-o, --output <file>", "Output file path", "report.html").action((runId, opts) => {
|
|
6106
|
+
try {
|
|
6107
|
+
let html;
|
|
6108
|
+
if (opts.latest || !runId) {
|
|
6109
|
+
html = generateLatestReport();
|
|
6110
|
+
} else {
|
|
6111
|
+
html = generateHtmlReport(runId);
|
|
6112
|
+
}
|
|
6113
|
+
writeFileSync3(opts.output, html, "utf-8");
|
|
6114
|
+
console.log(chalk4.green(`Report generated: ${opts.output}`));
|
|
6115
|
+
} catch (error) {
|
|
6116
|
+
console.error(chalk4.red(`Error: ${error instanceof Error ? error.message : String(error)}`));
|
|
6117
|
+
process.exit(1);
|
|
6118
|
+
}
|
|
6119
|
+
});
|
|
6120
|
+
var authCmd = program2.command("auth").description("Manage auth presets");
|
|
6121
|
+
authCmd.command("add <name>").description("Create an auth preset").requiredOption("--email <email>", "Login email").requiredOption("--password <password>", "Login password").option("--login-path <path>", "Login page path", "/login").action((name, opts) => {
|
|
6122
|
+
try {
|
|
6123
|
+
const preset = createAuthPreset({
|
|
6124
|
+
name,
|
|
6125
|
+
email: opts.email,
|
|
6126
|
+
password: opts.password,
|
|
6127
|
+
loginPath: opts.loginPath
|
|
6128
|
+
});
|
|
6129
|
+
console.log(chalk4.green(`Created auth preset ${chalk4.bold(preset.name)} (${preset.email})`));
|
|
6130
|
+
} catch (error) {
|
|
6131
|
+
console.error(chalk4.red(`Error: ${error instanceof Error ? error.message : String(error)}`));
|
|
6132
|
+
process.exit(1);
|
|
6133
|
+
}
|
|
6134
|
+
});
|
|
6135
|
+
authCmd.command("list").description("List auth presets").action(() => {
|
|
6136
|
+
try {
|
|
6137
|
+
const presets = listAuthPresets();
|
|
6138
|
+
if (presets.length === 0) {
|
|
6139
|
+
console.log(chalk4.dim("No auth presets found."));
|
|
6140
|
+
return;
|
|
6141
|
+
}
|
|
6142
|
+
console.log("");
|
|
6143
|
+
console.log(chalk4.bold(" Auth Presets"));
|
|
6144
|
+
console.log("");
|
|
6145
|
+
console.log(` ${"Name".padEnd(20)} ${"Email".padEnd(30)} ${"Login Path".padEnd(15)} Created`);
|
|
6146
|
+
console.log(` ${"\u2500".repeat(20)} ${"\u2500".repeat(30)} ${"\u2500".repeat(15)} ${"\u2500".repeat(22)}`);
|
|
6147
|
+
for (const p of presets) {
|
|
6148
|
+
console.log(` ${p.name.padEnd(20)} ${p.email.padEnd(30)} ${p.loginPath.padEnd(15)} ${p.createdAt}`);
|
|
6149
|
+
}
|
|
6150
|
+
console.log("");
|
|
6151
|
+
} catch (error) {
|
|
6152
|
+
console.error(chalk4.red(`Error: ${error instanceof Error ? error.message : String(error)}`));
|
|
6153
|
+
process.exit(1);
|
|
6154
|
+
}
|
|
6155
|
+
});
|
|
6156
|
+
authCmd.command("delete <name>").description("Delete an auth preset").action((name) => {
|
|
6157
|
+
try {
|
|
6158
|
+
const deleted = deleteAuthPreset(name);
|
|
6159
|
+
if (deleted) {
|
|
6160
|
+
console.log(chalk4.green(`Deleted auth preset: ${name}`));
|
|
6161
|
+
} else {
|
|
6162
|
+
console.error(chalk4.red(`Auth preset not found: ${name}`));
|
|
6163
|
+
process.exit(1);
|
|
6164
|
+
}
|
|
6165
|
+
} catch (error) {
|
|
6166
|
+
console.error(chalk4.red(`Error: ${error instanceof Error ? error.message : String(error)}`));
|
|
6167
|
+
process.exit(1);
|
|
6168
|
+
}
|
|
6169
|
+
});
|
|
6170
|
+
program2.command("costs").description("Show cost tracking and budget status").option("--project <id>", "Project ID").option("--period <period>", "Time period", "month").option("--json", "JSON output", false).action((opts) => {
|
|
6171
|
+
try {
|
|
6172
|
+
const summary = getCostSummary({ projectId: resolveProject(opts.project), period: opts.period });
|
|
6173
|
+
if (opts.json) {
|
|
6174
|
+
console.log(formatCostsJSON(summary));
|
|
6175
|
+
} else {
|
|
6176
|
+
console.log(formatCostsTerminal(summary));
|
|
6177
|
+
}
|
|
4107
6178
|
} catch (error) {
|
|
4108
|
-
console.error(
|
|
6179
|
+
console.error(chalk4.red(`Error: ${error instanceof Error ? error.message : String(error)}`));
|
|
4109
6180
|
process.exit(1);
|
|
4110
6181
|
}
|
|
4111
6182
|
});
|