@hasna/testers 0.0.2 → 0.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +72 -1
- package/dist/cli/index.js +1769 -447
- package/dist/db/auth-presets.d.ts +20 -0
- package/dist/db/auth-presets.d.ts.map +1 -0
- package/dist/db/database.d.ts.map +1 -1
- package/dist/db/screenshots.d.ts +3 -0
- package/dist/db/screenshots.d.ts.map +1 -1
- package/dist/index.d.ts +16 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1741 -358
- package/dist/lib/ai-client.d.ts +6 -0
- package/dist/lib/ai-client.d.ts.map +1 -1
- package/dist/lib/costs.d.ts +36 -0
- package/dist/lib/costs.d.ts.map +1 -0
- package/dist/lib/diff.d.ts +25 -0
- package/dist/lib/diff.d.ts.map +1 -0
- package/dist/lib/init.d.ts +28 -0
- package/dist/lib/init.d.ts.map +1 -0
- package/dist/lib/report.d.ts +4 -0
- package/dist/lib/report.d.ts.map +1 -0
- package/dist/lib/runner.d.ts.map +1 -1
- package/dist/lib/screenshotter.d.ts +27 -25
- package/dist/lib/screenshotter.d.ts.map +1 -1
- package/dist/lib/smoke.d.ts +25 -0
- package/dist/lib/smoke.d.ts.map +1 -0
- package/dist/lib/templates.d.ts +5 -0
- package/dist/lib/templates.d.ts.map +1 -0
- package/dist/lib/watch.d.ts +9 -0
- package/dist/lib/watch.d.ts.map +1 -0
- package/dist/lib/webhooks.d.ts +41 -0
- package/dist/lib/webhooks.d.ts.map +1 -0
- package/dist/mcp/index.js +138 -18
- package/dist/server/index.js +138 -18
- package/dist/types/index.d.ts +6 -0
- package/dist/types/index.d.ts.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -1,10 +1,17 @@
|
|
|
1
1
|
// @bun
|
|
2
|
-
|
|
3
|
-
var
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
2
|
+
var __defProp = Object.defineProperty;
|
|
3
|
+
var __export = (target, all) => {
|
|
4
|
+
for (var name in all)
|
|
5
|
+
__defProp(target, name, {
|
|
6
|
+
get: all[name],
|
|
7
|
+
enumerable: true,
|
|
8
|
+
configurable: true,
|
|
9
|
+
set: (newValue) => all[name] = () => newValue
|
|
10
|
+
});
|
|
7
11
|
};
|
|
12
|
+
var __esm = (fn, res) => () => (fn && (res = fn(fn = 0)), res);
|
|
13
|
+
|
|
14
|
+
// src/types/index.ts
|
|
8
15
|
function projectFromRow(row) {
|
|
9
16
|
return {
|
|
10
17
|
id: row.id,
|
|
@@ -91,7 +98,10 @@ function screenshotFromRow(row) {
|
|
|
91
98
|
filePath: row.file_path,
|
|
92
99
|
width: row.width,
|
|
93
100
|
height: row.height,
|
|
94
|
-
timestamp: row.timestamp
|
|
101
|
+
timestamp: row.timestamp,
|
|
102
|
+
description: row.description,
|
|
103
|
+
pageUrl: row.page_url,
|
|
104
|
+
thumbnailPath: row.thumbnail_path
|
|
95
105
|
};
|
|
96
106
|
}
|
|
97
107
|
function scheduleFromRow(row) {
|
|
@@ -114,82 +124,80 @@ function scheduleFromRow(row) {
|
|
|
114
124
|
updatedAt: row.updated_at
|
|
115
125
|
};
|
|
116
126
|
}
|
|
127
|
+
var MODEL_MAP, ScenarioNotFoundError, RunNotFoundError, ResultNotFoundError, VersionConflictError, BrowserError, AIClientError, TodosConnectionError, ProjectNotFoundError, AgentNotFoundError, ScheduleNotFoundError;
|
|
128
|
+
var init_types = __esm(() => {
|
|
129
|
+
MODEL_MAP = {
|
|
130
|
+
quick: "claude-haiku-4-5-20251001",
|
|
131
|
+
thorough: "claude-sonnet-4-6-20260311",
|
|
132
|
+
deep: "claude-opus-4-6-20260311"
|
|
133
|
+
};
|
|
134
|
+
ScenarioNotFoundError = class ScenarioNotFoundError extends Error {
|
|
135
|
+
constructor(id) {
|
|
136
|
+
super(`Scenario not found: ${id}`);
|
|
137
|
+
this.name = "ScenarioNotFoundError";
|
|
138
|
+
}
|
|
139
|
+
};
|
|
140
|
+
RunNotFoundError = class RunNotFoundError extends Error {
|
|
141
|
+
constructor(id) {
|
|
142
|
+
super(`Run not found: ${id}`);
|
|
143
|
+
this.name = "RunNotFoundError";
|
|
144
|
+
}
|
|
145
|
+
};
|
|
146
|
+
ResultNotFoundError = class ResultNotFoundError extends Error {
|
|
147
|
+
constructor(id) {
|
|
148
|
+
super(`Result not found: ${id}`);
|
|
149
|
+
this.name = "ResultNotFoundError";
|
|
150
|
+
}
|
|
151
|
+
};
|
|
152
|
+
VersionConflictError = class VersionConflictError extends Error {
|
|
153
|
+
constructor(entity, id) {
|
|
154
|
+
super(`Version conflict on ${entity}: ${id}`);
|
|
155
|
+
this.name = "VersionConflictError";
|
|
156
|
+
}
|
|
157
|
+
};
|
|
158
|
+
BrowserError = class BrowserError extends Error {
|
|
159
|
+
constructor(message) {
|
|
160
|
+
super(message);
|
|
161
|
+
this.name = "BrowserError";
|
|
162
|
+
}
|
|
163
|
+
};
|
|
164
|
+
AIClientError = class AIClientError extends Error {
|
|
165
|
+
constructor(message) {
|
|
166
|
+
super(message);
|
|
167
|
+
this.name = "AIClientError";
|
|
168
|
+
}
|
|
169
|
+
};
|
|
170
|
+
TodosConnectionError = class TodosConnectionError extends Error {
|
|
171
|
+
constructor(message) {
|
|
172
|
+
super(message);
|
|
173
|
+
this.name = "TodosConnectionError";
|
|
174
|
+
}
|
|
175
|
+
};
|
|
176
|
+
ProjectNotFoundError = class ProjectNotFoundError extends Error {
|
|
177
|
+
constructor(id) {
|
|
178
|
+
super(`Project not found: ${id}`);
|
|
179
|
+
this.name = "ProjectNotFoundError";
|
|
180
|
+
}
|
|
181
|
+
};
|
|
182
|
+
AgentNotFoundError = class AgentNotFoundError extends Error {
|
|
183
|
+
constructor(id) {
|
|
184
|
+
super(`Agent not found: ${id}`);
|
|
185
|
+
this.name = "AgentNotFoundError";
|
|
186
|
+
}
|
|
187
|
+
};
|
|
188
|
+
ScheduleNotFoundError = class ScheduleNotFoundError extends Error {
|
|
189
|
+
constructor(id) {
|
|
190
|
+
super(`Schedule not found: ${id}`);
|
|
191
|
+
this.name = "ScheduleNotFoundError";
|
|
192
|
+
}
|
|
193
|
+
};
|
|
194
|
+
});
|
|
117
195
|
|
|
118
|
-
class ScenarioNotFoundError extends Error {
|
|
119
|
-
constructor(id) {
|
|
120
|
-
super(`Scenario not found: ${id}`);
|
|
121
|
-
this.name = "ScenarioNotFoundError";
|
|
122
|
-
}
|
|
123
|
-
}
|
|
124
|
-
|
|
125
|
-
class RunNotFoundError extends Error {
|
|
126
|
-
constructor(id) {
|
|
127
|
-
super(`Run not found: ${id}`);
|
|
128
|
-
this.name = "RunNotFoundError";
|
|
129
|
-
}
|
|
130
|
-
}
|
|
131
|
-
|
|
132
|
-
class ResultNotFoundError extends Error {
|
|
133
|
-
constructor(id) {
|
|
134
|
-
super(`Result not found: ${id}`);
|
|
135
|
-
this.name = "ResultNotFoundError";
|
|
136
|
-
}
|
|
137
|
-
}
|
|
138
|
-
|
|
139
|
-
class VersionConflictError extends Error {
|
|
140
|
-
constructor(entity, id) {
|
|
141
|
-
super(`Version conflict on ${entity}: ${id}`);
|
|
142
|
-
this.name = "VersionConflictError";
|
|
143
|
-
}
|
|
144
|
-
}
|
|
145
|
-
|
|
146
|
-
class BrowserError extends Error {
|
|
147
|
-
constructor(message) {
|
|
148
|
-
super(message);
|
|
149
|
-
this.name = "BrowserError";
|
|
150
|
-
}
|
|
151
|
-
}
|
|
152
|
-
|
|
153
|
-
class AIClientError extends Error {
|
|
154
|
-
constructor(message) {
|
|
155
|
-
super(message);
|
|
156
|
-
this.name = "AIClientError";
|
|
157
|
-
}
|
|
158
|
-
}
|
|
159
|
-
|
|
160
|
-
class TodosConnectionError extends Error {
|
|
161
|
-
constructor(message) {
|
|
162
|
-
super(message);
|
|
163
|
-
this.name = "TodosConnectionError";
|
|
164
|
-
}
|
|
165
|
-
}
|
|
166
|
-
|
|
167
|
-
class ProjectNotFoundError extends Error {
|
|
168
|
-
constructor(id) {
|
|
169
|
-
super(`Project not found: ${id}`);
|
|
170
|
-
this.name = "ProjectNotFoundError";
|
|
171
|
-
}
|
|
172
|
-
}
|
|
173
|
-
|
|
174
|
-
class AgentNotFoundError extends Error {
|
|
175
|
-
constructor(id) {
|
|
176
|
-
super(`Agent not found: ${id}`);
|
|
177
|
-
this.name = "AgentNotFoundError";
|
|
178
|
-
}
|
|
179
|
-
}
|
|
180
|
-
|
|
181
|
-
class ScheduleNotFoundError extends Error {
|
|
182
|
-
constructor(id) {
|
|
183
|
-
super(`Schedule not found: ${id}`);
|
|
184
|
-
this.name = "ScheduleNotFoundError";
|
|
185
|
-
}
|
|
186
|
-
}
|
|
187
196
|
// src/db/database.ts
|
|
188
197
|
import { Database } from "bun:sqlite";
|
|
189
198
|
import { mkdirSync, existsSync } from "fs";
|
|
190
199
|
import { dirname, join } from "path";
|
|
191
200
|
import { homedir } from "os";
|
|
192
|
-
var db = null;
|
|
193
201
|
function now() {
|
|
194
202
|
return new Date().toISOString();
|
|
195
203
|
}
|
|
@@ -208,8 +216,69 @@ function resolveDbPath() {
|
|
|
208
216
|
mkdirSync(dir, { recursive: true });
|
|
209
217
|
return join(dir, "testers.db");
|
|
210
218
|
}
|
|
211
|
-
|
|
212
|
-
|
|
219
|
+
function applyMigrations(database) {
|
|
220
|
+
const applied = database.query("SELECT id FROM _migrations ORDER BY id").all();
|
|
221
|
+
const appliedIds = new Set(applied.map((r) => r.id));
|
|
222
|
+
for (let i = 0;i < MIGRATIONS.length; i++) {
|
|
223
|
+
const migrationId = i + 1;
|
|
224
|
+
if (appliedIds.has(migrationId))
|
|
225
|
+
continue;
|
|
226
|
+
const migration = MIGRATIONS[i];
|
|
227
|
+
database.exec(migration);
|
|
228
|
+
database.query("INSERT INTO _migrations (id, applied_at) VALUES (?, ?)").run(migrationId, now());
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
function getDatabase() {
|
|
232
|
+
if (db)
|
|
233
|
+
return db;
|
|
234
|
+
const dbPath = resolveDbPath();
|
|
235
|
+
const dir = dirname(dbPath);
|
|
236
|
+
if (dbPath !== ":memory:" && !existsSync(dir)) {
|
|
237
|
+
mkdirSync(dir, { recursive: true });
|
|
238
|
+
}
|
|
239
|
+
db = new Database(dbPath);
|
|
240
|
+
db.exec("PRAGMA journal_mode = WAL");
|
|
241
|
+
db.exec("PRAGMA foreign_keys = ON");
|
|
242
|
+
db.exec("PRAGMA busy_timeout = 5000");
|
|
243
|
+
db.exec(`
|
|
244
|
+
CREATE TABLE IF NOT EXISTS _migrations (
|
|
245
|
+
id INTEGER PRIMARY KEY,
|
|
246
|
+
applied_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
247
|
+
);
|
|
248
|
+
`);
|
|
249
|
+
applyMigrations(db);
|
|
250
|
+
return db;
|
|
251
|
+
}
|
|
252
|
+
function closeDatabase() {
|
|
253
|
+
if (db) {
|
|
254
|
+
db.close();
|
|
255
|
+
db = null;
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
function resetDatabase() {
|
|
259
|
+
closeDatabase();
|
|
260
|
+
const database = getDatabase();
|
|
261
|
+
database.exec("DELETE FROM screenshots");
|
|
262
|
+
database.exec("DELETE FROM results");
|
|
263
|
+
database.exec("DELETE FROM webhooks");
|
|
264
|
+
database.exec("DELETE FROM auth_presets");
|
|
265
|
+
database.exec("DELETE FROM schedules");
|
|
266
|
+
database.exec("DELETE FROM runs");
|
|
267
|
+
database.exec("DELETE FROM scenarios");
|
|
268
|
+
database.exec("DELETE FROM agents");
|
|
269
|
+
database.exec("DELETE FROM projects");
|
|
270
|
+
}
|
|
271
|
+
function resolvePartialId(table, partialId) {
|
|
272
|
+
const database = getDatabase();
|
|
273
|
+
const rows = database.query(`SELECT id FROM ${table} WHERE id LIKE ? || '%'`).all(partialId);
|
|
274
|
+
if (rows.length === 1)
|
|
275
|
+
return rows[0].id;
|
|
276
|
+
return null;
|
|
277
|
+
}
|
|
278
|
+
var db = null, MIGRATIONS;
|
|
279
|
+
var init_database = __esm(() => {
|
|
280
|
+
MIGRATIONS = [
|
|
281
|
+
`
|
|
213
282
|
CREATE TABLE IF NOT EXISTS projects (
|
|
214
283
|
id TEXT PRIMARY KEY,
|
|
215
284
|
name TEXT NOT NULL UNIQUE,
|
|
@@ -298,7 +367,7 @@ var MIGRATIONS = [
|
|
|
298
367
|
applied_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
299
368
|
);
|
|
300
369
|
`,
|
|
301
|
-
|
|
370
|
+
`
|
|
302
371
|
CREATE INDEX IF NOT EXISTS idx_scenarios_project ON scenarios(project_id);
|
|
303
372
|
CREATE INDEX IF NOT EXISTS idx_scenarios_priority ON scenarios(priority);
|
|
304
373
|
CREATE INDEX IF NOT EXISTS idx_scenarios_short_id ON scenarios(short_id);
|
|
@@ -309,11 +378,11 @@ var MIGRATIONS = [
|
|
|
309
378
|
CREATE INDEX IF NOT EXISTS idx_results_status ON results(status);
|
|
310
379
|
CREATE INDEX IF NOT EXISTS idx_screenshots_result ON screenshots(result_id);
|
|
311
380
|
`,
|
|
312
|
-
|
|
381
|
+
`
|
|
313
382
|
ALTER TABLE projects ADD COLUMN scenario_prefix TEXT DEFAULT 'TST';
|
|
314
383
|
ALTER TABLE projects ADD COLUMN scenario_counter INTEGER DEFAULT 0;
|
|
315
384
|
`,
|
|
316
|
-
|
|
385
|
+
`
|
|
317
386
|
CREATE TABLE IF NOT EXISTS schedules (
|
|
318
387
|
id TEXT PRIMARY KEY,
|
|
319
388
|
project_id TEXT REFERENCES projects(id) ON DELETE CASCADE,
|
|
@@ -336,111 +405,71 @@ var MIGRATIONS = [
|
|
|
336
405
|
CREATE INDEX IF NOT EXISTS idx_schedules_project ON schedules(project_id);
|
|
337
406
|
CREATE INDEX IF NOT EXISTS idx_schedules_enabled ON schedules(enabled);
|
|
338
407
|
CREATE INDEX IF NOT EXISTS idx_schedules_next_run ON schedules(next_run_at);
|
|
408
|
+
`,
|
|
409
|
+
`
|
|
410
|
+
ALTER TABLE screenshots ADD COLUMN description TEXT;
|
|
411
|
+
ALTER TABLE screenshots ADD COLUMN page_url TEXT;
|
|
412
|
+
ALTER TABLE screenshots ADD COLUMN thumbnail_path TEXT;
|
|
413
|
+
`,
|
|
414
|
+
`
|
|
415
|
+
CREATE TABLE IF NOT EXISTS auth_presets (
|
|
416
|
+
id TEXT PRIMARY KEY,
|
|
417
|
+
name TEXT NOT NULL UNIQUE,
|
|
418
|
+
email TEXT NOT NULL,
|
|
419
|
+
password TEXT NOT NULL,
|
|
420
|
+
login_path TEXT DEFAULT '/login',
|
|
421
|
+
metadata TEXT DEFAULT '{}',
|
|
422
|
+
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
423
|
+
);
|
|
424
|
+
`,
|
|
425
|
+
`
|
|
426
|
+
CREATE TABLE IF NOT EXISTS webhooks (
|
|
427
|
+
id TEXT PRIMARY KEY,
|
|
428
|
+
url TEXT NOT NULL,
|
|
429
|
+
events TEXT NOT NULL DEFAULT '["failed"]',
|
|
430
|
+
project_id TEXT REFERENCES projects(id) ON DELETE CASCADE,
|
|
431
|
+
secret TEXT,
|
|
432
|
+
active INTEGER NOT NULL DEFAULT 1,
|
|
433
|
+
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
434
|
+
);
|
|
435
|
+
CREATE INDEX IF NOT EXISTS idx_webhooks_active ON webhooks(active);
|
|
339
436
|
`
|
|
340
|
-
];
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
function getDatabase() {
|
|
354
|
-
if (db)
|
|
355
|
-
return db;
|
|
356
|
-
const dbPath = resolveDbPath();
|
|
357
|
-
const dir = dirname(dbPath);
|
|
358
|
-
if (dbPath !== ":memory:" && !existsSync(dir)) {
|
|
359
|
-
mkdirSync(dir, { recursive: true });
|
|
360
|
-
}
|
|
361
|
-
db = new Database(dbPath);
|
|
362
|
-
db.exec("PRAGMA journal_mode = WAL");
|
|
363
|
-
db.exec("PRAGMA foreign_keys = ON");
|
|
364
|
-
db.exec("PRAGMA busy_timeout = 5000");
|
|
365
|
-
db.exec(`
|
|
366
|
-
CREATE TABLE IF NOT EXISTS _migrations (
|
|
367
|
-
id INTEGER PRIMARY KEY,
|
|
368
|
-
applied_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
369
|
-
);
|
|
370
|
-
`);
|
|
371
|
-
applyMigrations(db);
|
|
372
|
-
return db;
|
|
373
|
-
}
|
|
374
|
-
function closeDatabase() {
|
|
375
|
-
if (db) {
|
|
376
|
-
db.close();
|
|
377
|
-
db = null;
|
|
378
|
-
}
|
|
379
|
-
}
|
|
380
|
-
function resetDatabase() {
|
|
381
|
-
closeDatabase();
|
|
382
|
-
const database = getDatabase();
|
|
383
|
-
database.exec("DELETE FROM screenshots");
|
|
384
|
-
database.exec("DELETE FROM results");
|
|
385
|
-
database.exec("DELETE FROM schedules");
|
|
386
|
-
database.exec("DELETE FROM runs");
|
|
387
|
-
database.exec("DELETE FROM scenarios");
|
|
388
|
-
database.exec("DELETE FROM agents");
|
|
389
|
-
database.exec("DELETE FROM projects");
|
|
390
|
-
}
|
|
391
|
-
function resolvePartialId(table, partialId) {
|
|
392
|
-
const database = getDatabase();
|
|
393
|
-
const rows = database.query(`SELECT id FROM ${table} WHERE id LIKE ? || '%'`).all(partialId);
|
|
394
|
-
if (rows.length === 1)
|
|
395
|
-
return rows[0].id;
|
|
396
|
-
return null;
|
|
397
|
-
}
|
|
398
|
-
// src/db/scenarios.ts
|
|
399
|
-
function nextShortId(projectId) {
|
|
400
|
-
const db2 = getDatabase();
|
|
401
|
-
if (projectId) {
|
|
402
|
-
const project = db2.query("SELECT scenario_prefix, scenario_counter FROM projects WHERE id = ?").get(projectId);
|
|
403
|
-
if (project) {
|
|
404
|
-
const next = project.scenario_counter + 1;
|
|
405
|
-
db2.query("UPDATE projects SET scenario_counter = ? WHERE id = ?").run(next, projectId);
|
|
406
|
-
return `${project.scenario_prefix}-${next}`;
|
|
407
|
-
}
|
|
408
|
-
}
|
|
409
|
-
return shortUuid();
|
|
410
|
-
}
|
|
411
|
-
function createScenario(input) {
|
|
437
|
+
];
|
|
438
|
+
});
|
|
439
|
+
|
|
440
|
+
// src/db/runs.ts
|
|
441
|
+
var exports_runs = {};
|
|
442
|
+
__export(exports_runs, {
|
|
443
|
+
updateRun: () => updateRun,
|
|
444
|
+
listRuns: () => listRuns,
|
|
445
|
+
getRun: () => getRun,
|
|
446
|
+
deleteRun: () => deleteRun,
|
|
447
|
+
createRun: () => createRun
|
|
448
|
+
});
|
|
449
|
+
function createRun(input) {
|
|
412
450
|
const db2 = getDatabase();
|
|
413
451
|
const id = uuid();
|
|
414
|
-
const short_id = nextShortId(input.projectId);
|
|
415
452
|
const timestamp = now();
|
|
416
453
|
db2.query(`
|
|
417
|
-
INSERT INTO
|
|
418
|
-
VALUES (?, ?,
|
|
419
|
-
`).run(id,
|
|
420
|
-
return
|
|
454
|
+
INSERT INTO runs (id, project_id, status, url, model, headed, parallel, total, passed, failed, started_at, finished_at, metadata)
|
|
455
|
+
VALUES (?, ?, 'pending', ?, ?, ?, ?, 0, 0, 0, ?, NULL, ?)
|
|
456
|
+
`).run(id, input.projectId ?? null, input.url, input.model, input.headed ? 1 : 0, input.parallel ?? 1, timestamp, input.model ? JSON.stringify({}) : null);
|
|
457
|
+
return getRun(id);
|
|
421
458
|
}
|
|
422
|
-
function
|
|
459
|
+
function getRun(id) {
|
|
423
460
|
const db2 = getDatabase();
|
|
424
|
-
let row = db2.query("SELECT * FROM
|
|
425
|
-
if (row)
|
|
426
|
-
return scenarioFromRow(row);
|
|
427
|
-
row = db2.query("SELECT * FROM scenarios WHERE short_id = ?").get(id);
|
|
461
|
+
let row = db2.query("SELECT * FROM runs WHERE id = ?").get(id);
|
|
428
462
|
if (row)
|
|
429
|
-
return
|
|
430
|
-
const fullId = resolvePartialId("
|
|
463
|
+
return runFromRow(row);
|
|
464
|
+
const fullId = resolvePartialId("runs", id);
|
|
431
465
|
if (fullId) {
|
|
432
|
-
row = db2.query("SELECT * FROM
|
|
466
|
+
row = db2.query("SELECT * FROM runs WHERE id = ?").get(fullId);
|
|
433
467
|
if (row)
|
|
434
|
-
return
|
|
468
|
+
return runFromRow(row);
|
|
435
469
|
}
|
|
436
470
|
return null;
|
|
437
471
|
}
|
|
438
|
-
function
|
|
439
|
-
const db2 = getDatabase();
|
|
440
|
-
const row = db2.query("SELECT * FROM scenarios WHERE short_id = ?").get(shortId);
|
|
441
|
-
return row ? scenarioFromRow(row) : null;
|
|
442
|
-
}
|
|
443
|
-
function listScenarios(filter) {
|
|
472
|
+
function listRuns(filter) {
|
|
444
473
|
const db2 = getDatabase();
|
|
445
474
|
const conditions = [];
|
|
446
475
|
const params = [];
|
|
@@ -448,19 +477,170 @@ function listScenarios(filter) {
|
|
|
448
477
|
conditions.push("project_id = ?");
|
|
449
478
|
params.push(filter.projectId);
|
|
450
479
|
}
|
|
451
|
-
if (filter?.
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
params.push(`%"${tag}"%`);
|
|
455
|
-
}
|
|
480
|
+
if (filter?.status) {
|
|
481
|
+
conditions.push("status = ?");
|
|
482
|
+
params.push(filter.status);
|
|
456
483
|
}
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
484
|
+
let sql = "SELECT * FROM runs";
|
|
485
|
+
if (conditions.length > 0) {
|
|
486
|
+
sql += " WHERE " + conditions.join(" AND ");
|
|
460
487
|
}
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
488
|
+
sql += " ORDER BY started_at DESC";
|
|
489
|
+
if (filter?.limit) {
|
|
490
|
+
sql += " LIMIT ?";
|
|
491
|
+
params.push(filter.limit);
|
|
492
|
+
}
|
|
493
|
+
if (filter?.offset) {
|
|
494
|
+
sql += " OFFSET ?";
|
|
495
|
+
params.push(filter.offset);
|
|
496
|
+
}
|
|
497
|
+
const rows = db2.query(sql).all(...params);
|
|
498
|
+
return rows.map(runFromRow);
|
|
499
|
+
}
|
|
500
|
+
function updateRun(id, updates) {
|
|
501
|
+
const db2 = getDatabase();
|
|
502
|
+
const existing = getRun(id);
|
|
503
|
+
if (!existing) {
|
|
504
|
+
throw new Error(`Run not found: ${id}`);
|
|
505
|
+
}
|
|
506
|
+
const sets = [];
|
|
507
|
+
const params = [];
|
|
508
|
+
if (updates.status !== undefined) {
|
|
509
|
+
sets.push("status = ?");
|
|
510
|
+
params.push(updates.status);
|
|
511
|
+
}
|
|
512
|
+
if (updates.url !== undefined) {
|
|
513
|
+
sets.push("url = ?");
|
|
514
|
+
params.push(updates.url);
|
|
515
|
+
}
|
|
516
|
+
if (updates.model !== undefined) {
|
|
517
|
+
sets.push("model = ?");
|
|
518
|
+
params.push(updates.model);
|
|
519
|
+
}
|
|
520
|
+
if (updates.headed !== undefined) {
|
|
521
|
+
sets.push("headed = ?");
|
|
522
|
+
params.push(updates.headed);
|
|
523
|
+
}
|
|
524
|
+
if (updates.parallel !== undefined) {
|
|
525
|
+
sets.push("parallel = ?");
|
|
526
|
+
params.push(updates.parallel);
|
|
527
|
+
}
|
|
528
|
+
if (updates.total !== undefined) {
|
|
529
|
+
sets.push("total = ?");
|
|
530
|
+
params.push(updates.total);
|
|
531
|
+
}
|
|
532
|
+
if (updates.passed !== undefined) {
|
|
533
|
+
sets.push("passed = ?");
|
|
534
|
+
params.push(updates.passed);
|
|
535
|
+
}
|
|
536
|
+
if (updates.failed !== undefined) {
|
|
537
|
+
sets.push("failed = ?");
|
|
538
|
+
params.push(updates.failed);
|
|
539
|
+
}
|
|
540
|
+
if (updates.started_at !== undefined) {
|
|
541
|
+
sets.push("started_at = ?");
|
|
542
|
+
params.push(updates.started_at);
|
|
543
|
+
}
|
|
544
|
+
if (updates.finished_at !== undefined) {
|
|
545
|
+
sets.push("finished_at = ?");
|
|
546
|
+
params.push(updates.finished_at);
|
|
547
|
+
}
|
|
548
|
+
if (updates.metadata !== undefined) {
|
|
549
|
+
sets.push("metadata = ?");
|
|
550
|
+
params.push(updates.metadata);
|
|
551
|
+
}
|
|
552
|
+
if (sets.length === 0) {
|
|
553
|
+
return existing;
|
|
554
|
+
}
|
|
555
|
+
params.push(existing.id);
|
|
556
|
+
db2.query(`UPDATE runs SET ${sets.join(", ")} WHERE id = ?`).run(...params);
|
|
557
|
+
return getRun(existing.id);
|
|
558
|
+
}
|
|
559
|
+
function deleteRun(id) {
|
|
560
|
+
const db2 = getDatabase();
|
|
561
|
+
const run = getRun(id);
|
|
562
|
+
if (!run)
|
|
563
|
+
return false;
|
|
564
|
+
const result = db2.query("DELETE FROM runs WHERE id = ?").run(run.id);
|
|
565
|
+
return result.changes > 0;
|
|
566
|
+
}
|
|
567
|
+
var init_runs = __esm(() => {
|
|
568
|
+
init_types();
|
|
569
|
+
init_database();
|
|
570
|
+
});
|
|
571
|
+
|
|
572
|
+
// src/index.ts
|
|
573
|
+
init_types();
|
|
574
|
+
init_database();
|
|
575
|
+
|
|
576
|
+
// src/db/scenarios.ts
|
|
577
|
+
init_types();
|
|
578
|
+
init_database();
|
|
579
|
+
function nextShortId(projectId) {
|
|
580
|
+
const db2 = getDatabase();
|
|
581
|
+
if (projectId) {
|
|
582
|
+
const project = db2.query("SELECT scenario_prefix, scenario_counter FROM projects WHERE id = ?").get(projectId);
|
|
583
|
+
if (project) {
|
|
584
|
+
const next = project.scenario_counter + 1;
|
|
585
|
+
db2.query("UPDATE projects SET scenario_counter = ? WHERE id = ?").run(next, projectId);
|
|
586
|
+
return `${project.scenario_prefix}-${next}`;
|
|
587
|
+
}
|
|
588
|
+
}
|
|
589
|
+
return shortUuid();
|
|
590
|
+
}
|
|
591
|
+
function createScenario(input) {
|
|
592
|
+
const db2 = getDatabase();
|
|
593
|
+
const id = uuid();
|
|
594
|
+
const short_id = nextShortId(input.projectId);
|
|
595
|
+
const timestamp = now();
|
|
596
|
+
db2.query(`
|
|
597
|
+
INSERT INTO scenarios (id, short_id, project_id, name, description, steps, tags, priority, model, timeout_ms, target_path, requires_auth, auth_config, metadata, version, created_at, updated_at)
|
|
598
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1, ?, ?)
|
|
599
|
+
`).run(id, short_id, input.projectId ?? null, input.name, input.description, JSON.stringify(input.steps ?? []), JSON.stringify(input.tags ?? []), input.priority ?? "medium", input.model ?? null, input.timeoutMs ?? null, input.targetPath ?? null, input.requiresAuth ? 1 : 0, input.authConfig ? JSON.stringify(input.authConfig) : null, input.metadata ? JSON.stringify(input.metadata) : null, timestamp, timestamp);
|
|
600
|
+
return getScenario(id);
|
|
601
|
+
}
|
|
602
|
+
function getScenario(id) {
|
|
603
|
+
const db2 = getDatabase();
|
|
604
|
+
let row = db2.query("SELECT * FROM scenarios WHERE id = ?").get(id);
|
|
605
|
+
if (row)
|
|
606
|
+
return scenarioFromRow(row);
|
|
607
|
+
row = db2.query("SELECT * FROM scenarios WHERE short_id = ?").get(id);
|
|
608
|
+
if (row)
|
|
609
|
+
return scenarioFromRow(row);
|
|
610
|
+
const fullId = resolvePartialId("scenarios", id);
|
|
611
|
+
if (fullId) {
|
|
612
|
+
row = db2.query("SELECT * FROM scenarios WHERE id = ?").get(fullId);
|
|
613
|
+
if (row)
|
|
614
|
+
return scenarioFromRow(row);
|
|
615
|
+
}
|
|
616
|
+
return null;
|
|
617
|
+
}
|
|
618
|
+
function getScenarioByShortId(shortId) {
|
|
619
|
+
const db2 = getDatabase();
|
|
620
|
+
const row = db2.query("SELECT * FROM scenarios WHERE short_id = ?").get(shortId);
|
|
621
|
+
return row ? scenarioFromRow(row) : null;
|
|
622
|
+
}
|
|
623
|
+
function listScenarios(filter) {
|
|
624
|
+
const db2 = getDatabase();
|
|
625
|
+
const conditions = [];
|
|
626
|
+
const params = [];
|
|
627
|
+
if (filter?.projectId) {
|
|
628
|
+
conditions.push("project_id = ?");
|
|
629
|
+
params.push(filter.projectId);
|
|
630
|
+
}
|
|
631
|
+
if (filter?.tags && filter.tags.length > 0) {
|
|
632
|
+
for (const tag of filter.tags) {
|
|
633
|
+
conditions.push("tags LIKE ?");
|
|
634
|
+
params.push(`%"${tag}"%`);
|
|
635
|
+
}
|
|
636
|
+
}
|
|
637
|
+
if (filter?.priority) {
|
|
638
|
+
conditions.push("priority = ?");
|
|
639
|
+
params.push(filter.priority);
|
|
640
|
+
}
|
|
641
|
+
if (filter?.search) {
|
|
642
|
+
conditions.push("(name LIKE ? OR description LIKE ?)");
|
|
643
|
+
const term = `%${filter.search}%`;
|
|
464
644
|
params.push(term, term);
|
|
465
645
|
}
|
|
466
646
|
let sql = "SELECT * FROM scenarios";
|
|
@@ -557,63 +737,46 @@ function deleteScenario(id) {
|
|
|
557
737
|
const result = db2.query("DELETE FROM scenarios WHERE id = ?").run(scenario.id);
|
|
558
738
|
return result.changes > 0;
|
|
559
739
|
}
|
|
560
|
-
|
|
561
|
-
|
|
740
|
+
|
|
741
|
+
// src/index.ts
|
|
742
|
+
init_runs();
|
|
743
|
+
|
|
744
|
+
// src/db/results.ts
|
|
745
|
+
init_types();
|
|
746
|
+
init_database();
|
|
747
|
+
function createResult(input) {
|
|
562
748
|
const db2 = getDatabase();
|
|
563
749
|
const id = uuid();
|
|
564
750
|
const timestamp = now();
|
|
565
751
|
db2.query(`
|
|
566
|
-
INSERT INTO
|
|
567
|
-
VALUES (?, ?, '
|
|
568
|
-
`).run(id, input.
|
|
569
|
-
return
|
|
752
|
+
INSERT INTO results (id, run_id, scenario_id, status, reasoning, error, steps_completed, steps_total, duration_ms, model, tokens_used, cost_cents, metadata, created_at)
|
|
753
|
+
VALUES (?, ?, ?, 'skipped', NULL, NULL, 0, ?, 0, ?, 0, 0, '{}', ?)
|
|
754
|
+
`).run(id, input.runId, input.scenarioId, input.stepsTotal, input.model, timestamp);
|
|
755
|
+
return getResult(id);
|
|
570
756
|
}
|
|
571
|
-
function
|
|
757
|
+
function getResult(id) {
|
|
572
758
|
const db2 = getDatabase();
|
|
573
|
-
let row = db2.query("SELECT * FROM
|
|
759
|
+
let row = db2.query("SELECT * FROM results WHERE id = ?").get(id);
|
|
574
760
|
if (row)
|
|
575
|
-
return
|
|
576
|
-
const fullId = resolvePartialId("
|
|
761
|
+
return resultFromRow(row);
|
|
762
|
+
const fullId = resolvePartialId("results", id);
|
|
577
763
|
if (fullId) {
|
|
578
|
-
row = db2.query("SELECT * FROM
|
|
764
|
+
row = db2.query("SELECT * FROM results WHERE id = ?").get(fullId);
|
|
579
765
|
if (row)
|
|
580
|
-
return
|
|
766
|
+
return resultFromRow(row);
|
|
581
767
|
}
|
|
582
768
|
return null;
|
|
583
769
|
}
|
|
584
|
-
function
|
|
770
|
+
function listResults(runId) {
|
|
585
771
|
const db2 = getDatabase();
|
|
586
|
-
const
|
|
587
|
-
|
|
588
|
-
if (filter?.projectId) {
|
|
589
|
-
conditions.push("project_id = ?");
|
|
590
|
-
params.push(filter.projectId);
|
|
591
|
-
}
|
|
592
|
-
if (filter?.status) {
|
|
593
|
-
conditions.push("status = ?");
|
|
594
|
-
params.push(filter.status);
|
|
595
|
-
}
|
|
596
|
-
let sql = "SELECT * FROM runs";
|
|
597
|
-
if (conditions.length > 0) {
|
|
598
|
-
sql += " WHERE " + conditions.join(" AND ");
|
|
599
|
-
}
|
|
600
|
-
sql += " ORDER BY started_at DESC";
|
|
601
|
-
if (filter?.limit) {
|
|
602
|
-
sql += " LIMIT ?";
|
|
603
|
-
params.push(filter.limit);
|
|
604
|
-
}
|
|
605
|
-
if (filter?.offset) {
|
|
606
|
-
sql += " OFFSET ?";
|
|
607
|
-
params.push(filter.offset);
|
|
608
|
-
}
|
|
609
|
-
const rows = db2.query(sql).all(...params);
|
|
610
|
-
return rows.map(runFromRow);
|
|
772
|
+
const rows = db2.query("SELECT * FROM results WHERE run_id = ? ORDER BY created_at ASC").all(runId);
|
|
773
|
+
return rows.map(resultFromRow);
|
|
611
774
|
}
|
|
612
|
-
function
|
|
775
|
+
function updateResult(id, updates) {
|
|
613
776
|
const db2 = getDatabase();
|
|
614
|
-
const existing =
|
|
777
|
+
const existing = getResult(id);
|
|
615
778
|
if (!existing) {
|
|
616
|
-
throw new Error(`
|
|
779
|
+
throw new Error(`Result not found: ${id}`);
|
|
617
780
|
}
|
|
618
781
|
const sets = [];
|
|
619
782
|
const params = [];
|
|
@@ -621,128 +784,32 @@ function updateRun(id, updates) {
|
|
|
621
784
|
sets.push("status = ?");
|
|
622
785
|
params.push(updates.status);
|
|
623
786
|
}
|
|
624
|
-
if (updates.
|
|
625
|
-
sets.push("
|
|
626
|
-
params.push(updates.
|
|
627
|
-
}
|
|
628
|
-
if (updates.model !== undefined) {
|
|
629
|
-
sets.push("model = ?");
|
|
630
|
-
params.push(updates.model);
|
|
787
|
+
if (updates.reasoning !== undefined) {
|
|
788
|
+
sets.push("reasoning = ?");
|
|
789
|
+
params.push(updates.reasoning);
|
|
631
790
|
}
|
|
632
|
-
if (updates.
|
|
633
|
-
sets.push("
|
|
634
|
-
params.push(updates.
|
|
791
|
+
if (updates.error !== undefined) {
|
|
792
|
+
sets.push("error = ?");
|
|
793
|
+
params.push(updates.error);
|
|
635
794
|
}
|
|
636
|
-
if (updates.
|
|
637
|
-
sets.push("
|
|
638
|
-
params.push(updates.
|
|
795
|
+
if (updates.stepsCompleted !== undefined) {
|
|
796
|
+
sets.push("steps_completed = ?");
|
|
797
|
+
params.push(updates.stepsCompleted);
|
|
639
798
|
}
|
|
640
|
-
if (updates.
|
|
641
|
-
sets.push("
|
|
642
|
-
params.push(updates.
|
|
799
|
+
if (updates.durationMs !== undefined) {
|
|
800
|
+
sets.push("duration_ms = ?");
|
|
801
|
+
params.push(updates.durationMs);
|
|
643
802
|
}
|
|
644
|
-
if (updates.
|
|
645
|
-
sets.push("
|
|
646
|
-
params.push(updates.
|
|
803
|
+
if (updates.tokensUsed !== undefined) {
|
|
804
|
+
sets.push("tokens_used = ?");
|
|
805
|
+
params.push(updates.tokensUsed);
|
|
647
806
|
}
|
|
648
|
-
if (updates.
|
|
649
|
-
sets.push("
|
|
650
|
-
params.push(updates.
|
|
807
|
+
if (updates.costCents !== undefined) {
|
|
808
|
+
sets.push("cost_cents = ?");
|
|
809
|
+
params.push(updates.costCents);
|
|
651
810
|
}
|
|
652
|
-
if (
|
|
653
|
-
|
|
654
|
-
params.push(updates.started_at);
|
|
655
|
-
}
|
|
656
|
-
if (updates.finished_at !== undefined) {
|
|
657
|
-
sets.push("finished_at = ?");
|
|
658
|
-
params.push(updates.finished_at);
|
|
659
|
-
}
|
|
660
|
-
if (updates.metadata !== undefined) {
|
|
661
|
-
sets.push("metadata = ?");
|
|
662
|
-
params.push(updates.metadata);
|
|
663
|
-
}
|
|
664
|
-
if (sets.length === 0) {
|
|
665
|
-
return existing;
|
|
666
|
-
}
|
|
667
|
-
params.push(existing.id);
|
|
668
|
-
db2.query(`UPDATE runs SET ${sets.join(", ")} WHERE id = ?`).run(...params);
|
|
669
|
-
return getRun(existing.id);
|
|
670
|
-
}
|
|
671
|
-
function deleteRun(id) {
|
|
672
|
-
const db2 = getDatabase();
|
|
673
|
-
const run = getRun(id);
|
|
674
|
-
if (!run)
|
|
675
|
-
return false;
|
|
676
|
-
const result = db2.query("DELETE FROM runs WHERE id = ?").run(run.id);
|
|
677
|
-
return result.changes > 0;
|
|
678
|
-
}
|
|
679
|
-
// src/db/results.ts
|
|
680
|
-
function createResult(input) {
|
|
681
|
-
const db2 = getDatabase();
|
|
682
|
-
const id = uuid();
|
|
683
|
-
const timestamp = now();
|
|
684
|
-
db2.query(`
|
|
685
|
-
INSERT INTO results (id, run_id, scenario_id, status, reasoning, error, steps_completed, steps_total, duration_ms, model, tokens_used, cost_cents, metadata, created_at)
|
|
686
|
-
VALUES (?, ?, ?, 'skipped', NULL, NULL, 0, ?, 0, ?, 0, 0, '{}', ?)
|
|
687
|
-
`).run(id, input.runId, input.scenarioId, input.stepsTotal, input.model, timestamp);
|
|
688
|
-
return getResult(id);
|
|
689
|
-
}
|
|
690
|
-
function getResult(id) {
|
|
691
|
-
const db2 = getDatabase();
|
|
692
|
-
let row = db2.query("SELECT * FROM results WHERE id = ?").get(id);
|
|
693
|
-
if (row)
|
|
694
|
-
return resultFromRow(row);
|
|
695
|
-
const fullId = resolvePartialId("results", id);
|
|
696
|
-
if (fullId) {
|
|
697
|
-
row = db2.query("SELECT * FROM results WHERE id = ?").get(fullId);
|
|
698
|
-
if (row)
|
|
699
|
-
return resultFromRow(row);
|
|
700
|
-
}
|
|
701
|
-
return null;
|
|
702
|
-
}
|
|
703
|
-
function listResults(runId) {
|
|
704
|
-
const db2 = getDatabase();
|
|
705
|
-
const rows = db2.query("SELECT * FROM results WHERE run_id = ? ORDER BY created_at ASC").all(runId);
|
|
706
|
-
return rows.map(resultFromRow);
|
|
707
|
-
}
|
|
708
|
-
function updateResult(id, updates) {
|
|
709
|
-
const db2 = getDatabase();
|
|
710
|
-
const existing = getResult(id);
|
|
711
|
-
if (!existing) {
|
|
712
|
-
throw new Error(`Result not found: ${id}`);
|
|
713
|
-
}
|
|
714
|
-
const sets = [];
|
|
715
|
-
const params = [];
|
|
716
|
-
if (updates.status !== undefined) {
|
|
717
|
-
sets.push("status = ?");
|
|
718
|
-
params.push(updates.status);
|
|
719
|
-
}
|
|
720
|
-
if (updates.reasoning !== undefined) {
|
|
721
|
-
sets.push("reasoning = ?");
|
|
722
|
-
params.push(updates.reasoning);
|
|
723
|
-
}
|
|
724
|
-
if (updates.error !== undefined) {
|
|
725
|
-
sets.push("error = ?");
|
|
726
|
-
params.push(updates.error);
|
|
727
|
-
}
|
|
728
|
-
if (updates.stepsCompleted !== undefined) {
|
|
729
|
-
sets.push("steps_completed = ?");
|
|
730
|
-
params.push(updates.stepsCompleted);
|
|
731
|
-
}
|
|
732
|
-
if (updates.durationMs !== undefined) {
|
|
733
|
-
sets.push("duration_ms = ?");
|
|
734
|
-
params.push(updates.durationMs);
|
|
735
|
-
}
|
|
736
|
-
if (updates.tokensUsed !== undefined) {
|
|
737
|
-
sets.push("tokens_used = ?");
|
|
738
|
-
params.push(updates.tokensUsed);
|
|
739
|
-
}
|
|
740
|
-
if (updates.costCents !== undefined) {
|
|
741
|
-
sets.push("cost_cents = ?");
|
|
742
|
-
params.push(updates.costCents);
|
|
743
|
-
}
|
|
744
|
-
if (sets.length === 0) {
|
|
745
|
-
return existing;
|
|
811
|
+
if (sets.length === 0) {
|
|
812
|
+
return existing;
|
|
746
813
|
}
|
|
747
814
|
params.push(existing.id);
|
|
748
815
|
db2.query(`UPDATE results SET ${sets.join(", ")} WHERE id = ?`).run(...params);
|
|
@@ -752,14 +819,16 @@ function getResultsByRun(runId) {
|
|
|
752
819
|
return listResults(runId);
|
|
753
820
|
}
|
|
754
821
|
// src/db/screenshots.ts
|
|
822
|
+
init_types();
|
|
823
|
+
init_database();
|
|
755
824
|
function createScreenshot(input) {
|
|
756
825
|
const db2 = getDatabase();
|
|
757
826
|
const id = uuid();
|
|
758
827
|
const timestamp = now();
|
|
759
828
|
db2.query(`
|
|
760
|
-
INSERT INTO screenshots (id, result_id, step_number, action, file_path, width, height, timestamp)
|
|
761
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
762
|
-
`).run(id, input.resultId, input.stepNumber, input.action, input.filePath, input.width, input.height, timestamp);
|
|
829
|
+
INSERT INTO screenshots (id, result_id, step_number, action, file_path, width, height, timestamp, description, page_url, thumbnail_path)
|
|
830
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
831
|
+
`).run(id, input.resultId, input.stepNumber, input.action, input.filePath, input.width, input.height, timestamp, input.description ?? null, input.pageUrl ?? null, input.thumbnailPath ?? null);
|
|
763
832
|
return getScreenshot(id);
|
|
764
833
|
}
|
|
765
834
|
function getScreenshot(id) {
|
|
@@ -776,6 +845,8 @@ function getScreenshotsByResult(resultId) {
|
|
|
776
845
|
return listScreenshots(resultId);
|
|
777
846
|
}
|
|
778
847
|
// src/db/projects.ts
|
|
848
|
+
init_types();
|
|
849
|
+
init_database();
|
|
779
850
|
function createProject(input) {
|
|
780
851
|
const db2 = getDatabase();
|
|
781
852
|
const id = uuid();
|
|
@@ -812,6 +883,8 @@ function ensureProject(name, path) {
|
|
|
812
883
|
return createProject({ name, path });
|
|
813
884
|
}
|
|
814
885
|
// src/db/agents.ts
|
|
886
|
+
init_types();
|
|
887
|
+
init_database();
|
|
815
888
|
function registerAgent(input) {
|
|
816
889
|
const db2 = getDatabase();
|
|
817
890
|
const existing = db2.query("SELECT * FROM agents WHERE name = ?").get(input.name);
|
|
@@ -843,6 +916,9 @@ function listAgents() {
|
|
|
843
916
|
return rows.map(agentFromRow);
|
|
844
917
|
}
|
|
845
918
|
// src/db/schedules.ts
|
|
919
|
+
init_database();
|
|
920
|
+
init_types();
|
|
921
|
+
init_database();
|
|
846
922
|
function createSchedule(input) {
|
|
847
923
|
const db2 = getDatabase();
|
|
848
924
|
const id = uuid();
|
|
@@ -968,6 +1044,7 @@ function updateLastRun(id, runId, nextRunAt) {
|
|
|
968
1044
|
`).run(runId, timestamp, nextRunAt, timestamp, id);
|
|
969
1045
|
}
|
|
970
1046
|
// src/lib/config.ts
|
|
1047
|
+
init_types();
|
|
971
1048
|
import { homedir as homedir2 } from "os";
|
|
972
1049
|
import { join as join2 } from "path";
|
|
973
1050
|
import { readFileSync, existsSync as existsSync2 } from "fs";
|
|
@@ -1028,6 +1105,7 @@ function resolveModel(nameOrId) {
|
|
|
1028
1105
|
return nameOrId;
|
|
1029
1106
|
}
|
|
1030
1107
|
// src/lib/browser.ts
|
|
1108
|
+
init_types();
|
|
1031
1109
|
import { chromium } from "playwright";
|
|
1032
1110
|
import { execSync } from "child_process";
|
|
1033
1111
|
var DEFAULT_VIEWPORT = { width: 1280, height: 720 };
|
|
@@ -1132,7 +1210,7 @@ async function installBrowser() {
|
|
|
1132
1210
|
}
|
|
1133
1211
|
}
|
|
1134
1212
|
// src/lib/screenshotter.ts
|
|
1135
|
-
import { mkdirSync as mkdirSync2, existsSync as existsSync3 } from "fs";
|
|
1213
|
+
import { mkdirSync as mkdirSync2, existsSync as existsSync3, writeFileSync } from "fs";
|
|
1136
1214
|
import { join as join3 } from "path";
|
|
1137
1215
|
import { homedir as homedir3 } from "os";
|
|
1138
1216
|
function slugify(text) {
|
|
@@ -1141,16 +1219,63 @@ function slugify(text) {
|
|
|
1141
1219
|
function generateFilename(stepNumber, action) {
|
|
1142
1220
|
const padded = String(stepNumber).padStart(3, "0");
|
|
1143
1221
|
const slug = slugify(action);
|
|
1144
|
-
return `${padded}
|
|
1222
|
+
return `${padded}_${slug}.png`;
|
|
1223
|
+
}
|
|
1224
|
+
function formatDate(date) {
|
|
1225
|
+
return date.toISOString().slice(0, 10);
|
|
1145
1226
|
}
|
|
1146
|
-
function
|
|
1147
|
-
return
|
|
1227
|
+
function formatTime(date) {
|
|
1228
|
+
return date.toISOString().slice(11, 19).replace(/:/g, "-");
|
|
1229
|
+
}
|
|
1230
|
+
function getScreenshotDir(baseDir, runId, scenarioSlug, projectName, timestamp) {
|
|
1231
|
+
const now2 = timestamp ?? new Date;
|
|
1232
|
+
const project = projectName ?? "default";
|
|
1233
|
+
const dateDir = formatDate(now2);
|
|
1234
|
+
const timeDir = `${formatTime(now2)}_${runId.slice(0, 8)}`;
|
|
1235
|
+
return join3(baseDir, project, dateDir, timeDir, scenarioSlug);
|
|
1148
1236
|
}
|
|
1149
1237
|
function ensureDir(dirPath) {
|
|
1150
1238
|
if (!existsSync3(dirPath)) {
|
|
1151
1239
|
mkdirSync2(dirPath, { recursive: true });
|
|
1152
1240
|
}
|
|
1153
1241
|
}
|
|
1242
|
+
function writeMetaSidecar(screenshotPath, meta) {
|
|
1243
|
+
const metaPath = screenshotPath.replace(/\.png$/, ".meta.json").replace(/\.jpeg$/, ".meta.json");
|
|
1244
|
+
try {
|
|
1245
|
+
writeFileSync(metaPath, JSON.stringify(meta, null, 2), "utf-8");
|
|
1246
|
+
} catch {}
|
|
1247
|
+
}
|
|
1248
|
+
function writeRunMeta(dir, meta) {
|
|
1249
|
+
ensureDir(dir);
|
|
1250
|
+
try {
|
|
1251
|
+
writeFileSync(join3(dir, "_run-meta.json"), JSON.stringify(meta, null, 2), "utf-8");
|
|
1252
|
+
} catch {}
|
|
1253
|
+
}
|
|
1254
|
+
function writeScenarioMeta(dir, meta) {
|
|
1255
|
+
ensureDir(dir);
|
|
1256
|
+
try {
|
|
1257
|
+
writeFileSync(join3(dir, "_scenario-meta.json"), JSON.stringify(meta, null, 2), "utf-8");
|
|
1258
|
+
} catch {}
|
|
1259
|
+
}
|
|
1260
|
+
async function generateThumbnail(page, screenshotDir, filename) {
|
|
1261
|
+
try {
|
|
1262
|
+
const thumbDir = join3(screenshotDir, "_thumbnail");
|
|
1263
|
+
ensureDir(thumbDir);
|
|
1264
|
+
const thumbFilename = filename.replace(/\.(png|jpeg)$/, ".thumb.$1");
|
|
1265
|
+
const thumbPath = join3(thumbDir, thumbFilename);
|
|
1266
|
+
const viewport = page.viewportSize();
|
|
1267
|
+
if (viewport) {
|
|
1268
|
+
await page.screenshot({
|
|
1269
|
+
path: thumbPath,
|
|
1270
|
+
type: "png",
|
|
1271
|
+
clip: { x: 0, y: 0, width: Math.min(viewport.width, 1280), height: Math.min(viewport.height, 720) }
|
|
1272
|
+
});
|
|
1273
|
+
}
|
|
1274
|
+
return thumbPath;
|
|
1275
|
+
} catch {
|
|
1276
|
+
return null;
|
|
1277
|
+
}
|
|
1278
|
+
}
|
|
1154
1279
|
var DEFAULT_BASE_DIR = join3(homedir3(), ".testers", "screenshots");
|
|
1155
1280
|
|
|
1156
1281
|
class Screenshotter {
|
|
@@ -1158,15 +1283,20 @@ class Screenshotter {
|
|
|
1158
1283
|
format;
|
|
1159
1284
|
quality;
|
|
1160
1285
|
fullPage;
|
|
1286
|
+
projectName;
|
|
1287
|
+
runTimestamp;
|
|
1161
1288
|
constructor(options = {}) {
|
|
1162
1289
|
this.baseDir = options.baseDir ?? DEFAULT_BASE_DIR;
|
|
1163
1290
|
this.format = options.format ?? "png";
|
|
1164
1291
|
this.quality = options.quality ?? 90;
|
|
1165
1292
|
this.fullPage = options.fullPage ?? false;
|
|
1293
|
+
this.projectName = options.projectName ?? "default";
|
|
1294
|
+
this.runTimestamp = new Date;
|
|
1166
1295
|
}
|
|
1167
1296
|
async capture(page, options) {
|
|
1168
|
-
const
|
|
1169
|
-
const
|
|
1297
|
+
const action = options.description ?? options.action;
|
|
1298
|
+
const dir = getScreenshotDir(this.baseDir, options.runId, options.scenarioSlug, this.projectName, this.runTimestamp);
|
|
1299
|
+
const filename = generateFilename(options.stepNumber, action);
|
|
1170
1300
|
const filePath = join3(dir, filename);
|
|
1171
1301
|
ensureDir(dir);
|
|
1172
1302
|
await page.screenshot({
|
|
@@ -1176,16 +1306,32 @@ class Screenshotter {
|
|
|
1176
1306
|
quality: this.format === "jpeg" ? this.quality : undefined
|
|
1177
1307
|
});
|
|
1178
1308
|
const viewport = page.viewportSize() ?? { width: 0, height: 0 };
|
|
1309
|
+
const pageUrl = page.url();
|
|
1310
|
+
const timestamp = new Date().toISOString();
|
|
1311
|
+
writeMetaSidecar(filePath, {
|
|
1312
|
+
stepNumber: options.stepNumber,
|
|
1313
|
+
action: options.action,
|
|
1314
|
+
description: options.description ?? null,
|
|
1315
|
+
pageUrl,
|
|
1316
|
+
viewport,
|
|
1317
|
+
timestamp,
|
|
1318
|
+
filePath
|
|
1319
|
+
});
|
|
1320
|
+
const thumbnailPath = await generateThumbnail(page, dir, filename);
|
|
1179
1321
|
return {
|
|
1180
1322
|
filePath,
|
|
1181
1323
|
width: viewport.width,
|
|
1182
1324
|
height: viewport.height,
|
|
1183
|
-
timestamp
|
|
1325
|
+
timestamp,
|
|
1326
|
+
description: options.description ?? null,
|
|
1327
|
+
pageUrl,
|
|
1328
|
+
thumbnailPath
|
|
1184
1329
|
};
|
|
1185
1330
|
}
|
|
1186
1331
|
async captureFullPage(page, options) {
|
|
1187
|
-
const
|
|
1188
|
-
const
|
|
1332
|
+
const action = options.description ?? options.action;
|
|
1333
|
+
const dir = getScreenshotDir(this.baseDir, options.runId, options.scenarioSlug, this.projectName, this.runTimestamp);
|
|
1334
|
+
const filename = generateFilename(options.stepNumber, action);
|
|
1189
1335
|
const filePath = join3(dir, filename);
|
|
1190
1336
|
ensureDir(dir);
|
|
1191
1337
|
await page.screenshot({
|
|
@@ -1195,16 +1341,32 @@ class Screenshotter {
|
|
|
1195
1341
|
quality: this.format === "jpeg" ? this.quality : undefined
|
|
1196
1342
|
});
|
|
1197
1343
|
const viewport = page.viewportSize() ?? { width: 0, height: 0 };
|
|
1344
|
+
const pageUrl = page.url();
|
|
1345
|
+
const timestamp = new Date().toISOString();
|
|
1346
|
+
writeMetaSidecar(filePath, {
|
|
1347
|
+
stepNumber: options.stepNumber,
|
|
1348
|
+
action: options.action,
|
|
1349
|
+
description: options.description ?? null,
|
|
1350
|
+
pageUrl,
|
|
1351
|
+
viewport,
|
|
1352
|
+
timestamp,
|
|
1353
|
+
filePath
|
|
1354
|
+
});
|
|
1355
|
+
const thumbnailPath = await generateThumbnail(page, dir, filename);
|
|
1198
1356
|
return {
|
|
1199
1357
|
filePath,
|
|
1200
1358
|
width: viewport.width,
|
|
1201
1359
|
height: viewport.height,
|
|
1202
|
-
timestamp
|
|
1360
|
+
timestamp,
|
|
1361
|
+
description: options.description ?? null,
|
|
1362
|
+
pageUrl,
|
|
1363
|
+
thumbnailPath
|
|
1203
1364
|
};
|
|
1204
1365
|
}
|
|
1205
1366
|
async captureElement(page, selector, options) {
|
|
1206
|
-
const
|
|
1207
|
-
const
|
|
1367
|
+
const action = options.description ?? options.action;
|
|
1368
|
+
const dir = getScreenshotDir(this.baseDir, options.runId, options.scenarioSlug, this.projectName, this.runTimestamp);
|
|
1369
|
+
const filename = generateFilename(options.stepNumber, action);
|
|
1208
1370
|
const filePath = join3(dir, filename);
|
|
1209
1371
|
ensureDir(dir);
|
|
1210
1372
|
await page.locator(selector).screenshot({
|
|
@@ -1213,15 +1375,30 @@ class Screenshotter {
|
|
|
1213
1375
|
quality: this.format === "jpeg" ? this.quality : undefined
|
|
1214
1376
|
});
|
|
1215
1377
|
const viewport = page.viewportSize() ?? { width: 0, height: 0 };
|
|
1378
|
+
const pageUrl = page.url();
|
|
1379
|
+
const timestamp = new Date().toISOString();
|
|
1380
|
+
writeMetaSidecar(filePath, {
|
|
1381
|
+
stepNumber: options.stepNumber,
|
|
1382
|
+
action: options.action,
|
|
1383
|
+
description: options.description ?? null,
|
|
1384
|
+
pageUrl,
|
|
1385
|
+
viewport,
|
|
1386
|
+
timestamp,
|
|
1387
|
+
filePath
|
|
1388
|
+
});
|
|
1216
1389
|
return {
|
|
1217
1390
|
filePath,
|
|
1218
1391
|
width: viewport.width,
|
|
1219
1392
|
height: viewport.height,
|
|
1220
|
-
timestamp
|
|
1393
|
+
timestamp,
|
|
1394
|
+
description: options.description ?? null,
|
|
1395
|
+
pageUrl,
|
|
1396
|
+
thumbnailPath: null
|
|
1221
1397
|
};
|
|
1222
1398
|
}
|
|
1223
1399
|
}
|
|
1224
1400
|
// src/lib/ai-client.ts
|
|
1401
|
+
init_types();
|
|
1225
1402
|
import Anthropic from "@anthropic-ai/sdk";
|
|
1226
1403
|
function resolveModel2(nameOrPreset) {
|
|
1227
1404
|
if (nameOrPreset in MODEL_MAP) {
|
|
@@ -1899,6 +2076,7 @@ function createClient(apiKey) {
|
|
|
1899
2076
|
return new Anthropic({ apiKey: key });
|
|
1900
2077
|
}
|
|
1901
2078
|
// src/lib/runner.ts
|
|
2079
|
+
init_runs();
|
|
1902
2080
|
var eventHandler = null;
|
|
1903
2081
|
function onRunEvent(handler) {
|
|
1904
2082
|
eventHandler = handler;
|
|
@@ -1946,7 +2124,10 @@ async function runSingleScenario(scenario, runId, options) {
|
|
|
1946
2124
|
action: ss.action,
|
|
1947
2125
|
filePath: ss.filePath,
|
|
1948
2126
|
width: ss.width,
|
|
1949
|
-
height: ss.height
|
|
2127
|
+
height: ss.height,
|
|
2128
|
+
description: ss.description,
|
|
2129
|
+
pageUrl: ss.pageUrl,
|
|
2130
|
+
thumbnailPath: ss.thumbnailPath
|
|
1950
2131
|
});
|
|
1951
2132
|
emit({ type: "screenshot:captured", screenshotPath: ss.filePath, scenarioId: scenario.id, runId });
|
|
1952
2133
|
}
|
|
@@ -2733,6 +2914,7 @@ import { Database as Database2 } from "bun:sqlite";
|
|
|
2733
2914
|
import { existsSync as existsSync4 } from "fs";
|
|
2734
2915
|
import { join as join4 } from "path";
|
|
2735
2916
|
import { homedir as homedir4 } from "os";
|
|
2917
|
+
init_types();
|
|
2736
2918
|
function resolveTodosDbPath() {
|
|
2737
2919
|
const envPath = process.env["TODOS_DB_PATH"];
|
|
2738
2920
|
if (envPath)
|
|
@@ -2844,6 +3026,7 @@ function markTodoDone(taskId) {
|
|
|
2844
3026
|
}
|
|
2845
3027
|
}
|
|
2846
3028
|
// src/lib/scheduler.ts
|
|
3029
|
+
init_types();
|
|
2847
3030
|
function parseCronField(field, min, max) {
|
|
2848
3031
|
const results = new Set;
|
|
2849
3032
|
const parts = field.split(",");
|
|
@@ -3055,20 +3238,1193 @@ class Scheduler {
|
|
|
3055
3238
|
}
|
|
3056
3239
|
}
|
|
3057
3240
|
}
|
|
3241
|
+
// src/lib/init.ts
|
|
3242
|
+
import { existsSync as existsSync5, readFileSync as readFileSync2, writeFileSync as writeFileSync2, mkdirSync as mkdirSync3 } from "fs";
|
|
3243
|
+
import { join as join5, basename } from "path";
|
|
3244
|
+
import { homedir as homedir5 } from "os";
|
|
3245
|
+
function detectFramework(dir) {
|
|
3246
|
+
const pkgPath = join5(dir, "package.json");
|
|
3247
|
+
if (!existsSync5(pkgPath))
|
|
3248
|
+
return null;
|
|
3249
|
+
let pkg;
|
|
3250
|
+
try {
|
|
3251
|
+
pkg = JSON.parse(readFileSync2(pkgPath, "utf-8"));
|
|
3252
|
+
} catch {
|
|
3253
|
+
return null;
|
|
3254
|
+
}
|
|
3255
|
+
const allDeps = { ...pkg.dependencies, ...pkg.devDependencies };
|
|
3256
|
+
const depNames = Object.keys(allDeps);
|
|
3257
|
+
const features = [];
|
|
3258
|
+
const hasAuth = depNames.some((d) => d === "next-auth" || d.startsWith("@auth/") || d === "passport" || d === "lucia");
|
|
3259
|
+
if (hasAuth)
|
|
3260
|
+
features.push("hasAuth");
|
|
3261
|
+
const hasForms = depNames.some((d) => d === "react-hook-form" || d === "formik" || d === "zod");
|
|
3262
|
+
if (hasForms)
|
|
3263
|
+
features.push("hasForms");
|
|
3264
|
+
if ("next" in allDeps) {
|
|
3265
|
+
return { name: "Next.js", defaultUrl: "http://localhost:3000", features };
|
|
3266
|
+
}
|
|
3267
|
+
if ("vite" in allDeps) {
|
|
3268
|
+
return { name: "Vite", defaultUrl: "http://localhost:5173", features };
|
|
3269
|
+
}
|
|
3270
|
+
if (depNames.some((d) => d.startsWith("@remix-run"))) {
|
|
3271
|
+
return { name: "Remix", defaultUrl: "http://localhost:3000", features };
|
|
3272
|
+
}
|
|
3273
|
+
if ("nuxt" in allDeps) {
|
|
3274
|
+
return { name: "Nuxt", defaultUrl: "http://localhost:3000", features };
|
|
3275
|
+
}
|
|
3276
|
+
if (depNames.some((d) => d.startsWith("svelte") || d === "@sveltejs/kit")) {
|
|
3277
|
+
return { name: "SvelteKit", defaultUrl: "http://localhost:5173", features };
|
|
3278
|
+
}
|
|
3279
|
+
if (depNames.some((d) => d.startsWith("@angular"))) {
|
|
3280
|
+
return { name: "Angular", defaultUrl: "http://localhost:4200", features };
|
|
3281
|
+
}
|
|
3282
|
+
if ("express" in allDeps) {
|
|
3283
|
+
return { name: "Express", defaultUrl: "http://localhost:3000", features };
|
|
3284
|
+
}
|
|
3285
|
+
return null;
|
|
3286
|
+
}
|
|
3287
|
+
function getStarterScenarios(framework, projectId) {
|
|
3288
|
+
const scenarios = [
|
|
3289
|
+
{
|
|
3290
|
+
name: "Landing page loads",
|
|
3291
|
+
description: "Navigate to the landing page and verify it loads correctly with no console errors. Check that the main heading, navigation, and primary CTA are visible.",
|
|
3292
|
+
tags: ["smoke"],
|
|
3293
|
+
priority: "high",
|
|
3294
|
+
projectId
|
|
3295
|
+
},
|
|
3296
|
+
{
|
|
3297
|
+
name: "Navigation works",
|
|
3298
|
+
description: "Click through main navigation links and verify each page loads without errors.",
|
|
3299
|
+
tags: ["smoke"],
|
|
3300
|
+
priority: "medium",
|
|
3301
|
+
projectId
|
|
3302
|
+
},
|
|
3303
|
+
{
|
|
3304
|
+
name: "No console errors",
|
|
3305
|
+
description: "Navigate through the main pages and check the browser console for any JavaScript errors or warnings.",
|
|
3306
|
+
tags: ["smoke"],
|
|
3307
|
+
priority: "high",
|
|
3308
|
+
projectId
|
|
3309
|
+
}
|
|
3310
|
+
];
|
|
3311
|
+
if (framework.features.includes("hasAuth")) {
|
|
3312
|
+
scenarios.push({
|
|
3313
|
+
name: "Login flow",
|
|
3314
|
+
description: "Navigate to the login page, enter valid credentials, and verify successful authentication and redirect.",
|
|
3315
|
+
tags: ["auth"],
|
|
3316
|
+
priority: "critical",
|
|
3317
|
+
projectId
|
|
3318
|
+
}, {
|
|
3319
|
+
name: "Signup flow",
|
|
3320
|
+
description: "Navigate to the signup page, fill in registration details, and verify account creation succeeds.",
|
|
3321
|
+
tags: ["auth"],
|
|
3322
|
+
priority: "medium",
|
|
3323
|
+
projectId
|
|
3324
|
+
});
|
|
3325
|
+
}
|
|
3326
|
+
if (framework.features.includes("hasForms")) {
|
|
3327
|
+
scenarios.push({
|
|
3328
|
+
name: "Form validation",
|
|
3329
|
+
description: "Submit forms with empty/invalid data and verify validation errors appear correctly.",
|
|
3330
|
+
tags: ["forms"],
|
|
3331
|
+
priority: "medium",
|
|
3332
|
+
projectId
|
|
3333
|
+
});
|
|
3334
|
+
}
|
|
3335
|
+
return scenarios;
|
|
3336
|
+
}
|
|
3337
|
+
function initProject(options) {
|
|
3338
|
+
const dir = options.dir ?? process.cwd();
|
|
3339
|
+
const name = options.name ?? basename(dir);
|
|
3340
|
+
const framework = detectFramework(dir);
|
|
3341
|
+
const url = options.url ?? framework?.defaultUrl ?? "http://localhost:3000";
|
|
3342
|
+
const projectPath = options.path ?? dir;
|
|
3343
|
+
const project = ensureProject(name, projectPath);
|
|
3344
|
+
const starterInputs = getStarterScenarios(framework ?? { name: "Unknown", features: [] }, project.id);
|
|
3345
|
+
const scenarios = starterInputs.map((input) => createScenario(input));
|
|
3346
|
+
const configDir = join5(homedir5(), ".testers");
|
|
3347
|
+
const configPath = join5(configDir, "config.json");
|
|
3348
|
+
if (!existsSync5(configDir)) {
|
|
3349
|
+
mkdirSync3(configDir, { recursive: true });
|
|
3350
|
+
}
|
|
3351
|
+
let config = {};
|
|
3352
|
+
if (existsSync5(configPath)) {
|
|
3353
|
+
try {
|
|
3354
|
+
config = JSON.parse(readFileSync2(configPath, "utf-8"));
|
|
3355
|
+
} catch {}
|
|
3356
|
+
}
|
|
3357
|
+
config.activeProject = project.id;
|
|
3358
|
+
writeFileSync2(configPath, JSON.stringify(config, null, 2), "utf-8");
|
|
3359
|
+
return { project, scenarios, framework, url };
|
|
3360
|
+
}
|
|
3361
|
+
// src/lib/smoke.ts
|
|
3362
|
+
init_runs();
|
|
3363
|
+
var SMOKE_DESCRIPTION = `You are performing an autonomous smoke test of this web application. Your job is to explore as many pages as possible and find issues. Follow these instructions:
|
|
3364
|
+
|
|
3365
|
+
1. Start at the given URL and take a screenshot
|
|
3366
|
+
2. Find all visible navigation links and click through each one
|
|
3367
|
+
3. On each page: check for visible error messages, broken layouts, missing images
|
|
3368
|
+
4. Use get_page_html to check for error indicators in the HTML
|
|
3369
|
+
5. Try clicking the main interactive elements (buttons, links, forms)
|
|
3370
|
+
6. Keep track of every page you visit
|
|
3371
|
+
7. After exploring at least 5 different pages (or all available pages), report your findings
|
|
3372
|
+
|
|
3373
|
+
In your report_result, include:
|
|
3374
|
+
- Total pages visited
|
|
3375
|
+
- Any JavaScript errors you noticed
|
|
3376
|
+
- Any broken links (pages that show 404 or error)
|
|
3377
|
+
- Any visual issues (broken layouts, missing images, overlapping text)
|
|
3378
|
+
- Any forms that don't work
|
|
3379
|
+
- Rate each issue as critical/high/medium/low severity`;
|
|
3380
|
+
async function runSmoke(options) {
|
|
3381
|
+
const config = loadConfig();
|
|
3382
|
+
const model = resolveModel2(options.model ?? config.defaultModel);
|
|
3383
|
+
const scenario = createScenario({
|
|
3384
|
+
name: "Smoke Test",
|
|
3385
|
+
description: SMOKE_DESCRIPTION,
|
|
3386
|
+
tags: ["smoke", "auto"],
|
|
3387
|
+
priority: "high",
|
|
3388
|
+
projectId: options.projectId
|
|
3389
|
+
});
|
|
3390
|
+
const run = createRun({
|
|
3391
|
+
url: options.url,
|
|
3392
|
+
model,
|
|
3393
|
+
headed: options.headed,
|
|
3394
|
+
parallel: 1,
|
|
3395
|
+
projectId: options.projectId
|
|
3396
|
+
});
|
|
3397
|
+
updateRun(run.id, { status: "running", total: 1 });
|
|
3398
|
+
let result;
|
|
3399
|
+
try {
|
|
3400
|
+
result = await runSingleScenario(scenario, run.id, {
|
|
3401
|
+
url: options.url,
|
|
3402
|
+
model: options.model,
|
|
3403
|
+
headed: options.headed,
|
|
3404
|
+
timeout: options.timeout,
|
|
3405
|
+
projectId: options.projectId,
|
|
3406
|
+
apiKey: options.apiKey
|
|
3407
|
+
});
|
|
3408
|
+
const finalStatus = result.status === "passed" ? "passed" : "failed";
|
|
3409
|
+
updateRun(run.id, {
|
|
3410
|
+
status: finalStatus,
|
|
3411
|
+
passed: result.status === "passed" ? 1 : 0,
|
|
3412
|
+
failed: result.status === "passed" ? 0 : 1,
|
|
3413
|
+
total: 1,
|
|
3414
|
+
finished_at: new Date().toISOString()
|
|
3415
|
+
});
|
|
3416
|
+
} catch (error) {
|
|
3417
|
+
updateRun(run.id, {
|
|
3418
|
+
status: "failed",
|
|
3419
|
+
failed: 1,
|
|
3420
|
+
total: 1,
|
|
3421
|
+
finished_at: new Date().toISOString()
|
|
3422
|
+
});
|
|
3423
|
+
throw error;
|
|
3424
|
+
} finally {
|
|
3425
|
+
deleteScenario(scenario.id);
|
|
3426
|
+
}
|
|
3427
|
+
const issues = parseSmokeIssues(result.reasoning ?? "");
|
|
3428
|
+
const pagesVisited = extractPagesVisited(result.reasoning ?? "");
|
|
3429
|
+
const { getRun: getRun2 } = await Promise.resolve().then(() => (init_runs(), exports_runs));
|
|
3430
|
+
const finalRun = getRun2(run.id);
|
|
3431
|
+
return {
|
|
3432
|
+
run: finalRun,
|
|
3433
|
+
result,
|
|
3434
|
+
pagesVisited,
|
|
3435
|
+
issuesFound: issues
|
|
3436
|
+
};
|
|
3437
|
+
}
|
|
3438
|
+
var SEVERITY_PATTERN = /\b(CRITICAL|HIGH|MEDIUM|LOW)\b[:\s-]*(.+)/gi;
|
|
3439
|
+
var PAGES_VISITED_PATTERN = /(\d+)\s*(?:pages?\s*visited|pages?\s*explored|pages?\s*checked|total\s*pages?)/i;
|
|
3440
|
+
var URL_PATTERN = /https?:\/\/[^\s,)]+/g;
|
|
3441
|
+
var ISSUE_TYPE_MAP = {
|
|
3442
|
+
javascript: "js-error",
|
|
3443
|
+
"js error": "js-error",
|
|
3444
|
+
"js-error": "js-error",
|
|
3445
|
+
"console error": "js-error",
|
|
3446
|
+
"404": "404",
|
|
3447
|
+
"not found": "404",
|
|
3448
|
+
"broken link": "broken-link",
|
|
3449
|
+
"dead link": "broken-link",
|
|
3450
|
+
"broken image": "broken-image",
|
|
3451
|
+
"missing image": "broken-image",
|
|
3452
|
+
visual: "visual",
|
|
3453
|
+
layout: "visual",
|
|
3454
|
+
overlap: "visual",
|
|
3455
|
+
"broken layout": "visual",
|
|
3456
|
+
performance: "performance",
|
|
3457
|
+
slow: "performance"
|
|
3458
|
+
};
|
|
3459
|
+
function inferIssueType(text) {
|
|
3460
|
+
const lower = text.toLowerCase();
|
|
3461
|
+
for (const [keyword, type] of Object.entries(ISSUE_TYPE_MAP)) {
|
|
3462
|
+
if (lower.includes(keyword))
|
|
3463
|
+
return type;
|
|
3464
|
+
}
|
|
3465
|
+
return "visual";
|
|
3466
|
+
}
|
|
3467
|
+
function extractUrl(text, fallback = "") {
|
|
3468
|
+
const match = text.match(URL_PATTERN);
|
|
3469
|
+
return match ? match[0] : fallback;
|
|
3470
|
+
}
|
|
3471
|
+
function parseSmokeIssues(reasoning) {
|
|
3472
|
+
const issues = [];
|
|
3473
|
+
const seen = new Set;
|
|
3474
|
+
let match;
|
|
3475
|
+
const severityRegex = new RegExp(SEVERITY_PATTERN.source, "gi");
|
|
3476
|
+
while ((match = severityRegex.exec(reasoning)) !== null) {
|
|
3477
|
+
const severity = match[1].toLowerCase();
|
|
3478
|
+
const description = match[2].trim();
|
|
3479
|
+
const key = `${severity}:${description.slice(0, 80)}`;
|
|
3480
|
+
if (seen.has(key))
|
|
3481
|
+
continue;
|
|
3482
|
+
seen.add(key);
|
|
3483
|
+
issues.push({
|
|
3484
|
+
type: inferIssueType(description),
|
|
3485
|
+
severity,
|
|
3486
|
+
description,
|
|
3487
|
+
url: extractUrl(description)
|
|
3488
|
+
});
|
|
3489
|
+
}
|
|
3490
|
+
const bulletLines = reasoning.split(`
|
|
3491
|
+
`).filter((line) => /^\s*[-*]\s/.test(line) && /\b(error|broken|missing|404|fail|issue|bug|problem)\b/i.test(line));
|
|
3492
|
+
for (const line of bulletLines) {
|
|
3493
|
+
const cleaned = line.replace(/^\s*[-*]\s*/, "").trim();
|
|
3494
|
+
const key = `bullet:${cleaned.slice(0, 80)}`;
|
|
3495
|
+
if (seen.has(key))
|
|
3496
|
+
continue;
|
|
3497
|
+
seen.add(key);
|
|
3498
|
+
let severity = "medium";
|
|
3499
|
+
if (/\bcritical\b/i.test(cleaned))
|
|
3500
|
+
severity = "critical";
|
|
3501
|
+
else if (/\bhigh\b/i.test(cleaned))
|
|
3502
|
+
severity = "high";
|
|
3503
|
+
else if (/\blow\b/i.test(cleaned))
|
|
3504
|
+
severity = "low";
|
|
3505
|
+
else if (/\b(error|fail|broken|crash)\b/i.test(cleaned))
|
|
3506
|
+
severity = "high";
|
|
3507
|
+
issues.push({
|
|
3508
|
+
type: inferIssueType(cleaned),
|
|
3509
|
+
severity,
|
|
3510
|
+
description: cleaned,
|
|
3511
|
+
url: extractUrl(cleaned)
|
|
3512
|
+
});
|
|
3513
|
+
}
|
|
3514
|
+
return issues;
|
|
3515
|
+
}
|
|
3516
|
+
function extractPagesVisited(reasoning) {
|
|
3517
|
+
const match = reasoning.match(PAGES_VISITED_PATTERN);
|
|
3518
|
+
if (match)
|
|
3519
|
+
return parseInt(match[1], 10);
|
|
3520
|
+
const urls = reasoning.match(URL_PATTERN);
|
|
3521
|
+
if (urls) {
|
|
3522
|
+
const unique = new Set(urls.map((u) => new URL(u).pathname));
|
|
3523
|
+
return unique.size;
|
|
3524
|
+
}
|
|
3525
|
+
return 0;
|
|
3526
|
+
}
|
|
3527
|
+
var SEVERITY_COLORS = {
|
|
3528
|
+
critical: (t) => `\x1B[41m\x1B[37m ${t} \x1B[0m`,
|
|
3529
|
+
high: (t) => `\x1B[31m${t}\x1B[0m`,
|
|
3530
|
+
medium: (t) => `\x1B[33m${t}\x1B[0m`,
|
|
3531
|
+
low: (t) => `\x1B[36m${t}\x1B[0m`
|
|
3532
|
+
};
|
|
3533
|
+
var SEVERITY_ORDER = ["critical", "high", "medium", "low"];
|
|
3534
|
+
function formatSmokeReport(result) {
|
|
3535
|
+
const lines = [];
|
|
3536
|
+
const url = result.run.url;
|
|
3537
|
+
lines.push("");
|
|
3538
|
+
lines.push(`\x1B[1m Smoke Test Report \x1B[2m- ${url}\x1B[0m`);
|
|
3539
|
+
lines.push(` ${"\u2500".repeat(60)}`);
|
|
3540
|
+
const issueCount = result.issuesFound.length;
|
|
3541
|
+
const criticalCount = result.issuesFound.filter((i) => i.severity === "critical").length;
|
|
3542
|
+
const highCount = result.issuesFound.filter((i) => i.severity === "high").length;
|
|
3543
|
+
lines.push("");
|
|
3544
|
+
lines.push(` Pages visited: \x1B[1m${result.pagesVisited}\x1B[0m`);
|
|
3545
|
+
lines.push(` Issues found: \x1B[1m${issueCount}\x1B[0m`);
|
|
3546
|
+
lines.push(` Duration: ${result.result.durationMs ? `${(result.result.durationMs / 1000).toFixed(1)}s` : "N/A"}`);
|
|
3547
|
+
lines.push(` Model: ${result.run.model}`);
|
|
3548
|
+
lines.push(` Tokens used: ${result.result.tokensUsed}`);
|
|
3549
|
+
if (issueCount > 0) {
|
|
3550
|
+
lines.push("");
|
|
3551
|
+
lines.push(`\x1B[1m Issues\x1B[0m`);
|
|
3552
|
+
lines.push("");
|
|
3553
|
+
for (const severity of SEVERITY_ORDER) {
|
|
3554
|
+
const group = result.issuesFound.filter((i) => i.severity === severity);
|
|
3555
|
+
if (group.length === 0)
|
|
3556
|
+
continue;
|
|
3557
|
+
const badge = SEVERITY_COLORS[severity](severity.toUpperCase());
|
|
3558
|
+
lines.push(` ${badge}`);
|
|
3559
|
+
for (const issue of group) {
|
|
3560
|
+
const urlSuffix = issue.url ? ` \x1B[2m(${issue.url})\x1B[0m` : "";
|
|
3561
|
+
lines.push(` - ${issue.description}${urlSuffix}`);
|
|
3562
|
+
}
|
|
3563
|
+
lines.push("");
|
|
3564
|
+
}
|
|
3565
|
+
}
|
|
3566
|
+
lines.push(` ${"\u2500".repeat(60)}`);
|
|
3567
|
+
const hasCritical = criticalCount > 0 || highCount > 0;
|
|
3568
|
+
if (hasCritical) {
|
|
3569
|
+
lines.push(` Verdict: \x1B[31m\x1B[1mFAIL\x1B[0m \x1B[2m(${criticalCount} critical, ${highCount} high severity issues)\x1B[0m`);
|
|
3570
|
+
} else if (issueCount > 0) {
|
|
3571
|
+
lines.push(` Verdict: \x1B[33m\x1B[1mWARN\x1B[0m \x1B[2m(${issueCount} issues found, none critical/high)\x1B[0m`);
|
|
3572
|
+
} else {
|
|
3573
|
+
lines.push(` Verdict: \x1B[32m\x1B[1mPASS\x1B[0m \x1B[2m(no issues found)\x1B[0m`);
|
|
3574
|
+
}
|
|
3575
|
+
lines.push("");
|
|
3576
|
+
return lines.join(`
|
|
3577
|
+
`);
|
|
3578
|
+
}
|
|
3579
|
+
// src/lib/diff.ts
|
|
3580
|
+
init_runs();
|
|
3581
|
+
function diffRuns(runId1, runId2) {
|
|
3582
|
+
const run1 = getRun(runId1);
|
|
3583
|
+
if (!run1) {
|
|
3584
|
+
throw new Error(`Run not found: ${runId1}`);
|
|
3585
|
+
}
|
|
3586
|
+
const run2 = getRun(runId2);
|
|
3587
|
+
if (!run2) {
|
|
3588
|
+
throw new Error(`Run not found: ${runId2}`);
|
|
3589
|
+
}
|
|
3590
|
+
const results1 = getResultsByRun(run1.id);
|
|
3591
|
+
const results2 = getResultsByRun(run2.id);
|
|
3592
|
+
const map1 = new Map;
|
|
3593
|
+
for (const r of results1) {
|
|
3594
|
+
map1.set(r.scenarioId, r);
|
|
3595
|
+
}
|
|
3596
|
+
const map2 = new Map;
|
|
3597
|
+
for (const r of results2) {
|
|
3598
|
+
map2.set(r.scenarioId, r);
|
|
3599
|
+
}
|
|
3600
|
+
const allScenarioIds = new Set([...map1.keys(), ...map2.keys()]);
|
|
3601
|
+
const regressions = [];
|
|
3602
|
+
const fixes = [];
|
|
3603
|
+
const unchanged = [];
|
|
3604
|
+
const newScenarios = [];
|
|
3605
|
+
const removedScenarios = [];
|
|
3606
|
+
for (const scenarioId of allScenarioIds) {
|
|
3607
|
+
const r1 = map1.get(scenarioId) ?? null;
|
|
3608
|
+
const r2 = map2.get(scenarioId) ?? null;
|
|
3609
|
+
const scenario = getScenario(scenarioId);
|
|
3610
|
+
const diff = {
|
|
3611
|
+
scenarioId,
|
|
3612
|
+
scenarioName: scenario?.name ?? null,
|
|
3613
|
+
scenarioShortId: scenario?.shortId ?? null,
|
|
3614
|
+
status1: r1?.status ?? null,
|
|
3615
|
+
status2: r2?.status ?? null,
|
|
3616
|
+
duration1: r1?.durationMs ?? null,
|
|
3617
|
+
duration2: r2?.durationMs ?? null,
|
|
3618
|
+
tokens1: r1?.tokensUsed ?? null,
|
|
3619
|
+
tokens2: r2?.tokensUsed ?? null
|
|
3620
|
+
};
|
|
3621
|
+
if (!r1 && r2) {
|
|
3622
|
+
newScenarios.push(diff);
|
|
3623
|
+
} else if (r1 && !r2) {
|
|
3624
|
+
removedScenarios.push(diff);
|
|
3625
|
+
} else if (r1 && r2) {
|
|
3626
|
+
const wasPass = r1.status === "passed";
|
|
3627
|
+
const nowPass = r2.status === "passed";
|
|
3628
|
+
const wasFail = r1.status === "failed" || r1.status === "error";
|
|
3629
|
+
const nowFail = r2.status === "failed" || r2.status === "error";
|
|
3630
|
+
if (wasPass && nowFail) {
|
|
3631
|
+
regressions.push(diff);
|
|
3632
|
+
} else if (wasFail && nowPass) {
|
|
3633
|
+
fixes.push(diff);
|
|
3634
|
+
} else {
|
|
3635
|
+
unchanged.push(diff);
|
|
3636
|
+
}
|
|
3637
|
+
}
|
|
3638
|
+
}
|
|
3639
|
+
return { run1, run2, regressions, fixes, unchanged, newScenarios, removedScenarios };
|
|
3640
|
+
}
|
|
3641
|
+
function formatScenarioLabel(diff) {
|
|
3642
|
+
if (diff.scenarioShortId && diff.scenarioName) {
|
|
3643
|
+
return `${diff.scenarioShortId}: ${diff.scenarioName}`;
|
|
3644
|
+
}
|
|
3645
|
+
if (diff.scenarioName) {
|
|
3646
|
+
return diff.scenarioName;
|
|
3647
|
+
}
|
|
3648
|
+
return diff.scenarioId.slice(0, 8);
|
|
3649
|
+
}
|
|
3650
|
+
function formatDuration(ms) {
|
|
3651
|
+
if (ms === null)
|
|
3652
|
+
return "-";
|
|
3653
|
+
if (ms < 1000)
|
|
3654
|
+
return `${ms}ms`;
|
|
3655
|
+
return `${(ms / 1000).toFixed(1)}s`;
|
|
3656
|
+
}
|
|
3657
|
+
function formatDurationComparison(d1, d2) {
|
|
3658
|
+
const s1 = formatDuration(d1);
|
|
3659
|
+
const s2 = formatDuration(d2);
|
|
3660
|
+
if (d1 !== null && d2 !== null) {
|
|
3661
|
+
const delta = d2 - d1;
|
|
3662
|
+
const sign = delta > 0 ? "+" : "";
|
|
3663
|
+
return `${s1} -> ${s2} (${sign}${formatDuration(delta)})`;
|
|
3664
|
+
}
|
|
3665
|
+
return `${s1} -> ${s2}`;
|
|
3666
|
+
}
|
|
3667
|
+
function formatDiffTerminal(diff) {
|
|
3668
|
+
const lines = [];
|
|
3669
|
+
lines.push("");
|
|
3670
|
+
lines.push(source_default.bold(" Run Comparison"));
|
|
3671
|
+
lines.push(` Run 1: ${source_default.dim(diff.run1.id.slice(0, 8))} (${diff.run1.status}) \u2014 ${diff.run1.startedAt}`);
|
|
3672
|
+
lines.push(` Run 2: ${source_default.dim(diff.run2.id.slice(0, 8))} (${diff.run2.status}) \u2014 ${diff.run2.startedAt}`);
|
|
3673
|
+
lines.push("");
|
|
3674
|
+
if (diff.regressions.length > 0) {
|
|
3675
|
+
lines.push(source_default.red.bold(` Regressions (${diff.regressions.length}):`));
|
|
3676
|
+
for (const d of diff.regressions) {
|
|
3677
|
+
const label = formatScenarioLabel(d);
|
|
3678
|
+
const dur = formatDurationComparison(d.duration1, d.duration2);
|
|
3679
|
+
lines.push(source_default.red(` \u2B07 ${label} ${d.status1} -> ${d.status2} ${source_default.dim(dur)}`));
|
|
3680
|
+
}
|
|
3681
|
+
lines.push("");
|
|
3682
|
+
}
|
|
3683
|
+
if (diff.fixes.length > 0) {
|
|
3684
|
+
lines.push(source_default.green.bold(` Fixes (${diff.fixes.length}):`));
|
|
3685
|
+
for (const d of diff.fixes) {
|
|
3686
|
+
const label = formatScenarioLabel(d);
|
|
3687
|
+
const dur = formatDurationComparison(d.duration1, d.duration2);
|
|
3688
|
+
lines.push(source_default.green(` \u2B06 ${label} ${d.status1} -> ${d.status2} ${source_default.dim(dur)}`));
|
|
3689
|
+
}
|
|
3690
|
+
lines.push("");
|
|
3691
|
+
}
|
|
3692
|
+
if (diff.unchanged.length > 0) {
|
|
3693
|
+
lines.push(source_default.dim(` Unchanged (${diff.unchanged.length}):`));
|
|
3694
|
+
for (const d of diff.unchanged) {
|
|
3695
|
+
const label = formatScenarioLabel(d);
|
|
3696
|
+
const dur = formatDurationComparison(d.duration1, d.duration2);
|
|
3697
|
+
lines.push(source_default.dim(` = ${label} ${d.status2} ${dur}`));
|
|
3698
|
+
}
|
|
3699
|
+
lines.push("");
|
|
3700
|
+
}
|
|
3701
|
+
if (diff.newScenarios.length > 0) {
|
|
3702
|
+
lines.push(source_default.cyan(` New in run 2 (${diff.newScenarios.length}):`));
|
|
3703
|
+
for (const d of diff.newScenarios) {
|
|
3704
|
+
const label = formatScenarioLabel(d);
|
|
3705
|
+
lines.push(source_default.cyan(` + ${label} ${d.status2}`));
|
|
3706
|
+
}
|
|
3707
|
+
lines.push("");
|
|
3708
|
+
}
|
|
3709
|
+
if (diff.removedScenarios.length > 0) {
|
|
3710
|
+
lines.push(source_default.yellow(` Removed from run 2 (${diff.removedScenarios.length}):`));
|
|
3711
|
+
for (const d of diff.removedScenarios) {
|
|
3712
|
+
const label = formatScenarioLabel(d);
|
|
3713
|
+
lines.push(source_default.yellow(` - ${label} was ${d.status1}`));
|
|
3714
|
+
}
|
|
3715
|
+
lines.push("");
|
|
3716
|
+
}
|
|
3717
|
+
lines.push(source_default.bold(` Summary: ${diff.regressions.length} regressions, ${diff.fixes.length} fixes, ${diff.unchanged.length} unchanged`));
|
|
3718
|
+
lines.push("");
|
|
3719
|
+
return lines.join(`
|
|
3720
|
+
`);
|
|
3721
|
+
}
|
|
3722
|
+
function formatDiffJSON(diff) {
|
|
3723
|
+
return JSON.stringify(diff, null, 2);
|
|
3724
|
+
}
|
|
3725
|
+
// src/lib/templates.ts
|
|
3726
|
+
var SCENARIO_TEMPLATES = {
|
|
3727
|
+
auth: [
|
|
3728
|
+
{ name: "Login with valid credentials", description: "Navigate to the login page, enter valid credentials, submit the form, and verify redirect to authenticated area. Check that user menu/avatar is visible.", tags: ["auth", "smoke"], priority: "critical", requiresAuth: false, steps: ["Navigate to login page", "Enter email and password", "Submit login form", "Verify redirect to dashboard/home", "Verify user menu or avatar is visible"] },
|
|
3729
|
+
{ name: "Signup flow", description: "Navigate to signup page, fill all required fields with valid data, submit, and verify account creation succeeds.", tags: ["auth"], priority: "high", steps: ["Navigate to signup page", "Fill all required fields", "Submit registration form", "Verify success message or redirect"] },
|
|
3730
|
+
{ name: "Logout flow", description: "While authenticated, find and click the logout button/link, verify redirect to public page.", tags: ["auth"], priority: "medium", requiresAuth: true, steps: ["Click user menu or profile", "Click logout", "Verify redirect to login or home page"] }
|
|
3731
|
+
],
|
|
3732
|
+
crud: [
|
|
3733
|
+
{ name: "Create new item", description: "Navigate to the create form, fill all fields, submit, and verify the new item appears in the list.", tags: ["crud"], priority: "high", steps: ["Navigate to the list/index page", "Click create/add button", "Fill all required fields", "Submit the form", "Verify new item appears in list"] },
|
|
3734
|
+
{ name: "Read/view item details", description: "Click on an existing item to view its details page. Verify all fields are displayed correctly.", tags: ["crud"], priority: "medium", steps: ["Navigate to list page", "Click on an item", "Verify detail page shows all fields"] },
|
|
3735
|
+
{ name: "Update existing item", description: "Edit an existing item, change some fields, save, and verify changes persisted.", tags: ["crud"], priority: "high", steps: ["Navigate to item detail", "Click edit button", "Modify fields", "Save changes", "Verify updated values"] },
|
|
3736
|
+
{ name: "Delete item", description: "Delete an existing item and verify it's removed from the list.", tags: ["crud"], priority: "medium", steps: ["Navigate to list page", "Click delete on an item", "Confirm deletion", "Verify item removed from list"] }
|
|
3737
|
+
],
|
|
3738
|
+
forms: [
|
|
3739
|
+
{ name: "Form validation - empty submission", description: "Submit a form with all fields empty and verify validation errors appear for required fields.", tags: ["forms", "validation"], priority: "high", steps: ["Navigate to form page", "Click submit without filling fields", "Verify validation errors appear for each required field"] },
|
|
3740
|
+
{ name: "Form validation - invalid data", description: "Submit a form with invalid data (bad email, short password, etc) and verify appropriate error messages.", tags: ["forms", "validation"], priority: "high", steps: ["Navigate to form page", "Enter invalid email format", "Enter too-short password", "Submit form", "Verify specific validation error messages"] },
|
|
3741
|
+
{ name: "Form successful submission", description: "Fill form with valid data, submit, and verify success state (redirect, success message, or data saved).", tags: ["forms"], priority: "high", steps: ["Navigate to form page", "Fill all fields with valid data", "Submit form", "Verify success state"] }
|
|
3742
|
+
],
|
|
3743
|
+
nav: [
|
|
3744
|
+
{ name: "Main navigation links work", description: "Click through each main navigation link and verify each page loads correctly without errors.", tags: ["navigation", "smoke"], priority: "high", steps: ["Click each nav link", "Verify page loads", "Verify no error states", "Verify breadcrumbs if present"] },
|
|
3745
|
+
{ name: "Mobile navigation", description: "At mobile viewport, verify hamburger menu opens, navigation links are accessible, and pages load correctly.", tags: ["navigation", "responsive"], priority: "medium", steps: ["Resize to mobile viewport", "Click hamburger/menu icon", "Verify nav links appear", "Click a nav link", "Verify page loads"] }
|
|
3746
|
+
],
|
|
3747
|
+
a11y: [
|
|
3748
|
+
{ name: "Keyboard navigation", description: "Navigate the page using only keyboard (Tab, Enter, Escape). Verify all interactive elements are reachable and focusable.", tags: ["a11y", "keyboard"], priority: "high", steps: ["Press Tab to move through elements", "Verify focus indicators are visible", "Press Enter on buttons/links", "Verify actions trigger correctly", "Press Escape to close modals/dropdowns"] },
|
|
3749
|
+
{ name: "Image alt text", description: "Check that all images have meaningful alt text attributes.", tags: ["a11y"], priority: "medium", steps: ["Find all images on the page", "Check each image has an alt attribute", "Verify alt text is descriptive, not empty or generic"] }
|
|
3750
|
+
]
|
|
3751
|
+
};
|
|
3752
|
+
function getTemplate(name) {
|
|
3753
|
+
return SCENARIO_TEMPLATES[name] ?? null;
|
|
3754
|
+
}
|
|
3755
|
+
function listTemplateNames() {
|
|
3756
|
+
return Object.keys(SCENARIO_TEMPLATES);
|
|
3757
|
+
}
|
|
3758
|
+
// src/db/auth-presets.ts
|
|
3759
|
+
init_database();
|
|
3760
|
+
function fromRow(row) {
|
|
3761
|
+
return {
|
|
3762
|
+
id: row.id,
|
|
3763
|
+
name: row.name,
|
|
3764
|
+
email: row.email,
|
|
3765
|
+
password: row.password,
|
|
3766
|
+
loginPath: row.login_path,
|
|
3767
|
+
metadata: JSON.parse(row.metadata),
|
|
3768
|
+
createdAt: row.created_at
|
|
3769
|
+
};
|
|
3770
|
+
}
|
|
3771
|
+
function createAuthPreset(input) {
|
|
3772
|
+
const db2 = getDatabase();
|
|
3773
|
+
const id = uuid();
|
|
3774
|
+
const timestamp = now();
|
|
3775
|
+
db2.query(`
|
|
3776
|
+
INSERT INTO auth_presets (id, name, email, password, login_path, metadata, created_at)
|
|
3777
|
+
VALUES (?, ?, ?, ?, ?, '{}', ?)
|
|
3778
|
+
`).run(id, input.name, input.email, input.password, input.loginPath ?? "/login", timestamp);
|
|
3779
|
+
return getAuthPreset(input.name);
|
|
3780
|
+
}
|
|
3781
|
+
function getAuthPreset(name) {
|
|
3782
|
+
const db2 = getDatabase();
|
|
3783
|
+
const row = db2.query("SELECT * FROM auth_presets WHERE name = ?").get(name);
|
|
3784
|
+
return row ? fromRow(row) : null;
|
|
3785
|
+
}
|
|
3786
|
+
function listAuthPresets() {
|
|
3787
|
+
const db2 = getDatabase();
|
|
3788
|
+
const rows = db2.query("SELECT * FROM auth_presets ORDER BY created_at DESC").all();
|
|
3789
|
+
return rows.map(fromRow);
|
|
3790
|
+
}
|
|
3791
|
+
function deleteAuthPreset(name) {
|
|
3792
|
+
const db2 = getDatabase();
|
|
3793
|
+
const result = db2.query("DELETE FROM auth_presets WHERE name = ?").run(name);
|
|
3794
|
+
return result.changes > 0;
|
|
3795
|
+
}
|
|
3796
|
+
// src/lib/report.ts
|
|
3797
|
+
init_runs();
|
|
3798
|
+
import { readFileSync as readFileSync3, existsSync as existsSync6 } from "fs";
|
|
3799
|
+
function imageToBase64(filePath) {
|
|
3800
|
+
if (!filePath || !existsSync6(filePath))
|
|
3801
|
+
return "";
|
|
3802
|
+
try {
|
|
3803
|
+
const buffer = readFileSync3(filePath);
|
|
3804
|
+
const base64 = buffer.toString("base64");
|
|
3805
|
+
return `data:image/png;base64,${base64}`;
|
|
3806
|
+
} catch {
|
|
3807
|
+
return "";
|
|
3808
|
+
}
|
|
3809
|
+
}
|
|
3810
|
+
function escapeHtml(text) {
|
|
3811
|
+
return text.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, """).replace(/'/g, "'");
|
|
3812
|
+
}
|
|
3813
|
+
function formatDuration2(ms) {
|
|
3814
|
+
if (ms < 1000)
|
|
3815
|
+
return `${ms}ms`;
|
|
3816
|
+
if (ms < 60000)
|
|
3817
|
+
return `${(ms / 1000).toFixed(1)}s`;
|
|
3818
|
+
const mins = Math.floor(ms / 60000);
|
|
3819
|
+
const secs = (ms % 60000 / 1000).toFixed(0);
|
|
3820
|
+
return `${mins}m ${secs}s`;
|
|
3821
|
+
}
|
|
3822
|
+
function formatCost(cents) {
|
|
3823
|
+
if (cents < 1)
|
|
3824
|
+
return `$${(cents / 100).toFixed(4)}`;
|
|
3825
|
+
return `$${(cents / 100).toFixed(2)}`;
|
|
3826
|
+
}
|
|
3827
|
+
function statusBadge(status) {
|
|
3828
|
+
const colors = {
|
|
3829
|
+
passed: { bg: "#22c55e", text: "#000" },
|
|
3830
|
+
failed: { bg: "#ef4444", text: "#fff" },
|
|
3831
|
+
error: { bg: "#eab308", text: "#000" },
|
|
3832
|
+
skipped: { bg: "#6b7280", text: "#fff" }
|
|
3833
|
+
};
|
|
3834
|
+
const c = colors[status] ?? { bg: "#6b7280", text: "#fff" };
|
|
3835
|
+
const label = status.toUpperCase();
|
|
3836
|
+
return `<span style="display:inline-block;padding:2px 10px;border-radius:4px;font-size:12px;font-weight:700;background:${c.bg};color:${c.text};letter-spacing:0.5px;">${label}</span>`;
|
|
3837
|
+
}
|
|
3838
|
+
function renderScreenshots(screenshots) {
|
|
3839
|
+
if (screenshots.length === 0)
|
|
3840
|
+
return "";
|
|
3841
|
+
let html = `<div style="display:flex;flex-wrap:wrap;gap:12px;margin-top:12px;">`;
|
|
3842
|
+
for (let i = 0;i < screenshots.length; i++) {
|
|
3843
|
+
const ss = screenshots[i];
|
|
3844
|
+
const dataUri = imageToBase64(ss.filePath);
|
|
3845
|
+
const checkId = `ss-${ss.id}`;
|
|
3846
|
+
if (dataUri) {
|
|
3847
|
+
html += `
|
|
3848
|
+
<div style="flex:0 0 auto;">
|
|
3849
|
+
<input type="checkbox" id="${checkId}" style="display:none;" />
|
|
3850
|
+
<label for="${checkId}" style="cursor:pointer;">
|
|
3851
|
+
<img src="${dataUri}" alt="Step ${ss.stepNumber}: ${escapeHtml(ss.action)}"
|
|
3852
|
+
style="max-width:200px;max-height:150px;border-radius:6px;border:1px solid #262626;display:block;" />
|
|
3853
|
+
</label>
|
|
3854
|
+
<div style="position:fixed;top:0;left:0;width:100%;height:100%;background:rgba(0,0,0,0.9);z-index:1000;display:none;align-items:center;justify-content:center;">
|
|
3855
|
+
<label for="${checkId}" style="position:absolute;top:0;left:0;width:100%;height:100%;cursor:pointer;"></label>
|
|
3856
|
+
<img src="${dataUri}" alt="Step ${ss.stepNumber}: ${escapeHtml(ss.action)}"
|
|
3857
|
+
style="max-width:600px;max-height:90vh;border-radius:8px;position:relative;z-index:1001;" />
|
|
3858
|
+
</div>
|
|
3859
|
+
<div style="font-size:11px;color:#888;margin-top:4px;max-width:200px;overflow:hidden;text-overflow:ellipsis;white-space:nowrap;">
|
|
3860
|
+
${ss.stepNumber}. ${escapeHtml(ss.action)}
|
|
3861
|
+
</div>
|
|
3862
|
+
</div>`;
|
|
3863
|
+
} else {
|
|
3864
|
+
html += `
|
|
3865
|
+
<div style="flex:0 0 auto;width:200px;height:150px;background:#1a1a1a;border:1px dashed #333;border-radius:6px;display:flex;align-items:center;justify-content:center;color:#555;font-size:12px;">
|
|
3866
|
+
Screenshot not found
|
|
3867
|
+
<div style="font-size:11px;color:#888;margin-top:4px;">${ss.stepNumber}. ${escapeHtml(ss.action)}</div>
|
|
3868
|
+
</div>`;
|
|
3869
|
+
}
|
|
3870
|
+
}
|
|
3871
|
+
html += `</div>`;
|
|
3872
|
+
return html;
|
|
3873
|
+
}
|
|
3874
|
+
function generateHtmlReport(runId) {
|
|
3875
|
+
const run = getRun(runId);
|
|
3876
|
+
if (!run)
|
|
3877
|
+
throw new Error(`Run not found: ${runId}`);
|
|
3878
|
+
const results = getResultsByRun(run.id);
|
|
3879
|
+
const resultData = [];
|
|
3880
|
+
for (const result of results) {
|
|
3881
|
+
const screenshots = listScreenshots(result.id);
|
|
3882
|
+
const scenario = getScenario(result.scenarioId);
|
|
3883
|
+
resultData.push({
|
|
3884
|
+
result,
|
|
3885
|
+
scenarioName: scenario?.name ?? "Unknown Scenario",
|
|
3886
|
+
scenarioShortId: scenario?.shortId ?? result.scenarioId.slice(0, 8),
|
|
3887
|
+
screenshots
|
|
3888
|
+
});
|
|
3889
|
+
}
|
|
3890
|
+
const passedCount = results.filter((r) => r.status === "passed").length;
|
|
3891
|
+
const failedCount = results.filter((r) => r.status === "failed").length;
|
|
3892
|
+
const errorCount = results.filter((r) => r.status === "error").length;
|
|
3893
|
+
const totalCount = results.length;
|
|
3894
|
+
const totalTokens = results.reduce((sum, r) => sum + r.tokensUsed, 0);
|
|
3895
|
+
const totalCostCents = results.reduce((sum, r) => sum + r.costCents, 0);
|
|
3896
|
+
const totalDurationMs = run.finishedAt && run.startedAt ? new Date(run.finishedAt).getTime() - new Date(run.startedAt).getTime() : results.reduce((sum, r) => sum + r.durationMs, 0);
|
|
3897
|
+
const generatedAt = new Date().toISOString();
|
|
3898
|
+
let resultCards = "";
|
|
3899
|
+
for (const { result, scenarioName, scenarioShortId, screenshots } of resultData) {
|
|
3900
|
+
resultCards += `
|
|
3901
|
+
<div style="background:#141414;border:1px solid #262626;border-radius:8px;padding:20px;margin-bottom:16px;">
|
|
3902
|
+
<div style="display:flex;align-items:center;gap:12px;margin-bottom:12px;">
|
|
3903
|
+
${statusBadge(result.status)}
|
|
3904
|
+
<span style="font-size:16px;font-weight:600;color:#e5e5e5;">${escapeHtml(scenarioName)}</span>
|
|
3905
|
+
<span style="font-size:12px;color:#666;font-family:monospace;">${escapeHtml(scenarioShortId)}</span>
|
|
3906
|
+
</div>
|
|
3907
|
+
|
|
3908
|
+
${result.reasoning ? `<div style="color:#a3a3a3;font-size:14px;line-height:1.6;margin-bottom:12px;padding:12px;background:#0d0d0d;border-radius:6px;border-left:3px solid #333;">${escapeHtml(result.reasoning)}</div>` : ""}
|
|
3909
|
+
|
|
3910
|
+
${result.error ? `<div style="color:#ef4444;font-size:13px;margin-bottom:12px;padding:12px;background:#1a0a0a;border-radius:6px;border-left:3px solid #ef4444;font-family:monospace;">${escapeHtml(result.error)}</div>` : ""}
|
|
3911
|
+
|
|
3912
|
+
<div style="display:flex;gap:24px;font-size:13px;color:#888;">
|
|
3913
|
+
<span>Duration: <span style="color:#d4d4d4;">${formatDuration2(result.durationMs)}</span></span>
|
|
3914
|
+
<span>Steps: <span style="color:#d4d4d4;">${result.stepsCompleted}/${result.stepsTotal}</span></span>
|
|
3915
|
+
<span>Tokens: <span style="color:#d4d4d4;">${result.tokensUsed.toLocaleString()}</span></span>
|
|
3916
|
+
<span>Cost: <span style="color:#d4d4d4;">${formatCost(result.costCents)}</span></span>
|
|
3917
|
+
<span>Model: <span style="color:#d4d4d4;">${escapeHtml(result.model)}</span></span>
|
|
3918
|
+
</div>
|
|
3919
|
+
|
|
3920
|
+
${renderScreenshots(screenshots)}
|
|
3921
|
+
</div>`;
|
|
3922
|
+
}
|
|
3923
|
+
return `<!DOCTYPE html>
|
|
3924
|
+
<html lang="en">
|
|
3925
|
+
<head>
|
|
3926
|
+
<meta charset="UTF-8" />
|
|
3927
|
+
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
|
3928
|
+
<title>Test Report - ${escapeHtml(run.id.slice(0, 8))}</title>
|
|
3929
|
+
<style>
|
|
3930
|
+
* { margin: 0; padding: 0; box-sizing: border-box; }
|
|
3931
|
+
body { background: #0a0a0a; color: #e5e5e5; font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif; padding: 40px 20px; }
|
|
3932
|
+
.container { max-width: 960px; margin: 0 auto; }
|
|
3933
|
+
input[type="checkbox"]:checked ~ div:last-of-type { display: flex !important; }
|
|
3934
|
+
</style>
|
|
3935
|
+
</head>
|
|
3936
|
+
<body>
|
|
3937
|
+
<div class="container">
|
|
3938
|
+
<!-- Header -->
|
|
3939
|
+
<div style="margin-bottom:32px;">
|
|
3940
|
+
<h1 style="font-size:28px;font-weight:700;margin-bottom:8px;color:#fff;">Test Report</h1>
|
|
3941
|
+
<div style="display:flex;flex-wrap:wrap;gap:24px;font-size:14px;color:#888;">
|
|
3942
|
+
<span>Run: <span style="color:#d4d4d4;font-family:monospace;">${escapeHtml(run.id.slice(0, 8))}</span></span>
|
|
3943
|
+
<span>URL: <a href="${escapeHtml(run.url)}" style="color:#60a5fa;text-decoration:none;">${escapeHtml(run.url)}</a></span>
|
|
3944
|
+
<span>Model: <span style="color:#d4d4d4;">${escapeHtml(run.model)}</span></span>
|
|
3945
|
+
<span>Date: <span style="color:#d4d4d4;">${escapeHtml(run.startedAt)}</span></span>
|
|
3946
|
+
<span>Duration: <span style="color:#d4d4d4;">${formatDuration2(totalDurationMs)}</span></span>
|
|
3947
|
+
<span>Status: ${statusBadge(run.status)}</span>
|
|
3948
|
+
</div>
|
|
3949
|
+
</div>
|
|
3950
|
+
|
|
3951
|
+
<!-- Summary Bar -->
|
|
3952
|
+
<div style="display:flex;gap:16px;margin-bottom:32px;">
|
|
3953
|
+
<div style="flex:1;background:#141414;border:1px solid #262626;border-radius:8px;padding:16px;text-align:center;">
|
|
3954
|
+
<div style="font-size:28px;font-weight:700;color:#e5e5e5;">${totalCount}</div>
|
|
3955
|
+
<div style="font-size:12px;color:#888;margin-top:4px;">TOTAL</div>
|
|
3956
|
+
</div>
|
|
3957
|
+
<div style="flex:1;background:#141414;border:1px solid #262626;border-radius:8px;padding:16px;text-align:center;">
|
|
3958
|
+
<div style="font-size:28px;font-weight:700;color:#22c55e;">${passedCount}</div>
|
|
3959
|
+
<div style="font-size:12px;color:#888;margin-top:4px;">PASSED</div>
|
|
3960
|
+
</div>
|
|
3961
|
+
<div style="flex:1;background:#141414;border:1px solid #262626;border-radius:8px;padding:16px;text-align:center;">
|
|
3962
|
+
<div style="font-size:28px;font-weight:700;color:#ef4444;">${failedCount}</div>
|
|
3963
|
+
<div style="font-size:12px;color:#888;margin-top:4px;">FAILED</div>
|
|
3964
|
+
</div>
|
|
3965
|
+
${errorCount > 0 ? `
|
|
3966
|
+
<div style="flex:1;background:#141414;border:1px solid #262626;border-radius:8px;padding:16px;text-align:center;">
|
|
3967
|
+
<div style="font-size:28px;font-weight:700;color:#eab308;">${errorCount}</div>
|
|
3968
|
+
<div style="font-size:12px;color:#888;margin-top:4px;">ERRORS</div>
|
|
3969
|
+
</div>` : ""}
|
|
3970
|
+
</div>
|
|
3971
|
+
|
|
3972
|
+
<!-- Results -->
|
|
3973
|
+
${resultCards}
|
|
3974
|
+
|
|
3975
|
+
<!-- Footer -->
|
|
3976
|
+
<div style="margin-top:32px;padding-top:20px;border-top:1px solid #262626;display:flex;justify-content:space-between;font-size:13px;color:#666;">
|
|
3977
|
+
<div>
|
|
3978
|
+
Total tokens: ${totalTokens.toLocaleString()} | Total cost: ${formatCost(totalCostCents)}
|
|
3979
|
+
</div>
|
|
3980
|
+
<div>
|
|
3981
|
+
Generated: ${escapeHtml(generatedAt)}
|
|
3982
|
+
</div>
|
|
3983
|
+
</div>
|
|
3984
|
+
</div>
|
|
3985
|
+
</body>
|
|
3986
|
+
</html>`;
|
|
3987
|
+
}
|
|
3988
|
+
function generateLatestReport() {
|
|
3989
|
+
const runs = listRuns({ limit: 1 });
|
|
3990
|
+
if (runs.length === 0)
|
|
3991
|
+
throw new Error("No runs found");
|
|
3992
|
+
return generateHtmlReport(runs[0].id);
|
|
3993
|
+
}
|
|
3994
|
+
// src/lib/costs.ts
|
|
3995
|
+
init_database();
|
|
3996
|
+
function getDateFilter(period) {
|
|
3997
|
+
switch (period) {
|
|
3998
|
+
case "day":
|
|
3999
|
+
return "AND r.created_at >= date('now', 'start of day')";
|
|
4000
|
+
case "week":
|
|
4001
|
+
return "AND r.created_at >= date('now', '-7 days')";
|
|
4002
|
+
case "month":
|
|
4003
|
+
return "AND r.created_at >= date('now', '-30 days')";
|
|
4004
|
+
case "all":
|
|
4005
|
+
return "";
|
|
4006
|
+
}
|
|
4007
|
+
}
|
|
4008
|
+
function getPeriodDays(period) {
|
|
4009
|
+
switch (period) {
|
|
4010
|
+
case "day":
|
|
4011
|
+
return 1;
|
|
4012
|
+
case "week":
|
|
4013
|
+
return 7;
|
|
4014
|
+
case "month":
|
|
4015
|
+
return 30;
|
|
4016
|
+
case "all":
|
|
4017
|
+
return 30;
|
|
4018
|
+
}
|
|
4019
|
+
}
|
|
4020
|
+
function loadBudgetConfig() {
|
|
4021
|
+
const config = loadConfig();
|
|
4022
|
+
const budget = config.budget;
|
|
4023
|
+
return {
|
|
4024
|
+
maxPerRunCents: budget?.maxPerRunCents ?? 50,
|
|
4025
|
+
maxPerDayCents: budget?.maxPerDayCents ?? 500,
|
|
4026
|
+
warnAtPercent: budget?.warnAtPercent ?? 0.8
|
|
4027
|
+
};
|
|
4028
|
+
}
|
|
4029
|
+
function getCostSummary(options) {
|
|
4030
|
+
const db2 = getDatabase();
|
|
4031
|
+
const period = options?.period ?? "month";
|
|
4032
|
+
const projectId = options?.projectId;
|
|
4033
|
+
const dateFilter = getDateFilter(period);
|
|
4034
|
+
const projectFilter = projectId ? "AND ru.project_id = ?" : "";
|
|
4035
|
+
const projectParams = projectId ? [projectId] : [];
|
|
4036
|
+
const totalsRow = db2.query(`SELECT
|
|
4037
|
+
COALESCE(SUM(r.cost_cents), 0) as total_cost,
|
|
4038
|
+
COALESCE(SUM(r.tokens_used), 0) as total_tokens,
|
|
4039
|
+
COUNT(DISTINCT r.run_id) as run_count
|
|
4040
|
+
FROM results r
|
|
4041
|
+
JOIN runs ru ON r.run_id = ru.id
|
|
4042
|
+
WHERE 1=1 ${dateFilter} ${projectFilter}`).get(...projectParams);
|
|
4043
|
+
const modelRows = db2.query(`SELECT
|
|
4044
|
+
r.model,
|
|
4045
|
+
COALESCE(SUM(r.cost_cents), 0) as cost_cents,
|
|
4046
|
+
COALESCE(SUM(r.tokens_used), 0) as tokens,
|
|
4047
|
+
COUNT(DISTINCT r.run_id) as runs
|
|
4048
|
+
FROM results r
|
|
4049
|
+
JOIN runs ru ON r.run_id = ru.id
|
|
4050
|
+
WHERE 1=1 ${dateFilter} ${projectFilter}
|
|
4051
|
+
GROUP BY r.model
|
|
4052
|
+
ORDER BY cost_cents DESC`).all(...projectParams);
|
|
4053
|
+
const byModel = {};
|
|
4054
|
+
for (const row of modelRows) {
|
|
4055
|
+
byModel[row.model] = {
|
|
4056
|
+
costCents: row.cost_cents,
|
|
4057
|
+
tokens: row.tokens,
|
|
4058
|
+
runs: row.runs
|
|
4059
|
+
};
|
|
4060
|
+
}
|
|
4061
|
+
const scenarioRows = db2.query(`SELECT
|
|
4062
|
+
r.scenario_id,
|
|
4063
|
+
COALESCE(s.name, r.scenario_id) as name,
|
|
4064
|
+
COALESCE(SUM(r.cost_cents), 0) as cost_cents,
|
|
4065
|
+
COALESCE(SUM(r.tokens_used), 0) as tokens,
|
|
4066
|
+
COUNT(DISTINCT r.run_id) as runs
|
|
4067
|
+
FROM results r
|
|
4068
|
+
JOIN runs ru ON r.run_id = ru.id
|
|
4069
|
+
LEFT JOIN scenarios s ON r.scenario_id = s.id
|
|
4070
|
+
WHERE 1=1 ${dateFilter} ${projectFilter}
|
|
4071
|
+
GROUP BY r.scenario_id
|
|
4072
|
+
ORDER BY cost_cents DESC
|
|
4073
|
+
LIMIT 10`).all(...projectParams);
|
|
4074
|
+
const byScenario = scenarioRows.map((row) => ({
|
|
4075
|
+
scenarioId: row.scenario_id,
|
|
4076
|
+
name: row.name,
|
|
4077
|
+
costCents: row.cost_cents,
|
|
4078
|
+
tokens: row.tokens,
|
|
4079
|
+
runs: row.runs
|
|
4080
|
+
}));
|
|
4081
|
+
const runCount = totalsRow.run_count;
|
|
4082
|
+
const avgCostPerRun = runCount > 0 ? totalsRow.total_cost / runCount : 0;
|
|
4083
|
+
const periodDays = getPeriodDays(period);
|
|
4084
|
+
const estimatedMonthlyCents = periodDays > 0 ? totalsRow.total_cost / periodDays * 30 : 0;
|
|
4085
|
+
return {
|
|
4086
|
+
period,
|
|
4087
|
+
totalCostCents: totalsRow.total_cost,
|
|
4088
|
+
totalTokens: totalsRow.total_tokens,
|
|
4089
|
+
runCount,
|
|
4090
|
+
byModel,
|
|
4091
|
+
byScenario,
|
|
4092
|
+
avgCostPerRun,
|
|
4093
|
+
estimatedMonthlyCents
|
|
4094
|
+
};
|
|
4095
|
+
}
|
|
4096
|
+
function checkBudget(estimatedCostCents) {
|
|
4097
|
+
const budget = loadBudgetConfig();
|
|
4098
|
+
if (estimatedCostCents > budget.maxPerRunCents) {
|
|
4099
|
+
return {
|
|
4100
|
+
allowed: false,
|
|
4101
|
+
warning: `Estimated cost (${formatDollars(estimatedCostCents)}) exceeds per-run limit (${formatDollars(budget.maxPerRunCents)})`
|
|
4102
|
+
};
|
|
4103
|
+
}
|
|
4104
|
+
const todaySummary = getCostSummary({ period: "day" });
|
|
4105
|
+
const projectedDaily = todaySummary.totalCostCents + estimatedCostCents;
|
|
4106
|
+
if (projectedDaily > budget.maxPerDayCents) {
|
|
4107
|
+
return {
|
|
4108
|
+
allowed: false,
|
|
4109
|
+
warning: `Daily spending (${formatDollars(todaySummary.totalCostCents)}) + this run (${formatDollars(estimatedCostCents)}) would exceed daily limit (${formatDollars(budget.maxPerDayCents)})`
|
|
4110
|
+
};
|
|
4111
|
+
}
|
|
4112
|
+
if (projectedDaily > budget.maxPerDayCents * budget.warnAtPercent) {
|
|
4113
|
+
return {
|
|
4114
|
+
allowed: true,
|
|
4115
|
+
warning: `Approaching daily limit: ${formatDollars(projectedDaily)} of ${formatDollars(budget.maxPerDayCents)} (${Math.round(projectedDaily / budget.maxPerDayCents * 100)}%)`
|
|
4116
|
+
};
|
|
4117
|
+
}
|
|
4118
|
+
return { allowed: true };
|
|
4119
|
+
}
|
|
4120
|
+
function formatDollars(cents) {
|
|
4121
|
+
return `$${(cents / 100).toFixed(2)}`;
|
|
4122
|
+
}
|
|
4123
|
+
function formatTokens(tokens) {
|
|
4124
|
+
if (tokens >= 1e6)
|
|
4125
|
+
return `${(tokens / 1e6).toFixed(1)}M`;
|
|
4126
|
+
if (tokens >= 1000)
|
|
4127
|
+
return `${(tokens / 1000).toFixed(1)}K`;
|
|
4128
|
+
return String(tokens);
|
|
4129
|
+
}
|
|
4130
|
+
function formatCostsTerminal(summary) {
|
|
4131
|
+
const lines = [];
|
|
4132
|
+
lines.push("");
|
|
4133
|
+
lines.push(source_default.bold(` Cost Summary (${summary.period})`));
|
|
4134
|
+
lines.push("");
|
|
4135
|
+
lines.push(` Total: ${source_default.yellow(formatDollars(summary.totalCostCents))} (${formatTokens(summary.totalTokens)} tokens across ${summary.runCount} runs)`);
|
|
4136
|
+
lines.push(` Avg/run: ${source_default.yellow(formatDollars(summary.avgCostPerRun))}`);
|
|
4137
|
+
lines.push(` Est/month: ${source_default.yellow(formatDollars(summary.estimatedMonthlyCents))}`);
|
|
4138
|
+
const modelEntries = Object.entries(summary.byModel);
|
|
4139
|
+
if (modelEntries.length > 0) {
|
|
4140
|
+
lines.push("");
|
|
4141
|
+
lines.push(source_default.bold(" By Model"));
|
|
4142
|
+
lines.push(` ${"Model".padEnd(40)} ${"Cost".padEnd(12)} ${"Tokens".padEnd(12)} Runs`);
|
|
4143
|
+
lines.push(` ${"\u2500".repeat(40)} ${"\u2500".repeat(12)} ${"\u2500".repeat(12)} ${"\u2500".repeat(6)}`);
|
|
4144
|
+
for (const [model, data] of modelEntries) {
|
|
4145
|
+
lines.push(` ${model.padEnd(40)} ${formatDollars(data.costCents).padEnd(12)} ${formatTokens(data.tokens).padEnd(12)} ${data.runs}`);
|
|
4146
|
+
}
|
|
4147
|
+
}
|
|
4148
|
+
if (summary.byScenario.length > 0) {
|
|
4149
|
+
lines.push("");
|
|
4150
|
+
lines.push(source_default.bold(" Top Scenarios by Cost"));
|
|
4151
|
+
lines.push(` ${"Scenario".padEnd(40)} ${"Cost".padEnd(12)} ${"Tokens".padEnd(12)} Runs`);
|
|
4152
|
+
lines.push(` ${"\u2500".repeat(40)} ${"\u2500".repeat(12)} ${"\u2500".repeat(12)} ${"\u2500".repeat(6)}`);
|
|
4153
|
+
for (const s of summary.byScenario) {
|
|
4154
|
+
const label = s.name.length > 38 ? s.name.slice(0, 35) + "..." : s.name;
|
|
4155
|
+
lines.push(` ${label.padEnd(40)} ${formatDollars(s.costCents).padEnd(12)} ${formatTokens(s.tokens).padEnd(12)} ${s.runs}`);
|
|
4156
|
+
}
|
|
4157
|
+
}
|
|
4158
|
+
lines.push("");
|
|
4159
|
+
return lines.join(`
|
|
4160
|
+
`);
|
|
4161
|
+
}
|
|
4162
|
+
function formatCostsJSON(summary) {
|
|
4163
|
+
return JSON.stringify(summary, null, 2);
|
|
4164
|
+
}
|
|
4165
|
+
// src/lib/watch.ts
|
|
4166
|
+
import { watch } from "fs";
|
|
4167
|
+
import { resolve } from "path";
|
|
4168
|
+
async function startWatcher(options) {
|
|
4169
|
+
const {
|
|
4170
|
+
dir,
|
|
4171
|
+
url,
|
|
4172
|
+
debounceMs = 2000,
|
|
4173
|
+
tags,
|
|
4174
|
+
priority,
|
|
4175
|
+
...runOpts
|
|
4176
|
+
} = options;
|
|
4177
|
+
const watchDir = resolve(dir);
|
|
4178
|
+
let debounceTimer = null;
|
|
4179
|
+
let isRunning = false;
|
|
4180
|
+
let lastChange = null;
|
|
4181
|
+
console.log("");
|
|
4182
|
+
console.log(source_default.bold(" Testers Watch Mode"));
|
|
4183
|
+
console.log(source_default.dim(` Watching: ${watchDir}`));
|
|
4184
|
+
console.log(source_default.dim(` Target: ${url}`));
|
|
4185
|
+
console.log(source_default.dim(` Debounce: ${debounceMs}ms`));
|
|
4186
|
+
console.log("");
|
|
4187
|
+
console.log(source_default.dim(" Waiting for file changes... (Ctrl+C to stop)"));
|
|
4188
|
+
console.log("");
|
|
4189
|
+
const runTests = async () => {
|
|
4190
|
+
if (isRunning)
|
|
4191
|
+
return;
|
|
4192
|
+
isRunning = true;
|
|
4193
|
+
console.log(source_default.blue(` [running] Testing against ${url}...`));
|
|
4194
|
+
if (lastChange) {
|
|
4195
|
+
console.log(source_default.dim(` Triggered by: ${lastChange}`));
|
|
4196
|
+
}
|
|
4197
|
+
console.log("");
|
|
4198
|
+
try {
|
|
4199
|
+
const { run, results } = await runByFilter({
|
|
4200
|
+
url,
|
|
4201
|
+
tags,
|
|
4202
|
+
priority,
|
|
4203
|
+
...runOpts
|
|
4204
|
+
});
|
|
4205
|
+
console.log(formatTerminal(run, results));
|
|
4206
|
+
const exitCode = getExitCode(run);
|
|
4207
|
+
if (exitCode === 0) {
|
|
4208
|
+
console.log(source_default.green(" All tests passed!"));
|
|
4209
|
+
} else {
|
|
4210
|
+
console.log(source_default.red(` ${run.failed} test(s) failed.`));
|
|
4211
|
+
}
|
|
4212
|
+
} catch (error) {
|
|
4213
|
+
console.error(source_default.red(` Error: ${error instanceof Error ? error.message : String(error)}`));
|
|
4214
|
+
} finally {
|
|
4215
|
+
isRunning = false;
|
|
4216
|
+
console.log("");
|
|
4217
|
+
console.log(source_default.dim(" Waiting for file changes..."));
|
|
4218
|
+
console.log("");
|
|
4219
|
+
}
|
|
4220
|
+
};
|
|
4221
|
+
const watcher = watch(watchDir, { recursive: true }, (_event, filename) => {
|
|
4222
|
+
if (!filename)
|
|
4223
|
+
return;
|
|
4224
|
+
const ignored = [
|
|
4225
|
+
"node_modules",
|
|
4226
|
+
".git",
|
|
4227
|
+
"dist",
|
|
4228
|
+
".testers",
|
|
4229
|
+
".next",
|
|
4230
|
+
".nuxt",
|
|
4231
|
+
".svelte-kit"
|
|
4232
|
+
];
|
|
4233
|
+
if (ignored.some((dir2) => filename.includes(dir2)))
|
|
4234
|
+
return;
|
|
4235
|
+
const extensions = [".ts", ".tsx", ".js", ".jsx", ".vue", ".svelte", ".css", ".html"];
|
|
4236
|
+
if (!extensions.some((ext) => filename.endsWith(ext)))
|
|
4237
|
+
return;
|
|
4238
|
+
lastChange = filename;
|
|
4239
|
+
console.log(source_default.yellow(` [change] ${filename}`));
|
|
4240
|
+
if (debounceTimer)
|
|
4241
|
+
clearTimeout(debounceTimer);
|
|
4242
|
+
debounceTimer = setTimeout(() => {
|
|
4243
|
+
runTests();
|
|
4244
|
+
}, debounceMs);
|
|
4245
|
+
});
|
|
4246
|
+
const cleanup = () => {
|
|
4247
|
+
watcher.close();
|
|
4248
|
+
if (debounceTimer)
|
|
4249
|
+
clearTimeout(debounceTimer);
|
|
4250
|
+
console.log("");
|
|
4251
|
+
console.log(source_default.dim(" Watch mode stopped."));
|
|
4252
|
+
process.exit(0);
|
|
4253
|
+
};
|
|
4254
|
+
process.on("SIGINT", cleanup);
|
|
4255
|
+
process.on("SIGTERM", cleanup);
|
|
4256
|
+
await new Promise(() => {});
|
|
4257
|
+
}
|
|
4258
|
+
// src/lib/webhooks.ts
|
|
4259
|
+
init_database();
|
|
4260
|
+
function fromRow2(row) {
|
|
4261
|
+
return {
|
|
4262
|
+
id: row.id,
|
|
4263
|
+
url: row.url,
|
|
4264
|
+
events: JSON.parse(row.events),
|
|
4265
|
+
projectId: row.project_id,
|
|
4266
|
+
secret: row.secret,
|
|
4267
|
+
active: row.active === 1,
|
|
4268
|
+
createdAt: row.created_at
|
|
4269
|
+
};
|
|
4270
|
+
}
|
|
4271
|
+
function createWebhook(input) {
|
|
4272
|
+
const db2 = getDatabase();
|
|
4273
|
+
const id = uuid();
|
|
4274
|
+
const events = input.events ?? ["failed"];
|
|
4275
|
+
const secret = input.secret ?? crypto.randomUUID().replace(/-/g, "");
|
|
4276
|
+
db2.query(`
|
|
4277
|
+
INSERT INTO webhooks (id, url, events, project_id, secret, active, created_at)
|
|
4278
|
+
VALUES (?, ?, ?, ?, ?, 1, ?)
|
|
4279
|
+
`).run(id, input.url, JSON.stringify(events), input.projectId ?? null, secret, now());
|
|
4280
|
+
return getWebhook(id);
|
|
4281
|
+
}
|
|
4282
|
+
function getWebhook(id) {
|
|
4283
|
+
const db2 = getDatabase();
|
|
4284
|
+
const row = db2.query("SELECT * FROM webhooks WHERE id = ?").get(id);
|
|
4285
|
+
if (!row) {
|
|
4286
|
+
const rows = db2.query("SELECT * FROM webhooks WHERE id LIKE ? || '%'").all(id);
|
|
4287
|
+
if (rows.length === 1)
|
|
4288
|
+
return fromRow2(rows[0]);
|
|
4289
|
+
return null;
|
|
4290
|
+
}
|
|
4291
|
+
return fromRow2(row);
|
|
4292
|
+
}
|
|
4293
|
+
function listWebhooks(projectId) {
|
|
4294
|
+
const db2 = getDatabase();
|
|
4295
|
+
let query = "SELECT * FROM webhooks WHERE active = 1";
|
|
4296
|
+
const params = [];
|
|
4297
|
+
if (projectId) {
|
|
4298
|
+
query += " AND (project_id = ? OR project_id IS NULL)";
|
|
4299
|
+
params.push(projectId);
|
|
4300
|
+
}
|
|
4301
|
+
query += " ORDER BY created_at DESC";
|
|
4302
|
+
const rows = db2.query(query).all(...params);
|
|
4303
|
+
return rows.map(fromRow2);
|
|
4304
|
+
}
|
|
4305
|
+
function deleteWebhook(id) {
|
|
4306
|
+
const db2 = getDatabase();
|
|
4307
|
+
const webhook = getWebhook(id);
|
|
4308
|
+
if (!webhook)
|
|
4309
|
+
return false;
|
|
4310
|
+
db2.query("DELETE FROM webhooks WHERE id = ?").run(webhook.id);
|
|
4311
|
+
return true;
|
|
4312
|
+
}
|
|
4313
|
+
function signPayload(body, secret) {
|
|
4314
|
+
const encoder = new TextEncoder;
|
|
4315
|
+
const key = encoder.encode(secret);
|
|
4316
|
+
const data = encoder.encode(body);
|
|
4317
|
+
let hash = 0;
|
|
4318
|
+
for (let i = 0;i < data.length; i++) {
|
|
4319
|
+
hash = (hash << 5) - hash + data[i] + (key[i % key.length] ?? 0) | 0;
|
|
4320
|
+
}
|
|
4321
|
+
return `sha256=${Math.abs(hash).toString(16).padStart(16, "0")}`;
|
|
4322
|
+
}
|
|
4323
|
+
function formatSlackPayload(payload) {
|
|
4324
|
+
const status = payload.run.status === "passed" ? ":white_check_mark:" : ":x:";
|
|
4325
|
+
const color = payload.run.status === "passed" ? "#22c55e" : "#ef4444";
|
|
4326
|
+
return {
|
|
4327
|
+
attachments: [
|
|
4328
|
+
{
|
|
4329
|
+
color,
|
|
4330
|
+
blocks: [
|
|
4331
|
+
{
|
|
4332
|
+
type: "section",
|
|
4333
|
+
text: {
|
|
4334
|
+
type: "mrkdwn",
|
|
4335
|
+
text: `${status} *Test Run ${payload.run.status.toUpperCase()}*
|
|
4336
|
+
` + `URL: ${payload.run.url}
|
|
4337
|
+
` + `Results: ${payload.run.passed}/${payload.run.total} passed` + (payload.run.failed > 0 ? ` (${payload.run.failed} failed)` : "") + (payload.schedule ? `
|
|
4338
|
+
Schedule: ${payload.schedule.name}` : "")
|
|
4339
|
+
}
|
|
4340
|
+
}
|
|
4341
|
+
]
|
|
4342
|
+
}
|
|
4343
|
+
]
|
|
4344
|
+
};
|
|
4345
|
+
}
|
|
4346
|
+
async function dispatchWebhooks(event, run, schedule) {
|
|
4347
|
+
const webhooks = listWebhooks(run.projectId ?? undefined);
|
|
4348
|
+
const payload = {
|
|
4349
|
+
event,
|
|
4350
|
+
run: {
|
|
4351
|
+
id: run.id,
|
|
4352
|
+
url: run.url,
|
|
4353
|
+
status: run.status,
|
|
4354
|
+
passed: run.passed,
|
|
4355
|
+
failed: run.failed,
|
|
4356
|
+
total: run.total
|
|
4357
|
+
},
|
|
4358
|
+
schedule,
|
|
4359
|
+
timestamp: new Date().toISOString()
|
|
4360
|
+
};
|
|
4361
|
+
for (const webhook of webhooks) {
|
|
4362
|
+
if (!webhook.events.includes(event) && !webhook.events.includes("*"))
|
|
4363
|
+
continue;
|
|
4364
|
+
const isSlack = webhook.url.includes("hooks.slack.com");
|
|
4365
|
+
const body = isSlack ? JSON.stringify(formatSlackPayload(payload)) : JSON.stringify(payload);
|
|
4366
|
+
const headers = {
|
|
4367
|
+
"Content-Type": "application/json"
|
|
4368
|
+
};
|
|
4369
|
+
if (webhook.secret) {
|
|
4370
|
+
headers["X-Testers-Signature"] = signPayload(body, webhook.secret);
|
|
4371
|
+
}
|
|
4372
|
+
try {
|
|
4373
|
+
const response = await fetch(webhook.url, {
|
|
4374
|
+
method: "POST",
|
|
4375
|
+
headers,
|
|
4376
|
+
body
|
|
4377
|
+
});
|
|
4378
|
+
if (!response.ok) {
|
|
4379
|
+
await new Promise((r) => setTimeout(r, 5000));
|
|
4380
|
+
await fetch(webhook.url, { method: "POST", headers, body });
|
|
4381
|
+
}
|
|
4382
|
+
} catch {}
|
|
4383
|
+
}
|
|
4384
|
+
}
|
|
4385
|
+
async function testWebhook(id) {
|
|
4386
|
+
const webhook = getWebhook(id);
|
|
4387
|
+
if (!webhook)
|
|
4388
|
+
return false;
|
|
4389
|
+
const testPayload = {
|
|
4390
|
+
event: "test",
|
|
4391
|
+
run: { id: "test-run", url: "http://localhost:3000", status: "passed", passed: 3, failed: 0, total: 3 },
|
|
4392
|
+
timestamp: new Date().toISOString()
|
|
4393
|
+
};
|
|
4394
|
+
try {
|
|
4395
|
+
const body = JSON.stringify(testPayload);
|
|
4396
|
+
const response = await fetch(webhook.url, {
|
|
4397
|
+
method: "POST",
|
|
4398
|
+
headers: {
|
|
4399
|
+
"Content-Type": "application/json",
|
|
4400
|
+
...webhook.secret ? { "X-Testers-Signature": signPayload(body, webhook.secret) } : {}
|
|
4401
|
+
},
|
|
4402
|
+
body
|
|
4403
|
+
});
|
|
4404
|
+
return response.ok;
|
|
4405
|
+
} catch {
|
|
4406
|
+
return false;
|
|
4407
|
+
}
|
|
4408
|
+
}
|
|
3058
4409
|
export {
|
|
4410
|
+
writeScenarioMeta,
|
|
4411
|
+
writeRunMeta,
|
|
3059
4412
|
uuid,
|
|
3060
4413
|
updateSchedule,
|
|
3061
4414
|
updateScenario,
|
|
3062
4415
|
updateRun,
|
|
3063
4416
|
updateResult,
|
|
3064
4417
|
updateLastRun,
|
|
4418
|
+
testWebhook,
|
|
3065
4419
|
taskToScenarioInput,
|
|
4420
|
+
startWatcher,
|
|
3066
4421
|
slugify,
|
|
3067
4422
|
shouldRunAt,
|
|
3068
4423
|
shortUuid,
|
|
3069
4424
|
screenshotFromRow,
|
|
3070
4425
|
scheduleFromRow,
|
|
3071
4426
|
scenarioFromRow,
|
|
4427
|
+
runSmoke,
|
|
3072
4428
|
runSingleScenario,
|
|
3073
4429
|
runFromRow,
|
|
3074
4430
|
runByFilter,
|
|
@@ -3082,22 +4438,31 @@ export {
|
|
|
3082
4438
|
registerAgent,
|
|
3083
4439
|
pullTasks,
|
|
3084
4440
|
projectFromRow,
|
|
4441
|
+
parseSmokeIssues,
|
|
3085
4442
|
parseCronField,
|
|
3086
4443
|
parseCron,
|
|
3087
4444
|
onRunEvent,
|
|
3088
4445
|
now,
|
|
3089
4446
|
markTodoDone,
|
|
3090
4447
|
loadConfig,
|
|
4448
|
+
listWebhooks,
|
|
4449
|
+
listTemplateNames,
|
|
3091
4450
|
listScreenshots,
|
|
3092
4451
|
listSchedules,
|
|
3093
4452
|
listScenarios,
|
|
3094
4453
|
listRuns,
|
|
3095
4454
|
listResults,
|
|
3096
4455
|
listProjects,
|
|
4456
|
+
listAuthPresets,
|
|
3097
4457
|
listAgents,
|
|
3098
4458
|
launchBrowser,
|
|
3099
4459
|
installBrowser,
|
|
4460
|
+
initProject,
|
|
3100
4461
|
importFromTodos,
|
|
4462
|
+
imageToBase64,
|
|
4463
|
+
getWebhook,
|
|
4464
|
+
getTemplate,
|
|
4465
|
+
getStarterScenarios,
|
|
3101
4466
|
getScreenshotsByResult,
|
|
3102
4467
|
getScreenshotDir,
|
|
3103
4468
|
getScreenshot,
|
|
@@ -3115,21 +4480,36 @@ export {
|
|
|
3115
4480
|
getEnabledSchedules,
|
|
3116
4481
|
getDefaultConfig,
|
|
3117
4482
|
getDatabase,
|
|
4483
|
+
getCostSummary,
|
|
4484
|
+
getAuthPreset,
|
|
3118
4485
|
getAgentByName,
|
|
3119
4486
|
getAgent,
|
|
4487
|
+
generateLatestReport,
|
|
4488
|
+
generateHtmlReport,
|
|
3120
4489
|
generateFilename,
|
|
3121
4490
|
formatTerminal,
|
|
3122
4491
|
formatSummary,
|
|
4492
|
+
formatSmokeReport,
|
|
3123
4493
|
formatScenarioList,
|
|
3124
4494
|
formatRunList,
|
|
3125
4495
|
formatResultDetail,
|
|
3126
4496
|
formatJSON,
|
|
4497
|
+
formatDiffTerminal,
|
|
4498
|
+
formatDiffJSON,
|
|
4499
|
+
formatCostsTerminal,
|
|
4500
|
+
formatCostsJSON,
|
|
3127
4501
|
executeTool,
|
|
3128
4502
|
ensureProject,
|
|
3129
4503
|
ensureDir,
|
|
4504
|
+
dispatchWebhooks,
|
|
4505
|
+
diffRuns,
|
|
4506
|
+
detectFramework,
|
|
4507
|
+
deleteWebhook,
|
|
3130
4508
|
deleteSchedule,
|
|
3131
4509
|
deleteScenario,
|
|
3132
4510
|
deleteRun,
|
|
4511
|
+
deleteAuthPreset,
|
|
4512
|
+
createWebhook,
|
|
3133
4513
|
createScreenshot,
|
|
3134
4514
|
createSchedule,
|
|
3135
4515
|
createScenario,
|
|
@@ -3137,9 +4517,11 @@ export {
|
|
|
3137
4517
|
createResult,
|
|
3138
4518
|
createProject,
|
|
3139
4519
|
createClient,
|
|
4520
|
+
createAuthPreset,
|
|
3140
4521
|
connectToTodos,
|
|
3141
4522
|
closeDatabase,
|
|
3142
4523
|
closeBrowser,
|
|
4524
|
+
checkBudget,
|
|
3143
4525
|
agentFromRow,
|
|
3144
4526
|
VersionConflictError,
|
|
3145
4527
|
TodosConnectionError,
|
|
@@ -3147,6 +4529,7 @@ export {
|
|
|
3147
4529
|
Scheduler,
|
|
3148
4530
|
ScheduleNotFoundError,
|
|
3149
4531
|
ScenarioNotFoundError,
|
|
4532
|
+
SCENARIO_TEMPLATES,
|
|
3150
4533
|
RunNotFoundError,
|
|
3151
4534
|
ResultNotFoundError,
|
|
3152
4535
|
ProjectNotFoundError,
|