@hasna/testers 0.0.5 → 0.0.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/index.js +1580 -342
- package/dist/db/database.d.ts.map +1 -1
- package/dist/db/environments.d.ts +22 -0
- package/dist/db/environments.d.ts.map +1 -0
- package/dist/db/flows.d.ts +12 -0
- package/dist/db/flows.d.ts.map +1 -0
- package/dist/db/runs.d.ts.map +1 -1
- package/dist/db/scenarios.d.ts.map +1 -1
- package/dist/index.d.ts +3 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +481 -165
- package/dist/lib/assertions.d.ts +26 -0
- package/dist/lib/assertions.d.ts.map +1 -0
- package/dist/lib/ci.d.ts +2 -0
- package/dist/lib/ci.d.ts.map +1 -0
- package/dist/lib/openapi-import.d.ts +8 -0
- package/dist/lib/openapi-import.d.ts.map +1 -0
- package/dist/lib/recorder.d.ts +24 -0
- package/dist/lib/recorder.d.ts.map +1 -0
- package/dist/lib/runner.d.ts.map +1 -1
- package/dist/lib/visual-diff.d.ts +36 -0
- package/dist/lib/visual-diff.d.ts.map +1 -0
- package/dist/mcp/index.js +3102 -2693
- package/dist/server/index.js +500 -88
- package/dist/types/index.d.ts +44 -0
- package/dist/types/index.d.ts.map +1 -1
- package/package.json +1 -1
package/dist/server/index.js
CHANGED
|
@@ -1,17 +1,18 @@
|
|
|
1
1
|
#!/usr/bin/env bun
|
|
2
2
|
// @bun
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
3
|
+
var __defProp = Object.defineProperty;
|
|
4
|
+
var __export = (target, all) => {
|
|
5
|
+
for (var name in all)
|
|
6
|
+
__defProp(target, name, {
|
|
7
|
+
get: all[name],
|
|
8
|
+
enumerable: true,
|
|
9
|
+
configurable: true,
|
|
10
|
+
set: (newValue) => all[name] = () => newValue
|
|
11
|
+
});
|
|
12
|
+
};
|
|
13
|
+
var __esm = (fn, res) => () => (fn && (res = fn(fn = 0)), res);
|
|
8
14
|
|
|
9
15
|
// src/types/index.ts
|
|
10
|
-
var MODEL_MAP = {
|
|
11
|
-
quick: "claude-haiku-4-5-20251001",
|
|
12
|
-
thorough: "claude-sonnet-4-6-20260311",
|
|
13
|
-
deep: "claude-opus-4-6-20260311"
|
|
14
|
-
};
|
|
15
16
|
function scenarioFromRow(row) {
|
|
16
17
|
return {
|
|
17
18
|
id: row.id,
|
|
@@ -28,6 +29,7 @@ function scenarioFromRow(row) {
|
|
|
28
29
|
requiresAuth: row.requires_auth === 1,
|
|
29
30
|
authConfig: row.auth_config ? JSON.parse(row.auth_config) : null,
|
|
30
31
|
metadata: row.metadata ? JSON.parse(row.metadata) : null,
|
|
32
|
+
assertions: JSON.parse(row.assertions || "[]"),
|
|
31
33
|
version: row.version,
|
|
32
34
|
createdAt: row.created_at,
|
|
33
35
|
updatedAt: row.updated_at
|
|
@@ -47,7 +49,8 @@ function runFromRow(row) {
|
|
|
47
49
|
failed: row.failed,
|
|
48
50
|
startedAt: row.started_at,
|
|
49
51
|
finishedAt: row.finished_at,
|
|
50
|
-
metadata: row.metadata ? JSON.parse(row.metadata) : null
|
|
52
|
+
metadata: row.metadata ? JSON.parse(row.metadata) : null,
|
|
53
|
+
isBaseline: row.is_baseline === 1
|
|
51
54
|
};
|
|
52
55
|
}
|
|
53
56
|
function resultFromRow(row) {
|
|
@@ -103,39 +106,61 @@ function scheduleFromRow(row) {
|
|
|
103
106
|
updatedAt: row.updated_at
|
|
104
107
|
};
|
|
105
108
|
}
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
this.name = "BrowserError";
|
|
117
|
-
}
|
|
118
|
-
}
|
|
119
|
-
|
|
120
|
-
class AIClientError extends Error {
|
|
121
|
-
constructor(message) {
|
|
122
|
-
super(message);
|
|
123
|
-
this.name = "AIClientError";
|
|
124
|
-
}
|
|
125
|
-
}
|
|
126
|
-
class ScheduleNotFoundError extends Error {
|
|
127
|
-
constructor(id) {
|
|
128
|
-
super(`Schedule not found: ${id}`);
|
|
129
|
-
this.name = "ScheduleNotFoundError";
|
|
130
|
-
}
|
|
109
|
+
function flowFromRow(row) {
|
|
110
|
+
return {
|
|
111
|
+
id: row.id,
|
|
112
|
+
projectId: row.project_id,
|
|
113
|
+
name: row.name,
|
|
114
|
+
description: row.description,
|
|
115
|
+
scenarioIds: JSON.parse(row.scenario_ids),
|
|
116
|
+
createdAt: row.created_at,
|
|
117
|
+
updatedAt: row.updated_at
|
|
118
|
+
};
|
|
131
119
|
}
|
|
120
|
+
var MODEL_MAP, VersionConflictError, BrowserError, AIClientError, ScheduleNotFoundError, DependencyCycleError;
|
|
121
|
+
var init_types = __esm(() => {
|
|
122
|
+
MODEL_MAP = {
|
|
123
|
+
quick: "claude-haiku-4-5-20251001",
|
|
124
|
+
thorough: "claude-sonnet-4-6-20260311",
|
|
125
|
+
deep: "claude-opus-4-6-20260311"
|
|
126
|
+
};
|
|
127
|
+
VersionConflictError = class VersionConflictError extends Error {
|
|
128
|
+
constructor(entity, id) {
|
|
129
|
+
super(`Version conflict on ${entity}: ${id}`);
|
|
130
|
+
this.name = "VersionConflictError";
|
|
131
|
+
}
|
|
132
|
+
};
|
|
133
|
+
BrowserError = class BrowserError extends Error {
|
|
134
|
+
constructor(message) {
|
|
135
|
+
super(message);
|
|
136
|
+
this.name = "BrowserError";
|
|
137
|
+
}
|
|
138
|
+
};
|
|
139
|
+
AIClientError = class AIClientError extends Error {
|
|
140
|
+
constructor(message) {
|
|
141
|
+
super(message);
|
|
142
|
+
this.name = "AIClientError";
|
|
143
|
+
}
|
|
144
|
+
};
|
|
145
|
+
ScheduleNotFoundError = class ScheduleNotFoundError extends Error {
|
|
146
|
+
constructor(id) {
|
|
147
|
+
super(`Schedule not found: ${id}`);
|
|
148
|
+
this.name = "ScheduleNotFoundError";
|
|
149
|
+
}
|
|
150
|
+
};
|
|
151
|
+
DependencyCycleError = class DependencyCycleError extends Error {
|
|
152
|
+
constructor(scenarioId, dependsOn) {
|
|
153
|
+
super(`Adding dependency ${dependsOn} to ${scenarioId} would create a cycle`);
|
|
154
|
+
this.name = "DependencyCycleError";
|
|
155
|
+
}
|
|
156
|
+
};
|
|
157
|
+
});
|
|
132
158
|
|
|
133
159
|
// src/db/database.ts
|
|
134
160
|
import { Database } from "bun:sqlite";
|
|
135
161
|
import { mkdirSync, existsSync } from "fs";
|
|
136
162
|
import { dirname, join } from "path";
|
|
137
163
|
import { homedir } from "os";
|
|
138
|
-
var db = null;
|
|
139
164
|
function now() {
|
|
140
165
|
return new Date().toISOString();
|
|
141
166
|
}
|
|
@@ -154,8 +179,50 @@ function resolveDbPath() {
|
|
|
154
179
|
mkdirSync(dir, { recursive: true });
|
|
155
180
|
return join(dir, "testers.db");
|
|
156
181
|
}
|
|
157
|
-
|
|
158
|
-
|
|
182
|
+
function applyMigrations(database) {
|
|
183
|
+
const applied = database.query("SELECT id FROM _migrations ORDER BY id").all();
|
|
184
|
+
const appliedIds = new Set(applied.map((r) => r.id));
|
|
185
|
+
for (let i = 0;i < MIGRATIONS.length; i++) {
|
|
186
|
+
const migrationId = i + 1;
|
|
187
|
+
if (appliedIds.has(migrationId))
|
|
188
|
+
continue;
|
|
189
|
+
const migration = MIGRATIONS[i];
|
|
190
|
+
database.exec(migration);
|
|
191
|
+
database.query("INSERT INTO _migrations (id, applied_at) VALUES (?, ?)").run(migrationId, now());
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
function getDatabase() {
|
|
195
|
+
if (db)
|
|
196
|
+
return db;
|
|
197
|
+
const dbPath = resolveDbPath();
|
|
198
|
+
const dir = dirname(dbPath);
|
|
199
|
+
if (dbPath !== ":memory:" && !existsSync(dir)) {
|
|
200
|
+
mkdirSync(dir, { recursive: true });
|
|
201
|
+
}
|
|
202
|
+
db = new Database(dbPath);
|
|
203
|
+
db.exec("PRAGMA journal_mode = WAL");
|
|
204
|
+
db.exec("PRAGMA foreign_keys = ON");
|
|
205
|
+
db.exec("PRAGMA busy_timeout = 5000");
|
|
206
|
+
db.exec(`
|
|
207
|
+
CREATE TABLE IF NOT EXISTS _migrations (
|
|
208
|
+
id INTEGER PRIMARY KEY,
|
|
209
|
+
applied_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
210
|
+
);
|
|
211
|
+
`);
|
|
212
|
+
applyMigrations(db);
|
|
213
|
+
return db;
|
|
214
|
+
}
|
|
215
|
+
function resolvePartialId(table, partialId) {
|
|
216
|
+
const database = getDatabase();
|
|
217
|
+
const rows = database.query(`SELECT id FROM ${table} WHERE id LIKE ? || '%'`).all(partialId);
|
|
218
|
+
if (rows.length === 1)
|
|
219
|
+
return rows[0].id;
|
|
220
|
+
return null;
|
|
221
|
+
}
|
|
222
|
+
var db = null, MIGRATIONS;
|
|
223
|
+
var init_database = __esm(() => {
|
|
224
|
+
MIGRATIONS = [
|
|
225
|
+
`
|
|
159
226
|
CREATE TABLE IF NOT EXISTS projects (
|
|
160
227
|
id TEXT PRIMARY KEY,
|
|
161
228
|
name TEXT NOT NULL UNIQUE,
|
|
@@ -244,7 +311,7 @@ var MIGRATIONS = [
|
|
|
244
311
|
applied_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
245
312
|
);
|
|
246
313
|
`,
|
|
247
|
-
|
|
314
|
+
`
|
|
248
315
|
CREATE INDEX IF NOT EXISTS idx_scenarios_project ON scenarios(project_id);
|
|
249
316
|
CREATE INDEX IF NOT EXISTS idx_scenarios_priority ON scenarios(priority);
|
|
250
317
|
CREATE INDEX IF NOT EXISTS idx_scenarios_short_id ON scenarios(short_id);
|
|
@@ -255,11 +322,11 @@ var MIGRATIONS = [
|
|
|
255
322
|
CREATE INDEX IF NOT EXISTS idx_results_status ON results(status);
|
|
256
323
|
CREATE INDEX IF NOT EXISTS idx_screenshots_result ON screenshots(result_id);
|
|
257
324
|
`,
|
|
258
|
-
|
|
325
|
+
`
|
|
259
326
|
ALTER TABLE projects ADD COLUMN scenario_prefix TEXT DEFAULT 'TST';
|
|
260
327
|
ALTER TABLE projects ADD COLUMN scenario_counter INTEGER DEFAULT 0;
|
|
261
328
|
`,
|
|
262
|
-
|
|
329
|
+
`
|
|
263
330
|
CREATE TABLE IF NOT EXISTS schedules (
|
|
264
331
|
id TEXT PRIMARY KEY,
|
|
265
332
|
project_id TEXT REFERENCES projects(id) ON DELETE CASCADE,
|
|
@@ -283,12 +350,12 @@ var MIGRATIONS = [
|
|
|
283
350
|
CREATE INDEX IF NOT EXISTS idx_schedules_enabled ON schedules(enabled);
|
|
284
351
|
CREATE INDEX IF NOT EXISTS idx_schedules_next_run ON schedules(next_run_at);
|
|
285
352
|
`,
|
|
286
|
-
|
|
353
|
+
`
|
|
287
354
|
ALTER TABLE screenshots ADD COLUMN description TEXT;
|
|
288
355
|
ALTER TABLE screenshots ADD COLUMN page_url TEXT;
|
|
289
356
|
ALTER TABLE screenshots ADD COLUMN thumbnail_path TEXT;
|
|
290
357
|
`,
|
|
291
|
-
|
|
358
|
+
`
|
|
292
359
|
CREATE TABLE IF NOT EXISTS auth_presets (
|
|
293
360
|
id TEXT PRIMARY KEY,
|
|
294
361
|
name TEXT NOT NULL UNIQUE,
|
|
@@ -299,7 +366,7 @@ var MIGRATIONS = [
|
|
|
299
366
|
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
300
367
|
);
|
|
301
368
|
`,
|
|
302
|
-
|
|
369
|
+
`
|
|
303
370
|
CREATE TABLE IF NOT EXISTS webhooks (
|
|
304
371
|
id TEXT PRIMARY KEY,
|
|
305
372
|
url TEXT NOT NULL,
|
|
@@ -310,50 +377,209 @@ var MIGRATIONS = [
|
|
|
310
377
|
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
311
378
|
);
|
|
312
379
|
CREATE INDEX IF NOT EXISTS idx_webhooks_active ON webhooks(active);
|
|
380
|
+
`,
|
|
381
|
+
`
|
|
382
|
+
CREATE TABLE IF NOT EXISTS scenario_dependencies (
|
|
383
|
+
scenario_id TEXT NOT NULL REFERENCES scenarios(id) ON DELETE CASCADE,
|
|
384
|
+
depends_on TEXT NOT NULL REFERENCES scenarios(id) ON DELETE CASCADE,
|
|
385
|
+
PRIMARY KEY (scenario_id, depends_on),
|
|
386
|
+
CHECK (scenario_id != depends_on)
|
|
387
|
+
);
|
|
388
|
+
|
|
389
|
+
CREATE TABLE IF NOT EXISTS flows (
|
|
390
|
+
id TEXT PRIMARY KEY,
|
|
391
|
+
project_id TEXT REFERENCES projects(id) ON DELETE CASCADE,
|
|
392
|
+
name TEXT NOT NULL,
|
|
393
|
+
description TEXT,
|
|
394
|
+
scenario_ids TEXT NOT NULL DEFAULT '[]',
|
|
395
|
+
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
396
|
+
updated_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
397
|
+
);
|
|
398
|
+
|
|
399
|
+
CREATE INDEX IF NOT EXISTS idx_deps_scenario ON scenario_dependencies(scenario_id);
|
|
400
|
+
CREATE INDEX IF NOT EXISTS idx_deps_depends ON scenario_dependencies(depends_on);
|
|
401
|
+
CREATE INDEX IF NOT EXISTS idx_flows_project ON flows(project_id);
|
|
402
|
+
`,
|
|
403
|
+
`
|
|
404
|
+
ALTER TABLE scenarios ADD COLUMN assertions TEXT DEFAULT '[]';
|
|
405
|
+
`,
|
|
406
|
+
`
|
|
407
|
+
CREATE TABLE IF NOT EXISTS environments (
|
|
408
|
+
id TEXT PRIMARY KEY,
|
|
409
|
+
name TEXT NOT NULL UNIQUE,
|
|
410
|
+
url TEXT NOT NULL,
|
|
411
|
+
auth_preset_name TEXT,
|
|
412
|
+
project_id TEXT REFERENCES projects(id) ON DELETE CASCADE,
|
|
413
|
+
is_default INTEGER NOT NULL DEFAULT 0,
|
|
414
|
+
metadata TEXT DEFAULT '{}',
|
|
415
|
+
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
416
|
+
);
|
|
417
|
+
`,
|
|
418
|
+
`
|
|
419
|
+
ALTER TABLE runs ADD COLUMN is_baseline INTEGER NOT NULL DEFAULT 0;
|
|
313
420
|
`
|
|
314
|
-
];
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
421
|
+
];
|
|
422
|
+
});
|
|
423
|
+
|
|
424
|
+
// src/db/flows.ts
|
|
425
|
+
var exports_flows = {};
|
|
426
|
+
__export(exports_flows, {
|
|
427
|
+
topologicalSort: () => topologicalSort,
|
|
428
|
+
removeDependency: () => removeDependency,
|
|
429
|
+
listFlows: () => listFlows,
|
|
430
|
+
getTransitiveDependencies: () => getTransitiveDependencies,
|
|
431
|
+
getFlow: () => getFlow,
|
|
432
|
+
getDependents: () => getDependents,
|
|
433
|
+
getDependencies: () => getDependencies,
|
|
434
|
+
deleteFlow: () => deleteFlow,
|
|
435
|
+
createFlow: () => createFlow,
|
|
436
|
+
addDependency: () => addDependency
|
|
437
|
+
});
|
|
438
|
+
function addDependency(scenarioId, dependsOn) {
|
|
439
|
+
const db2 = getDatabase();
|
|
440
|
+
const visited = new Set;
|
|
441
|
+
const queue = [dependsOn];
|
|
442
|
+
while (queue.length > 0) {
|
|
443
|
+
const current = queue.shift();
|
|
444
|
+
if (current === scenarioId) {
|
|
445
|
+
throw new DependencyCycleError(scenarioId, dependsOn);
|
|
446
|
+
}
|
|
447
|
+
if (visited.has(current))
|
|
321
448
|
continue;
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
449
|
+
visited.add(current);
|
|
450
|
+
const deps = db2.query("SELECT depends_on FROM scenario_dependencies WHERE scenario_id = ?").all(current);
|
|
451
|
+
for (const dep of deps) {
|
|
452
|
+
if (!visited.has(dep.depends_on)) {
|
|
453
|
+
queue.push(dep.depends_on);
|
|
454
|
+
}
|
|
455
|
+
}
|
|
325
456
|
}
|
|
457
|
+
db2.query("INSERT OR IGNORE INTO scenario_dependencies (scenario_id, depends_on) VALUES (?, ?)").run(scenarioId, dependsOn);
|
|
326
458
|
}
|
|
327
|
-
function
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
459
|
+
function removeDependency(scenarioId, dependsOn) {
|
|
460
|
+
const db2 = getDatabase();
|
|
461
|
+
const result = db2.query("DELETE FROM scenario_dependencies WHERE scenario_id = ? AND depends_on = ?").run(scenarioId, dependsOn);
|
|
462
|
+
return result.changes > 0;
|
|
463
|
+
}
|
|
464
|
+
function getDependencies(scenarioId) {
|
|
465
|
+
const db2 = getDatabase();
|
|
466
|
+
const rows = db2.query("SELECT depends_on FROM scenario_dependencies WHERE scenario_id = ?").all(scenarioId);
|
|
467
|
+
return rows.map((r) => r.depends_on);
|
|
468
|
+
}
|
|
469
|
+
function getDependents(scenarioId) {
|
|
470
|
+
const db2 = getDatabase();
|
|
471
|
+
const rows = db2.query("SELECT scenario_id FROM scenario_dependencies WHERE depends_on = ?").all(scenarioId);
|
|
472
|
+
return rows.map((r) => r.scenario_id);
|
|
473
|
+
}
|
|
474
|
+
function getTransitiveDependencies(scenarioId) {
|
|
475
|
+
const db2 = getDatabase();
|
|
476
|
+
const visited = new Set;
|
|
477
|
+
const queue = [scenarioId];
|
|
478
|
+
while (queue.length > 0) {
|
|
479
|
+
const current = queue.shift();
|
|
480
|
+
const deps = db2.query("SELECT depends_on FROM scenario_dependencies WHERE scenario_id = ?").all(current);
|
|
481
|
+
for (const dep of deps) {
|
|
482
|
+
if (!visited.has(dep.depends_on)) {
|
|
483
|
+
visited.add(dep.depends_on);
|
|
484
|
+
queue.push(dep.depends_on);
|
|
485
|
+
}
|
|
486
|
+
}
|
|
334
487
|
}
|
|
335
|
-
|
|
336
|
-
db.exec("PRAGMA journal_mode = WAL");
|
|
337
|
-
db.exec("PRAGMA foreign_keys = ON");
|
|
338
|
-
db.exec("PRAGMA busy_timeout = 5000");
|
|
339
|
-
db.exec(`
|
|
340
|
-
CREATE TABLE IF NOT EXISTS _migrations (
|
|
341
|
-
id INTEGER PRIMARY KEY,
|
|
342
|
-
applied_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
343
|
-
);
|
|
344
|
-
`);
|
|
345
|
-
applyMigrations(db);
|
|
346
|
-
return db;
|
|
488
|
+
return Array.from(visited);
|
|
347
489
|
}
|
|
348
|
-
function
|
|
349
|
-
const
|
|
350
|
-
const
|
|
351
|
-
|
|
352
|
-
|
|
490
|
+
function topologicalSort(scenarioIds) {
|
|
491
|
+
const db2 = getDatabase();
|
|
492
|
+
const idSet = new Set(scenarioIds);
|
|
493
|
+
const inDegree = new Map;
|
|
494
|
+
const dependents = new Map;
|
|
495
|
+
for (const id of scenarioIds) {
|
|
496
|
+
inDegree.set(id, 0);
|
|
497
|
+
dependents.set(id, []);
|
|
498
|
+
}
|
|
499
|
+
for (const id of scenarioIds) {
|
|
500
|
+
const deps = db2.query("SELECT depends_on FROM scenario_dependencies WHERE scenario_id = ?").all(id);
|
|
501
|
+
for (const dep of deps) {
|
|
502
|
+
if (idSet.has(dep.depends_on)) {
|
|
503
|
+
inDegree.set(id, (inDegree.get(id) ?? 0) + 1);
|
|
504
|
+
dependents.get(dep.depends_on).push(id);
|
|
505
|
+
}
|
|
506
|
+
}
|
|
507
|
+
}
|
|
508
|
+
const queue = [];
|
|
509
|
+
for (const [id, deg] of inDegree) {
|
|
510
|
+
if (deg === 0)
|
|
511
|
+
queue.push(id);
|
|
512
|
+
}
|
|
513
|
+
const sorted = [];
|
|
514
|
+
while (queue.length > 0) {
|
|
515
|
+
const current = queue.shift();
|
|
516
|
+
sorted.push(current);
|
|
517
|
+
for (const dep of dependents.get(current) ?? []) {
|
|
518
|
+
const newDeg = (inDegree.get(dep) ?? 1) - 1;
|
|
519
|
+
inDegree.set(dep, newDeg);
|
|
520
|
+
if (newDeg === 0)
|
|
521
|
+
queue.push(dep);
|
|
522
|
+
}
|
|
523
|
+
}
|
|
524
|
+
if (sorted.length !== scenarioIds.length) {
|
|
525
|
+
throw new DependencyCycleError("multiple", "multiple");
|
|
526
|
+
}
|
|
527
|
+
return sorted;
|
|
528
|
+
}
|
|
529
|
+
function createFlow(input) {
|
|
530
|
+
const db2 = getDatabase();
|
|
531
|
+
const id = uuid();
|
|
532
|
+
const timestamp = now();
|
|
533
|
+
db2.query(`
|
|
534
|
+
INSERT INTO flows (id, project_id, name, description, scenario_ids, created_at, updated_at)
|
|
535
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
536
|
+
`).run(id, input.projectId ?? null, input.name, input.description ?? null, JSON.stringify(input.scenarioIds), timestamp, timestamp);
|
|
537
|
+
return getFlow(id);
|
|
538
|
+
}
|
|
539
|
+
function getFlow(id) {
|
|
540
|
+
const db2 = getDatabase();
|
|
541
|
+
let row = db2.query("SELECT * FROM flows WHERE id = ?").get(id);
|
|
542
|
+
if (row)
|
|
543
|
+
return flowFromRow(row);
|
|
544
|
+
const fullId = resolvePartialId("flows", id);
|
|
545
|
+
if (fullId) {
|
|
546
|
+
row = db2.query("SELECT * FROM flows WHERE id = ?").get(fullId);
|
|
547
|
+
if (row)
|
|
548
|
+
return flowFromRow(row);
|
|
549
|
+
}
|
|
353
550
|
return null;
|
|
354
551
|
}
|
|
552
|
+
function listFlows(projectId) {
|
|
553
|
+
const db2 = getDatabase();
|
|
554
|
+
if (projectId) {
|
|
555
|
+
const rows2 = db2.query("SELECT * FROM flows WHERE project_id = ? ORDER BY created_at DESC").all(projectId);
|
|
556
|
+
return rows2.map(flowFromRow);
|
|
557
|
+
}
|
|
558
|
+
const rows = db2.query("SELECT * FROM flows ORDER BY created_at DESC").all();
|
|
559
|
+
return rows.map(flowFromRow);
|
|
560
|
+
}
|
|
561
|
+
function deleteFlow(id) {
|
|
562
|
+
const db2 = getDatabase();
|
|
563
|
+
const flow = getFlow(id);
|
|
564
|
+
if (!flow)
|
|
565
|
+
return false;
|
|
566
|
+
const result = db2.query("DELETE FROM flows WHERE id = ?").run(flow.id);
|
|
567
|
+
return result.changes > 0;
|
|
568
|
+
}
|
|
569
|
+
var init_flows = __esm(() => {
|
|
570
|
+
init_database();
|
|
571
|
+
init_database();
|
|
572
|
+
init_types();
|
|
573
|
+
});
|
|
574
|
+
|
|
575
|
+
// src/server/index.ts
|
|
576
|
+
import { existsSync as existsSync4 } from "fs";
|
|
577
|
+
import { join as join4 } from "path";
|
|
578
|
+
import { homedir as homedir4 } from "os";
|
|
355
579
|
|
|
356
580
|
// src/db/scenarios.ts
|
|
581
|
+
init_types();
|
|
582
|
+
init_database();
|
|
357
583
|
function nextShortId(projectId) {
|
|
358
584
|
const db2 = getDatabase();
|
|
359
585
|
if (projectId) {
|
|
@@ -372,9 +598,9 @@ function createScenario(input) {
|
|
|
372
598
|
const short_id = nextShortId(input.projectId);
|
|
373
599
|
const timestamp = now();
|
|
374
600
|
db2.query(`
|
|
375
|
-
INSERT INTO scenarios (id, short_id, project_id, name, description, steps, tags, priority, model, timeout_ms, target_path, requires_auth, auth_config, metadata, version, created_at, updated_at)
|
|
376
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1, ?, ?)
|
|
377
|
-
`).run(id, short_id, input.projectId ?? null, input.name, input.description, JSON.stringify(input.steps ?? []), JSON.stringify(input.tags ?? []), input.priority ?? "medium", input.model ?? null, input.timeoutMs ?? null, input.targetPath ?? null, input.requiresAuth ? 1 : 0, input.authConfig ? JSON.stringify(input.authConfig) : null, input.metadata ? JSON.stringify(input.metadata) : null, timestamp, timestamp);
|
|
601
|
+
INSERT INTO scenarios (id, short_id, project_id, name, description, steps, tags, priority, model, timeout_ms, target_path, requires_auth, auth_config, metadata, assertions, version, created_at, updated_at)
|
|
602
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1, ?, ?)
|
|
603
|
+
`).run(id, short_id, input.projectId ?? null, input.name, input.description, JSON.stringify(input.steps ?? []), JSON.stringify(input.tags ?? []), input.priority ?? "medium", input.model ?? null, input.timeoutMs ?? null, input.targetPath ?? null, input.requiresAuth ? 1 : 0, input.authConfig ? JSON.stringify(input.authConfig) : null, input.metadata ? JSON.stringify(input.metadata) : null, JSON.stringify(input.assertions ?? []), timestamp, timestamp);
|
|
378
604
|
return getScenario(id);
|
|
379
605
|
}
|
|
380
606
|
function getScenario(id) {
|
|
@@ -492,6 +718,10 @@ function updateScenario(id, input, version) {
|
|
|
492
718
|
sets.push("metadata = ?");
|
|
493
719
|
params.push(JSON.stringify(input.metadata));
|
|
494
720
|
}
|
|
721
|
+
if (input.assertions !== undefined) {
|
|
722
|
+
sets.push("assertions = ?");
|
|
723
|
+
params.push(JSON.stringify(input.assertions));
|
|
724
|
+
}
|
|
495
725
|
if (sets.length === 0) {
|
|
496
726
|
return existing;
|
|
497
727
|
}
|
|
@@ -517,6 +747,8 @@ function deleteScenario(id) {
|
|
|
517
747
|
}
|
|
518
748
|
|
|
519
749
|
// src/db/runs.ts
|
|
750
|
+
init_types();
|
|
751
|
+
init_database();
|
|
520
752
|
function createRun(input) {
|
|
521
753
|
const db2 = getDatabase();
|
|
522
754
|
const id = uuid();
|
|
@@ -620,6 +852,10 @@ function updateRun(id, updates) {
|
|
|
620
852
|
sets.push("metadata = ?");
|
|
621
853
|
params.push(updates.metadata);
|
|
622
854
|
}
|
|
855
|
+
if (updates.is_baseline !== undefined) {
|
|
856
|
+
sets.push("is_baseline = ?");
|
|
857
|
+
params.push(updates.is_baseline);
|
|
858
|
+
}
|
|
623
859
|
if (sets.length === 0) {
|
|
624
860
|
return existing;
|
|
625
861
|
}
|
|
@@ -629,6 +865,8 @@ function updateRun(id, updates) {
|
|
|
629
865
|
}
|
|
630
866
|
|
|
631
867
|
// src/db/results.ts
|
|
868
|
+
init_types();
|
|
869
|
+
init_database();
|
|
632
870
|
function createResult(input) {
|
|
633
871
|
const db2 = getDatabase();
|
|
634
872
|
const id = uuid();
|
|
@@ -705,6 +943,8 @@ function getResultsByRun(runId) {
|
|
|
705
943
|
}
|
|
706
944
|
|
|
707
945
|
// src/db/screenshots.ts
|
|
946
|
+
init_types();
|
|
947
|
+
init_database();
|
|
708
948
|
function createScreenshot(input) {
|
|
709
949
|
const db2 = getDatabase();
|
|
710
950
|
const id = uuid();
|
|
@@ -728,6 +968,7 @@ function listScreenshots(resultId) {
|
|
|
728
968
|
|
|
729
969
|
// src/lib/browser.ts
|
|
730
970
|
import { chromium } from "playwright";
|
|
971
|
+
init_types();
|
|
731
972
|
var DEFAULT_VIEWPORT = { width: 1280, height: 720 };
|
|
732
973
|
async function launchBrowser(options) {
|
|
733
974
|
const headless = options?.headless ?? true;
|
|
@@ -947,6 +1188,7 @@ class Screenshotter {
|
|
|
947
1188
|
}
|
|
948
1189
|
|
|
949
1190
|
// src/lib/ai-client.ts
|
|
1191
|
+
init_types();
|
|
950
1192
|
import Anthropic from "@anthropic-ai/sdk";
|
|
951
1193
|
function resolveModel(nameOrPreset) {
|
|
952
1194
|
if (nameOrPreset in MODEL_MAP) {
|
|
@@ -1638,6 +1880,7 @@ function createClient(apiKey) {
|
|
|
1638
1880
|
}
|
|
1639
1881
|
|
|
1640
1882
|
// src/lib/config.ts
|
|
1883
|
+
init_types();
|
|
1641
1884
|
import { homedir as homedir3 } from "os";
|
|
1642
1885
|
import { join as join3 } from "path";
|
|
1643
1886
|
import { readFileSync, existsSync as existsSync3 } from "fs";
|
|
@@ -1692,12 +1935,124 @@ function loadConfig() {
|
|
|
1692
1935
|
return config;
|
|
1693
1936
|
}
|
|
1694
1937
|
|
|
1938
|
+
// src/lib/webhooks.ts
|
|
1939
|
+
init_database();
|
|
1940
|
+
function fromRow(row) {
|
|
1941
|
+
return {
|
|
1942
|
+
id: row.id,
|
|
1943
|
+
url: row.url,
|
|
1944
|
+
events: JSON.parse(row.events),
|
|
1945
|
+
projectId: row.project_id,
|
|
1946
|
+
secret: row.secret,
|
|
1947
|
+
active: row.active === 1,
|
|
1948
|
+
createdAt: row.created_at
|
|
1949
|
+
};
|
|
1950
|
+
}
|
|
1951
|
+
function listWebhooks(projectId) {
|
|
1952
|
+
const db2 = getDatabase();
|
|
1953
|
+
let query = "SELECT * FROM webhooks WHERE active = 1";
|
|
1954
|
+
const params = [];
|
|
1955
|
+
if (projectId) {
|
|
1956
|
+
query += " AND (project_id = ? OR project_id IS NULL)";
|
|
1957
|
+
params.push(projectId);
|
|
1958
|
+
}
|
|
1959
|
+
query += " ORDER BY created_at DESC";
|
|
1960
|
+
const rows = db2.query(query).all(...params);
|
|
1961
|
+
return rows.map(fromRow);
|
|
1962
|
+
}
|
|
1963
|
+
function signPayload(body, secret) {
|
|
1964
|
+
const encoder = new TextEncoder;
|
|
1965
|
+
const key = encoder.encode(secret);
|
|
1966
|
+
const data = encoder.encode(body);
|
|
1967
|
+
let hash = 0;
|
|
1968
|
+
for (let i = 0;i < data.length; i++) {
|
|
1969
|
+
hash = (hash << 5) - hash + data[i] + (key[i % key.length] ?? 0) | 0;
|
|
1970
|
+
}
|
|
1971
|
+
return `sha256=${Math.abs(hash).toString(16).padStart(16, "0")}`;
|
|
1972
|
+
}
|
|
1973
|
+
function formatSlackPayload(payload) {
|
|
1974
|
+
const status = payload.run.status === "passed" ? ":white_check_mark:" : ":x:";
|
|
1975
|
+
const color = payload.run.status === "passed" ? "#22c55e" : "#ef4444";
|
|
1976
|
+
return {
|
|
1977
|
+
attachments: [
|
|
1978
|
+
{
|
|
1979
|
+
color,
|
|
1980
|
+
blocks: [
|
|
1981
|
+
{
|
|
1982
|
+
type: "section",
|
|
1983
|
+
text: {
|
|
1984
|
+
type: "mrkdwn",
|
|
1985
|
+
text: `${status} *Test Run ${payload.run.status.toUpperCase()}*
|
|
1986
|
+
` + `URL: ${payload.run.url}
|
|
1987
|
+
` + `Results: ${payload.run.passed}/${payload.run.total} passed` + (payload.run.failed > 0 ? ` (${payload.run.failed} failed)` : "") + (payload.schedule ? `
|
|
1988
|
+
Schedule: ${payload.schedule.name}` : "")
|
|
1989
|
+
}
|
|
1990
|
+
}
|
|
1991
|
+
]
|
|
1992
|
+
}
|
|
1993
|
+
]
|
|
1994
|
+
};
|
|
1995
|
+
}
|
|
1996
|
+
async function dispatchWebhooks(event, run, schedule) {
|
|
1997
|
+
const webhooks = listWebhooks(run.projectId ?? undefined);
|
|
1998
|
+
const payload = {
|
|
1999
|
+
event,
|
|
2000
|
+
run: {
|
|
2001
|
+
id: run.id,
|
|
2002
|
+
url: run.url,
|
|
2003
|
+
status: run.status,
|
|
2004
|
+
passed: run.passed,
|
|
2005
|
+
failed: run.failed,
|
|
2006
|
+
total: run.total
|
|
2007
|
+
},
|
|
2008
|
+
schedule,
|
|
2009
|
+
timestamp: new Date().toISOString()
|
|
2010
|
+
};
|
|
2011
|
+
for (const webhook of webhooks) {
|
|
2012
|
+
if (!webhook.events.includes(event) && !webhook.events.includes("*"))
|
|
2013
|
+
continue;
|
|
2014
|
+
const isSlack = webhook.url.includes("hooks.slack.com");
|
|
2015
|
+
const body = isSlack ? JSON.stringify(formatSlackPayload(payload)) : JSON.stringify(payload);
|
|
2016
|
+
const headers = {
|
|
2017
|
+
"Content-Type": "application/json"
|
|
2018
|
+
};
|
|
2019
|
+
if (webhook.secret) {
|
|
2020
|
+
headers["X-Testers-Signature"] = signPayload(body, webhook.secret);
|
|
2021
|
+
}
|
|
2022
|
+
try {
|
|
2023
|
+
const response = await fetch(webhook.url, {
|
|
2024
|
+
method: "POST",
|
|
2025
|
+
headers,
|
|
2026
|
+
body
|
|
2027
|
+
});
|
|
2028
|
+
if (!response.ok) {
|
|
2029
|
+
await new Promise((r) => setTimeout(r, 5000));
|
|
2030
|
+
await fetch(webhook.url, { method: "POST", headers, body });
|
|
2031
|
+
}
|
|
2032
|
+
} catch {}
|
|
2033
|
+
}
|
|
2034
|
+
}
|
|
2035
|
+
|
|
1695
2036
|
// src/lib/runner.ts
|
|
1696
2037
|
var eventHandler = null;
|
|
1697
2038
|
function emit(event) {
|
|
1698
2039
|
if (eventHandler)
|
|
1699
2040
|
eventHandler(event);
|
|
1700
2041
|
}
|
|
2042
|
+
function withTimeout(promise, ms, label) {
|
|
2043
|
+
return new Promise((resolve, reject) => {
|
|
2044
|
+
const timer = setTimeout(() => {
|
|
2045
|
+
reject(new Error(`Scenario timeout after ${ms}ms: ${label}`));
|
|
2046
|
+
}, ms);
|
|
2047
|
+
promise.then((val) => {
|
|
2048
|
+
clearTimeout(timer);
|
|
2049
|
+
resolve(val);
|
|
2050
|
+
}, (err) => {
|
|
2051
|
+
clearTimeout(timer);
|
|
2052
|
+
reject(err);
|
|
2053
|
+
});
|
|
2054
|
+
});
|
|
2055
|
+
}
|
|
1701
2056
|
async function runSingleScenario(scenario, runId, options) {
|
|
1702
2057
|
const config = loadConfig();
|
|
1703
2058
|
const model = resolveModel(options.model ?? scenario.model ?? config.defaultModel);
|
|
@@ -1720,8 +2075,9 @@ async function runSingleScenario(scenario, runId, options) {
|
|
|
1720
2075
|
viewport: config.browser.viewport
|
|
1721
2076
|
});
|
|
1722
2077
|
const targetUrl = scenario.targetPath ? `${options.url.replace(/\/$/, "")}${scenario.targetPath}` : options.url;
|
|
1723
|
-
|
|
1724
|
-
|
|
2078
|
+
const scenarioTimeout = scenario.timeoutMs ?? options.timeout ?? config.browser.timeout ?? 60000;
|
|
2079
|
+
await page.goto(targetUrl, { timeout: Math.min(scenarioTimeout, 30000) });
|
|
2080
|
+
const agentResult = await withTimeout(runAgentLoop({
|
|
1725
2081
|
client,
|
|
1726
2082
|
page,
|
|
1727
2083
|
scenario,
|
|
@@ -1742,7 +2098,7 @@ async function runSingleScenario(scenario, runId, options) {
|
|
|
1742
2098
|
stepNumber: stepEvent.stepNumber
|
|
1743
2099
|
});
|
|
1744
2100
|
}
|
|
1745
|
-
});
|
|
2101
|
+
}), scenarioTimeout, scenario.name);
|
|
1746
2102
|
for (const ss of agentResult.screenshots) {
|
|
1747
2103
|
createScreenshot({
|
|
1748
2104
|
resultId: result.id,
|
|
@@ -1794,24 +2150,70 @@ async function runBatch(scenarios, options) {
|
|
|
1794
2150
|
projectId: options.projectId
|
|
1795
2151
|
});
|
|
1796
2152
|
updateRun(run.id, { status: "running", total: scenarios.length });
|
|
2153
|
+
let sortedScenarios = scenarios;
|
|
2154
|
+
try {
|
|
2155
|
+
const { topologicalSort: topologicalSort2 } = await Promise.resolve().then(() => (init_flows(), exports_flows));
|
|
2156
|
+
const scenarioIds = scenarios.map((s) => s.id);
|
|
2157
|
+
const sortedIds = topologicalSort2(scenarioIds);
|
|
2158
|
+
const scenarioMap = new Map(scenarios.map((s) => [s.id, s]));
|
|
2159
|
+
sortedScenarios = sortedIds.map((id) => scenarioMap.get(id)).filter((s) => s !== undefined);
|
|
2160
|
+
for (const s of scenarios) {
|
|
2161
|
+
if (!sortedIds.includes(s.id))
|
|
2162
|
+
sortedScenarios.push(s);
|
|
2163
|
+
}
|
|
2164
|
+
} catch {}
|
|
1797
2165
|
const results = [];
|
|
2166
|
+
const failedScenarioIds = new Set;
|
|
2167
|
+
const canRun = async (scenario) => {
|
|
2168
|
+
try {
|
|
2169
|
+
const { getDependencies: getDependencies2 } = await Promise.resolve().then(() => (init_flows(), exports_flows));
|
|
2170
|
+
const deps = getDependencies2(scenario.id);
|
|
2171
|
+
for (const depId of deps) {
|
|
2172
|
+
if (failedScenarioIds.has(depId))
|
|
2173
|
+
return false;
|
|
2174
|
+
}
|
|
2175
|
+
} catch {}
|
|
2176
|
+
return true;
|
|
2177
|
+
};
|
|
1798
2178
|
if (parallel <= 1) {
|
|
1799
|
-
for (const scenario of
|
|
2179
|
+
for (const scenario of sortedScenarios) {
|
|
2180
|
+
if (!await canRun(scenario)) {
|
|
2181
|
+
const result2 = createResult({ runId: run.id, scenarioId: scenario.id, model, stepsTotal: 0 });
|
|
2182
|
+
const skipped = updateResult(result2.id, { status: "skipped", error: "Skipped: dependency failed" });
|
|
2183
|
+
results.push(skipped);
|
|
2184
|
+
failedScenarioIds.add(scenario.id);
|
|
2185
|
+
emit({ type: "scenario:error", scenarioId: scenario.id, scenarioName: scenario.name, error: "Dependency failed \u2014 skipped", runId: run.id });
|
|
2186
|
+
continue;
|
|
2187
|
+
}
|
|
1800
2188
|
const result = await runSingleScenario(scenario, run.id, options);
|
|
1801
2189
|
results.push(result);
|
|
2190
|
+
if (result.status === "failed" || result.status === "error") {
|
|
2191
|
+
failedScenarioIds.add(scenario.id);
|
|
2192
|
+
}
|
|
1802
2193
|
}
|
|
1803
2194
|
} else {
|
|
1804
|
-
const queue = [...
|
|
2195
|
+
const queue = [...sortedScenarios];
|
|
1805
2196
|
const running = [];
|
|
1806
2197
|
const processNext = async () => {
|
|
1807
2198
|
const scenario = queue.shift();
|
|
1808
2199
|
if (!scenario)
|
|
1809
2200
|
return;
|
|
2201
|
+
if (!await canRun(scenario)) {
|
|
2202
|
+
const result2 = createResult({ runId: run.id, scenarioId: scenario.id, model, stepsTotal: 0 });
|
|
2203
|
+
const skipped = updateResult(result2.id, { status: "skipped", error: "Skipped: dependency failed" });
|
|
2204
|
+
results.push(skipped);
|
|
2205
|
+
failedScenarioIds.add(scenario.id);
|
|
2206
|
+
await processNext();
|
|
2207
|
+
return;
|
|
2208
|
+
}
|
|
1810
2209
|
const result = await runSingleScenario(scenario, run.id, options);
|
|
1811
2210
|
results.push(result);
|
|
2211
|
+
if (result.status === "failed" || result.status === "error") {
|
|
2212
|
+
failedScenarioIds.add(scenario.id);
|
|
2213
|
+
}
|
|
1812
2214
|
await processNext();
|
|
1813
2215
|
};
|
|
1814
|
-
const workers = Math.min(parallel,
|
|
2216
|
+
const workers = Math.min(parallel, sortedScenarios.length);
|
|
1815
2217
|
for (let i = 0;i < workers; i++) {
|
|
1816
2218
|
running.push(processNext());
|
|
1817
2219
|
}
|
|
@@ -1828,6 +2230,8 @@ async function runBatch(scenarios, options) {
|
|
|
1828
2230
|
finished_at: new Date().toISOString()
|
|
1829
2231
|
});
|
|
1830
2232
|
emit({ type: "run:complete", runId: run.id });
|
|
2233
|
+
const eventType = finalRun.status === "failed" ? "failed" : "completed";
|
|
2234
|
+
dispatchWebhooks(eventType, finalRun).catch(() => {});
|
|
1831
2235
|
return { run: finalRun, results };
|
|
1832
2236
|
}
|
|
1833
2237
|
async function runByFilter(options) {
|
|
@@ -1861,7 +2265,14 @@ function estimateCost(model, tokens) {
|
|
|
1861
2265
|
return tokens / 1e6 * costPer1M * 100;
|
|
1862
2266
|
}
|
|
1863
2267
|
|
|
2268
|
+
// src/server/index.ts
|
|
2269
|
+
init_types();
|
|
2270
|
+
init_database();
|
|
2271
|
+
|
|
1864
2272
|
// src/db/schedules.ts
|
|
2273
|
+
init_database();
|
|
2274
|
+
init_types();
|
|
2275
|
+
init_database();
|
|
1865
2276
|
function createSchedule(input) {
|
|
1866
2277
|
const db2 = getDatabase();
|
|
1867
2278
|
const id = uuid();
|
|
@@ -1988,6 +2399,7 @@ function updateLastRun(id, runId, nextRunAt) {
|
|
|
1988
2399
|
}
|
|
1989
2400
|
|
|
1990
2401
|
// src/lib/scheduler.ts
|
|
2402
|
+
init_types();
|
|
1991
2403
|
function parseCronField(field, min, max) {
|
|
1992
2404
|
const results = new Set;
|
|
1993
2405
|
const parts = field.split(",");
|