@hasna/testers 0.0.34 → 0.0.35
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/index.js +1137 -1016
- package/dist/db/workflows.d.ts.map +1 -1
- package/dist/index.d.ts +4 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +944 -190
- package/dist/lib/ai-client.d.ts +2 -0
- package/dist/lib/ai-client.d.ts.map +1 -1
- package/dist/lib/assertions.d.ts +4 -1
- package/dist/lib/assertions.d.ts.map +1 -1
- package/dist/lib/browser-compat.d.ts +14 -0
- package/dist/lib/browser-compat.d.ts.map +1 -0
- package/dist/lib/repo-discovery.d.ts.map +1 -1
- package/dist/lib/repo-executor.d.ts.map +1 -1
- package/dist/lib/runner.d.ts +29 -1
- package/dist/lib/runner.d.ts.map +1 -1
- package/dist/lib/workflow-runner.d.ts +73 -5
- package/dist/lib/workflow-runner.d.ts.map +1 -1
- package/dist/mcp/http.d.ts +1 -1
- package/dist/mcp/index.js +945 -819
- package/dist/mcp/server.d.ts.map +1 -1
- package/dist/sdk/index.d.ts +3 -3
- package/dist/sdk/index.d.ts.map +1 -1
- package/dist/server/index.js +897 -585
- package/dist/types/index.d.ts +23 -3
- package/dist/types/index.d.ts.map +1 -1
- package/package.json +7 -6
package/dist/index.js
CHANGED
|
@@ -17,6 +17,72 @@ var __esm = (fn, res) => () => (fn && (res = fn(fn = 0)), res);
|
|
|
17
17
|
var __require = import.meta.require;
|
|
18
18
|
|
|
19
19
|
// src/types/index.ts
|
|
20
|
+
function isRecord(value) {
|
|
21
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
22
|
+
}
|
|
23
|
+
function stringValue(value) {
|
|
24
|
+
return typeof value === "string" && value.trim() ? value : undefined;
|
|
25
|
+
}
|
|
26
|
+
function numberValue(value) {
|
|
27
|
+
return typeof value === "number" && Number.isFinite(value) ? value : undefined;
|
|
28
|
+
}
|
|
29
|
+
function stringMap(value) {
|
|
30
|
+
if (!isRecord(value))
|
|
31
|
+
return;
|
|
32
|
+
const entries = Object.entries(value).filter((entry) => typeof entry[1] === "string");
|
|
33
|
+
return entries.length > 0 ? Object.fromEntries(entries) : undefined;
|
|
34
|
+
}
|
|
35
|
+
function cleanupValue(value) {
|
|
36
|
+
if (value === "delete" || value === "stop" || value === "keep")
|
|
37
|
+
return value;
|
|
38
|
+
return;
|
|
39
|
+
}
|
|
40
|
+
function workflowExecutionFromValue(value) {
|
|
41
|
+
const input = isRecord(value) ? value : {};
|
|
42
|
+
const rawTarget = stringValue(input["target"]) ?? "local";
|
|
43
|
+
if (rawTarget === "local") {
|
|
44
|
+
const timeoutMs2 = numberValue(input["timeoutMs"]);
|
|
45
|
+
return timeoutMs2 === undefined ? { target: "local" } : { target: "local", timeoutMs: timeoutMs2 };
|
|
46
|
+
}
|
|
47
|
+
if (rawTarget !== "sandbox" && rawTarget !== "connector:e2b") {
|
|
48
|
+
throw new Error(`Unsupported workflow execution target: ${rawTarget}`);
|
|
49
|
+
}
|
|
50
|
+
const provider = rawTarget === "connector:e2b" ? "e2b" : stringValue(input["provider"]) ?? stringValue(input["connector"]);
|
|
51
|
+
const sandboxImage = stringValue(input["sandboxImage"]) ?? stringValue(input["sandboxTemplate"]);
|
|
52
|
+
const sandboxRemoteDir = stringValue(input["sandboxRemoteDir"]);
|
|
53
|
+
const sandboxCleanup = cleanupValue(input["sandboxCleanup"]);
|
|
54
|
+
const setupCommand = stringValue(input["setupCommand"]);
|
|
55
|
+
const packageSpec = stringValue(input["packageSpec"]);
|
|
56
|
+
const timeoutMs = numberValue(input["timeoutMs"]);
|
|
57
|
+
const env = stringMap(input["env"]);
|
|
58
|
+
return {
|
|
59
|
+
target: "sandbox",
|
|
60
|
+
...provider ? { provider } : {},
|
|
61
|
+
...sandboxImage ? { sandboxImage } : {},
|
|
62
|
+
...sandboxRemoteDir ? { sandboxRemoteDir } : {},
|
|
63
|
+
...sandboxCleanup ? { sandboxCleanup } : {},
|
|
64
|
+
...setupCommand ? { setupCommand } : {},
|
|
65
|
+
...packageSpec ? { packageSpec } : {},
|
|
66
|
+
...timeoutMs !== undefined ? { timeoutMs } : {},
|
|
67
|
+
...env ? { env } : {}
|
|
68
|
+
};
|
|
69
|
+
}
|
|
70
|
+
function workflowFromRow(row) {
|
|
71
|
+
return {
|
|
72
|
+
id: row.id,
|
|
73
|
+
projectId: row.project_id,
|
|
74
|
+
name: row.name,
|
|
75
|
+
description: row.description,
|
|
76
|
+
scenarioFilter: JSON.parse(row.scenario_filter || "{}"),
|
|
77
|
+
personaIds: JSON.parse(row.persona_ids || "[]"),
|
|
78
|
+
goal: row.goal ? JSON.parse(row.goal) : null,
|
|
79
|
+
execution: workflowExecutionFromValue(JSON.parse(row.execution || '{"target":"local"}')),
|
|
80
|
+
settings: JSON.parse(row.settings || "{}"),
|
|
81
|
+
enabled: row.enabled === 1,
|
|
82
|
+
createdAt: row.created_at,
|
|
83
|
+
updatedAt: row.updated_at
|
|
84
|
+
};
|
|
85
|
+
}
|
|
20
86
|
function projectFromRow(row) {
|
|
21
87
|
return {
|
|
22
88
|
id: row.id,
|
|
@@ -10456,7 +10522,7 @@ __export(exports_flows, {
|
|
|
10456
10522
|
addDependency: () => addDependency
|
|
10457
10523
|
});
|
|
10458
10524
|
function addDependency(scenarioId, dependsOn) {
|
|
10459
|
-
const
|
|
10525
|
+
const db3 = getDatabase();
|
|
10460
10526
|
const visited = new Set;
|
|
10461
10527
|
const queue = [dependsOn];
|
|
10462
10528
|
while (queue.length > 0) {
|
|
@@ -10467,37 +10533,37 @@ function addDependency(scenarioId, dependsOn) {
|
|
|
10467
10533
|
if (visited.has(current))
|
|
10468
10534
|
continue;
|
|
10469
10535
|
visited.add(current);
|
|
10470
|
-
const deps =
|
|
10536
|
+
const deps = db3.query("SELECT depends_on FROM scenario_dependencies WHERE scenario_id = ?").all(current);
|
|
10471
10537
|
for (const dep of deps) {
|
|
10472
10538
|
if (!visited.has(dep.depends_on)) {
|
|
10473
10539
|
queue.push(dep.depends_on);
|
|
10474
10540
|
}
|
|
10475
10541
|
}
|
|
10476
10542
|
}
|
|
10477
|
-
|
|
10543
|
+
db3.query("INSERT OR IGNORE INTO scenario_dependencies (scenario_id, depends_on) VALUES (?, ?)").run(scenarioId, dependsOn);
|
|
10478
10544
|
}
|
|
10479
10545
|
function removeDependency(scenarioId, dependsOn) {
|
|
10480
|
-
const
|
|
10481
|
-
const result =
|
|
10546
|
+
const db3 = getDatabase();
|
|
10547
|
+
const result = db3.query("DELETE FROM scenario_dependencies WHERE scenario_id = ? AND depends_on = ?").run(scenarioId, dependsOn);
|
|
10482
10548
|
return result.changes > 0;
|
|
10483
10549
|
}
|
|
10484
10550
|
function getDependencies(scenarioId) {
|
|
10485
|
-
const
|
|
10486
|
-
const rows =
|
|
10551
|
+
const db3 = getDatabase();
|
|
10552
|
+
const rows = db3.query("SELECT depends_on FROM scenario_dependencies WHERE scenario_id = ?").all(scenarioId);
|
|
10487
10553
|
return rows.map((r) => r.depends_on);
|
|
10488
10554
|
}
|
|
10489
10555
|
function getDependents(scenarioId) {
|
|
10490
|
-
const
|
|
10491
|
-
const rows =
|
|
10556
|
+
const db3 = getDatabase();
|
|
10557
|
+
const rows = db3.query("SELECT scenario_id FROM scenario_dependencies WHERE depends_on = ?").all(scenarioId);
|
|
10492
10558
|
return rows.map((r) => r.scenario_id);
|
|
10493
10559
|
}
|
|
10494
10560
|
function getTransitiveDependencies(scenarioId) {
|
|
10495
|
-
const
|
|
10561
|
+
const db3 = getDatabase();
|
|
10496
10562
|
const visited = new Set;
|
|
10497
10563
|
const queue = [scenarioId];
|
|
10498
10564
|
while (queue.length > 0) {
|
|
10499
10565
|
const current = queue.shift();
|
|
10500
|
-
const deps =
|
|
10566
|
+
const deps = db3.query("SELECT depends_on FROM scenario_dependencies WHERE scenario_id = ?").all(current);
|
|
10501
10567
|
for (const dep of deps) {
|
|
10502
10568
|
if (!visited.has(dep.depends_on)) {
|
|
10503
10569
|
visited.add(dep.depends_on);
|
|
@@ -10508,7 +10574,7 @@ function getTransitiveDependencies(scenarioId) {
|
|
|
10508
10574
|
return Array.from(visited);
|
|
10509
10575
|
}
|
|
10510
10576
|
function topologicalSort(scenarioIds) {
|
|
10511
|
-
const
|
|
10577
|
+
const db3 = getDatabase();
|
|
10512
10578
|
const idSet = new Set(scenarioIds);
|
|
10513
10579
|
const inDegree = new Map;
|
|
10514
10580
|
const dependents = new Map;
|
|
@@ -10517,7 +10583,7 @@ function topologicalSort(scenarioIds) {
|
|
|
10517
10583
|
dependents.set(id, []);
|
|
10518
10584
|
}
|
|
10519
10585
|
for (const id of scenarioIds) {
|
|
10520
|
-
const deps =
|
|
10586
|
+
const deps = db3.query("SELECT depends_on FROM scenario_dependencies WHERE scenario_id = ?").all(id);
|
|
10521
10587
|
for (const dep of deps) {
|
|
10522
10588
|
if (idSet.has(dep.depends_on)) {
|
|
10523
10589
|
inDegree.set(id, (inDegree.get(id) ?? 0) + 1);
|
|
@@ -10547,43 +10613,43 @@ function topologicalSort(scenarioIds) {
|
|
|
10547
10613
|
return sorted;
|
|
10548
10614
|
}
|
|
10549
10615
|
function createFlow(input) {
|
|
10550
|
-
const
|
|
10616
|
+
const db3 = getDatabase();
|
|
10551
10617
|
const id = uuid();
|
|
10552
10618
|
const timestamp = now();
|
|
10553
|
-
|
|
10619
|
+
db3.query(`
|
|
10554
10620
|
INSERT INTO flows (id, project_id, name, description, scenario_ids, created_at, updated_at)
|
|
10555
10621
|
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
10556
10622
|
`).run(id, input.projectId ?? null, input.name, input.description ?? null, JSON.stringify(input.scenarioIds), timestamp, timestamp);
|
|
10557
10623
|
return getFlow(id);
|
|
10558
10624
|
}
|
|
10559
10625
|
function getFlow(id) {
|
|
10560
|
-
const
|
|
10561
|
-
let row =
|
|
10626
|
+
const db3 = getDatabase();
|
|
10627
|
+
let row = db3.query("SELECT * FROM flows WHERE id = ?").get(id);
|
|
10562
10628
|
if (row)
|
|
10563
10629
|
return flowFromRow(row);
|
|
10564
10630
|
const fullId = resolvePartialId("flows", id);
|
|
10565
10631
|
if (fullId) {
|
|
10566
|
-
row =
|
|
10632
|
+
row = db3.query("SELECT * FROM flows WHERE id = ?").get(fullId);
|
|
10567
10633
|
if (row)
|
|
10568
10634
|
return flowFromRow(row);
|
|
10569
10635
|
}
|
|
10570
10636
|
return null;
|
|
10571
10637
|
}
|
|
10572
10638
|
function listFlows(projectId) {
|
|
10573
|
-
const
|
|
10639
|
+
const db3 = getDatabase();
|
|
10574
10640
|
if (projectId) {
|
|
10575
|
-
const rows2 =
|
|
10641
|
+
const rows2 = db3.query("SELECT * FROM flows WHERE project_id = ? ORDER BY created_at DESC").all(projectId);
|
|
10576
10642
|
return rows2.map(flowFromRow);
|
|
10577
10643
|
}
|
|
10578
|
-
const rows =
|
|
10644
|
+
const rows = db3.query("SELECT * FROM flows ORDER BY created_at DESC").all();
|
|
10579
10645
|
return rows.map(flowFromRow);
|
|
10580
10646
|
}
|
|
10581
10647
|
function deleteFlow(id) {
|
|
10582
|
-
const
|
|
10648
|
+
const db3 = getDatabase();
|
|
10583
10649
|
const flow = getFlow(id);
|
|
10584
10650
|
if (!flow)
|
|
10585
10651
|
return false;
|
|
10586
|
-
const result =
|
|
10652
|
+
const result = db3.query("DELETE FROM flows WHERE id = ?").run(flow.id);
|
|
10587
10653
|
return result.changes > 0;
|
|
10588
10654
|
}
|
|
10589
10655
|
var init_flows = __esm(() => {
|
|
@@ -12092,7 +12158,6 @@ async function executeTool(page, screenshotter, toolName, toolInput, context) {
|
|
|
12092
12158
|
const assertionType = toolInput.assertion_type;
|
|
12093
12159
|
const selector = toolInput.selector;
|
|
12094
12160
|
const expected = toolInput.expected;
|
|
12095
|
-
const sessionId = context.sessionId ?? "default";
|
|
12096
12161
|
switch (assertionType) {
|
|
12097
12162
|
case "element_exists": {
|
|
12098
12163
|
if (!selector)
|
|
@@ -12157,7 +12222,6 @@ async function executeTool(page, screenshotter, toolName, toolInput, context) {
|
|
|
12157
12222
|
case "browser_intercept": {
|
|
12158
12223
|
const action = toolInput.action;
|
|
12159
12224
|
const pattern = toolInput.pattern;
|
|
12160
|
-
const interceptAction = toolInput.intercept_action;
|
|
12161
12225
|
const statusCode = toolInput.status_code;
|
|
12162
12226
|
const body = toolInput.body;
|
|
12163
12227
|
const sessionId = context.sessionId ?? "default";
|
|
@@ -12234,7 +12298,28 @@ ${JSON.stringify(har, null, 2)}` };
|
|
|
12234
12298
|
}
|
|
12235
12299
|
case "browser_a11y": {
|
|
12236
12300
|
const level = toolInput.level ?? "AA";
|
|
12237
|
-
const snapshot = await page.
|
|
12301
|
+
const snapshot = await page.evaluate(() => {
|
|
12302
|
+
function readRole(el) {
|
|
12303
|
+
return el.getAttribute("role") ?? el.tagName.toLowerCase();
|
|
12304
|
+
}
|
|
12305
|
+
function readName(el) {
|
|
12306
|
+
const labelledBy = el.getAttribute("aria-labelledby");
|
|
12307
|
+
if (labelledBy) {
|
|
12308
|
+
const labelledText = labelledBy.split(/\s+/).map((id) => document.getElementById(id)?.textContent?.trim()).filter(Boolean).join(" ");
|
|
12309
|
+
if (labelledText)
|
|
12310
|
+
return labelledText;
|
|
12311
|
+
}
|
|
12312
|
+
return el.getAttribute("aria-label") ?? el.getAttribute("alt") ?? el.textContent?.trim() ?? "";
|
|
12313
|
+
}
|
|
12314
|
+
function walk(el) {
|
|
12315
|
+
return {
|
|
12316
|
+
role: readRole(el),
|
|
12317
|
+
name: readName(el),
|
|
12318
|
+
children: Array.from(el.children).map((child) => walk(child))
|
|
12319
|
+
};
|
|
12320
|
+
}
|
|
12321
|
+
return document.body ? walk(document.body) : null;
|
|
12322
|
+
});
|
|
12238
12323
|
if (!snapshot)
|
|
12239
12324
|
return { result: "Error: could not capture accessibility tree" };
|
|
12240
12325
|
const issues = [];
|
|
@@ -12276,6 +12361,38 @@ ${filtered.join(`
|
|
|
12276
12361
|
return { result: `Error executing ${toolName}: ${message}` };
|
|
12277
12362
|
}
|
|
12278
12363
|
}
|
|
12364
|
+
function resolveStartUrl(baseUrl, targetPath) {
|
|
12365
|
+
try {
|
|
12366
|
+
return new URL(targetPath, baseUrl.endsWith("/") ? baseUrl : `${baseUrl}/`).toString();
|
|
12367
|
+
} catch {
|
|
12368
|
+
return `${baseUrl.replace(/\/+$/, "")}/${targetPath.replace(/^\/+/, "")}`;
|
|
12369
|
+
}
|
|
12370
|
+
}
|
|
12371
|
+
function buildScenarioUserMessage(scenario, baseUrl) {
|
|
12372
|
+
const userParts = [
|
|
12373
|
+
`**Scenario:** ${scenario.name}`,
|
|
12374
|
+
`**Description:** ${scenario.description}`
|
|
12375
|
+
];
|
|
12376
|
+
if (baseUrl) {
|
|
12377
|
+
const normalizedBaseUrl = baseUrl.replace(/\/+$/, "");
|
|
12378
|
+
userParts.push(`**Base URL:** ${normalizedBaseUrl}`);
|
|
12379
|
+
if (scenario.targetPath) {
|
|
12380
|
+
userParts.push(`**Start URL:** ${resolveStartUrl(normalizedBaseUrl, scenario.targetPath)}`);
|
|
12381
|
+
}
|
|
12382
|
+
userParts.push("**Navigation Boundary:** Treat the Base URL as the application under test. Resolve relative paths and in-app navigation against this origin. Do not navigate to another host unless a step explicitly includes an absolute external URL.");
|
|
12383
|
+
}
|
|
12384
|
+
if (scenario.targetPath) {
|
|
12385
|
+
userParts.push(`**Target Path:** ${scenario.targetPath}`);
|
|
12386
|
+
}
|
|
12387
|
+
if (scenario.steps.length > 0) {
|
|
12388
|
+
userParts.push("**Steps:**");
|
|
12389
|
+
for (let i = 0;i < scenario.steps.length; i++) {
|
|
12390
|
+
userParts.push(`${i + 1}. ${scenario.steps[i]}`);
|
|
12391
|
+
}
|
|
12392
|
+
}
|
|
12393
|
+
return userParts.join(`
|
|
12394
|
+
`);
|
|
12395
|
+
}
|
|
12279
12396
|
async function runAgentLoop(options) {
|
|
12280
12397
|
const {
|
|
12281
12398
|
client,
|
|
@@ -12285,6 +12402,7 @@ async function runAgentLoop(options) {
|
|
|
12285
12402
|
model,
|
|
12286
12403
|
runId,
|
|
12287
12404
|
sessionId,
|
|
12405
|
+
baseUrl,
|
|
12288
12406
|
maxTurns = 30,
|
|
12289
12407
|
onStep,
|
|
12290
12408
|
persona,
|
|
@@ -12332,21 +12450,7 @@ Instructions: ${persona.instructions}` : "",
|
|
|
12332
12450
|
"- Verify both positive and negative states"
|
|
12333
12451
|
].join(`
|
|
12334
12452
|
`) + personaSection;
|
|
12335
|
-
const
|
|
12336
|
-
`**Scenario:** ${scenario.name}`,
|
|
12337
|
-
`**Description:** ${scenario.description}`
|
|
12338
|
-
];
|
|
12339
|
-
if (scenario.targetPath) {
|
|
12340
|
-
userParts.push(`**Target Path:** ${scenario.targetPath}`);
|
|
12341
|
-
}
|
|
12342
|
-
if (scenario.steps.length > 0) {
|
|
12343
|
-
userParts.push("**Steps:**");
|
|
12344
|
-
for (let i = 0;i < scenario.steps.length; i++) {
|
|
12345
|
-
userParts.push(`${i + 1}. ${scenario.steps[i]}`);
|
|
12346
|
-
}
|
|
12347
|
-
}
|
|
12348
|
-
const userMessage = userParts.join(`
|
|
12349
|
-
`);
|
|
12453
|
+
const userMessage = buildScenarioUserMessage(scenario, baseUrl);
|
|
12350
12454
|
const screenshots = [];
|
|
12351
12455
|
let tokensUsed = 0;
|
|
12352
12456
|
let stepNumber = 0;
|
|
@@ -12409,7 +12513,7 @@ Instructions: ${persona.instructions}` : "",
|
|
|
12409
12513
|
if (onStep) {
|
|
12410
12514
|
onStep({ type: "tool_call", toolName: toolBlock.name, toolInput, stepNumber });
|
|
12411
12515
|
}
|
|
12412
|
-
const execResult = await executeTool(page, screenshotter, toolBlock.name, toolInput, { runId, scenarioSlug, stepNumber, sessionId, a11y });
|
|
12516
|
+
const execResult = await executeTool(page, screenshotter, toolBlock.name, toolInput, { runId, scenarioSlug, stepNumber, sessionId: sessionId ?? runId, a11y });
|
|
12413
12517
|
if (onStep) {
|
|
12414
12518
|
onStep({ type: "tool_result", toolName: toolBlock.name, toolResult: execResult.result, stepNumber });
|
|
12415
12519
|
}
|
|
@@ -13608,6 +13712,127 @@ function updateLastRun(id, runId, nextRunAt) {
|
|
|
13608
13712
|
UPDATE schedules SET last_run_id = ?, last_run_at = ?, next_run_at = ?, updated_at = ? WHERE id = ?
|
|
13609
13713
|
`).run(runId, timestamp, nextRunAt, timestamp, id);
|
|
13610
13714
|
}
|
|
13715
|
+
// src/db/workflows.ts
|
|
13716
|
+
init_types();
|
|
13717
|
+
init_database();
|
|
13718
|
+
var DEFAULT_EXECUTION = { target: "local" };
|
|
13719
|
+
function normalizeGoal(input) {
|
|
13720
|
+
if (!input)
|
|
13721
|
+
return null;
|
|
13722
|
+
const prompt = input.prompt?.trim();
|
|
13723
|
+
if (!prompt)
|
|
13724
|
+
return null;
|
|
13725
|
+
return {
|
|
13726
|
+
prompt,
|
|
13727
|
+
successCriteria: input.successCriteria ?? [],
|
|
13728
|
+
maxIterations: input.maxIterations ?? 10
|
|
13729
|
+
};
|
|
13730
|
+
}
|
|
13731
|
+
function normalizeFilter(input) {
|
|
13732
|
+
return {
|
|
13733
|
+
scenarioIds: input?.scenarioIds?.filter(Boolean),
|
|
13734
|
+
tags: input?.tags?.filter(Boolean),
|
|
13735
|
+
priority: input?.priority
|
|
13736
|
+
};
|
|
13737
|
+
}
|
|
13738
|
+
function normalizeExecution(input) {
|
|
13739
|
+
return input ? workflowExecutionFromValue(input) : DEFAULT_EXECUTION;
|
|
13740
|
+
}
|
|
13741
|
+
function createTestingWorkflow(input) {
|
|
13742
|
+
const db2 = getDatabase();
|
|
13743
|
+
const id = uuid();
|
|
13744
|
+
const timestamp = now();
|
|
13745
|
+
db2.query(`
|
|
13746
|
+
INSERT INTO testing_workflows
|
|
13747
|
+
(id, project_id, name, description, scenario_filter, persona_ids, goal, execution, settings, enabled, created_at, updated_at)
|
|
13748
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
13749
|
+
`).run(id, input.projectId ?? null, input.name, input.description ?? null, JSON.stringify(normalizeFilter(input.scenarioFilter)), JSON.stringify(input.personaIds ?? []), JSON.stringify(normalizeGoal(input.goal)), JSON.stringify(normalizeExecution(input.execution)), JSON.stringify(input.settings ?? {}), input.enabled === false ? 0 : 1, timestamp, timestamp);
|
|
13750
|
+
return getTestingWorkflow(id);
|
|
13751
|
+
}
|
|
13752
|
+
function getTestingWorkflow(id) {
|
|
13753
|
+
const db2 = getDatabase();
|
|
13754
|
+
let row = db2.query("SELECT * FROM testing_workflows WHERE id = ?").get(id);
|
|
13755
|
+
if (row)
|
|
13756
|
+
return workflowFromRow(row);
|
|
13757
|
+
const fullId = resolvePartialId("testing_workflows", id);
|
|
13758
|
+
if (fullId) {
|
|
13759
|
+
row = db2.query("SELECT * FROM testing_workflows WHERE id = ?").get(fullId);
|
|
13760
|
+
if (row)
|
|
13761
|
+
return workflowFromRow(row);
|
|
13762
|
+
}
|
|
13763
|
+
row = db2.query("SELECT * FROM testing_workflows WHERE name = ?").get(id);
|
|
13764
|
+
return row ? workflowFromRow(row) : null;
|
|
13765
|
+
}
|
|
13766
|
+
function listTestingWorkflows(filter) {
|
|
13767
|
+
const db2 = getDatabase();
|
|
13768
|
+
const conditions = [];
|
|
13769
|
+
const params = [];
|
|
13770
|
+
if (filter?.projectId) {
|
|
13771
|
+
conditions.push("project_id = ?");
|
|
13772
|
+
params.push(filter.projectId);
|
|
13773
|
+
}
|
|
13774
|
+
if (filter?.enabled !== undefined) {
|
|
13775
|
+
conditions.push("enabled = ?");
|
|
13776
|
+
params.push(filter.enabled ? 1 : 0);
|
|
13777
|
+
}
|
|
13778
|
+
const where = conditions.length > 0 ? ` WHERE ${conditions.join(" AND ")}` : "";
|
|
13779
|
+
const rows = db2.query(`SELECT * FROM testing_workflows${where} ORDER BY created_at DESC`).all(...params);
|
|
13780
|
+
return rows.map(workflowFromRow);
|
|
13781
|
+
}
|
|
13782
|
+
function updateTestingWorkflow(id, input) {
|
|
13783
|
+
const existing = getTestingWorkflow(id);
|
|
13784
|
+
if (!existing)
|
|
13785
|
+
throw new Error(`Testing workflow not found: ${id}`);
|
|
13786
|
+
const fields = [];
|
|
13787
|
+
const values = [];
|
|
13788
|
+
if (input.name !== undefined) {
|
|
13789
|
+
fields.push("name = ?");
|
|
13790
|
+
values.push(input.name);
|
|
13791
|
+
}
|
|
13792
|
+
if (input.description !== undefined) {
|
|
13793
|
+
fields.push("description = ?");
|
|
13794
|
+
values.push(input.description);
|
|
13795
|
+
}
|
|
13796
|
+
if (input.scenarioFilter !== undefined) {
|
|
13797
|
+
fields.push("scenario_filter = ?");
|
|
13798
|
+
values.push(JSON.stringify(normalizeFilter(input.scenarioFilter)));
|
|
13799
|
+
}
|
|
13800
|
+
if (input.personaIds !== undefined) {
|
|
13801
|
+
fields.push("persona_ids = ?");
|
|
13802
|
+
values.push(JSON.stringify(input.personaIds));
|
|
13803
|
+
}
|
|
13804
|
+
if (input.goal !== undefined) {
|
|
13805
|
+
fields.push("goal = ?");
|
|
13806
|
+
values.push(JSON.stringify(normalizeGoal(input.goal)));
|
|
13807
|
+
}
|
|
13808
|
+
if (input.execution !== undefined) {
|
|
13809
|
+
fields.push("execution = ?");
|
|
13810
|
+
values.push(JSON.stringify(normalizeExecution(input.execution)));
|
|
13811
|
+
}
|
|
13812
|
+
if (input.settings !== undefined) {
|
|
13813
|
+
fields.push("settings = ?");
|
|
13814
|
+
values.push(JSON.stringify(input.settings));
|
|
13815
|
+
}
|
|
13816
|
+
if (input.enabled !== undefined) {
|
|
13817
|
+
fields.push("enabled = ?");
|
|
13818
|
+
values.push(input.enabled ? 1 : 0);
|
|
13819
|
+
}
|
|
13820
|
+
if (fields.length === 0)
|
|
13821
|
+
return existing;
|
|
13822
|
+
fields.push("updated_at = ?");
|
|
13823
|
+
values.push(now(), existing.id);
|
|
13824
|
+
db2().query(`UPDATE testing_workflows SET ${fields.join(", ")} WHERE id = ?`).run(...values);
|
|
13825
|
+
return getTestingWorkflow(existing.id);
|
|
13826
|
+
}
|
|
13827
|
+
function deleteTestingWorkflow(id) {
|
|
13828
|
+
const existing = getTestingWorkflow(id);
|
|
13829
|
+
if (!existing)
|
|
13830
|
+
return false;
|
|
13831
|
+
return getDatabase().query("DELETE FROM testing_workflows WHERE id = ?").run(existing.id).changes > 0;
|
|
13832
|
+
}
|
|
13833
|
+
function db2() {
|
|
13834
|
+
return getDatabase();
|
|
13835
|
+
}
|
|
13611
13836
|
|
|
13612
13837
|
// src/index.ts
|
|
13613
13838
|
init_flows();
|
|
@@ -14905,20 +15130,20 @@ function loadBudgetConfig() {
|
|
|
14905
15130
|
};
|
|
14906
15131
|
}
|
|
14907
15132
|
function getCostSummary(options) {
|
|
14908
|
-
const
|
|
15133
|
+
const db3 = getDatabase();
|
|
14909
15134
|
const period = options?.period ?? "month";
|
|
14910
15135
|
const projectId = options?.projectId;
|
|
14911
15136
|
const dateFilter = getDateFilter(period);
|
|
14912
15137
|
const projectFilter = projectId ? "AND ru.project_id = ?" : "";
|
|
14913
15138
|
const projectParams = projectId ? [projectId] : [];
|
|
14914
|
-
const totalsRow =
|
|
15139
|
+
const totalsRow = db3.query(`SELECT
|
|
14915
15140
|
COALESCE(SUM(r.cost_cents), 0) as total_cost,
|
|
14916
15141
|
COALESCE(SUM(r.tokens_used), 0) as total_tokens,
|
|
14917
15142
|
COUNT(DISTINCT r.run_id) as run_count
|
|
14918
15143
|
FROM results r
|
|
14919
15144
|
JOIN runs ru ON r.run_id = ru.id
|
|
14920
15145
|
WHERE 1=1 ${dateFilter} ${projectFilter}`).get(...projectParams);
|
|
14921
|
-
const modelRows =
|
|
15146
|
+
const modelRows = db3.query(`SELECT
|
|
14922
15147
|
r.model,
|
|
14923
15148
|
COALESCE(SUM(r.cost_cents), 0) as cost_cents,
|
|
14924
15149
|
COALESCE(SUM(r.tokens_used), 0) as tokens,
|
|
@@ -14936,7 +15161,7 @@ function getCostSummary(options) {
|
|
|
14936
15161
|
runs: row.runs
|
|
14937
15162
|
};
|
|
14938
15163
|
}
|
|
14939
|
-
const scenarioRows =
|
|
15164
|
+
const scenarioRows = db3.query(`SELECT
|
|
14940
15165
|
r.scenario_id,
|
|
14941
15166
|
COALESCE(s.name, r.scenario_id) as name,
|
|
14942
15167
|
COALESCE(SUM(r.cost_cents), 0) as cost_cents,
|
|
@@ -15088,22 +15313,22 @@ function formatCostsJSON(summary) {
|
|
|
15088
15313
|
// src/db/step-results.ts
|
|
15089
15314
|
init_database();
|
|
15090
15315
|
function createStepResult(input) {
|
|
15091
|
-
const
|
|
15316
|
+
const db3 = getDatabase();
|
|
15092
15317
|
const id = uuid();
|
|
15093
15318
|
const timestamp = now();
|
|
15094
|
-
|
|
15319
|
+
db3.query(`
|
|
15095
15320
|
INSERT INTO step_results (id, result_id, step_number, action, status, tool_name, tool_input, thinking, created_at)
|
|
15096
15321
|
VALUES (?, ?, ?, ?, 'running', ?, ?, ?, ?)
|
|
15097
15322
|
`).run(id, input.resultId, input.stepNumber, input.action, input.toolName ?? null, input.toolInput ? JSON.stringify(input.toolInput) : null, input.thinking ?? null, timestamp);
|
|
15098
15323
|
return getStepResult(id);
|
|
15099
15324
|
}
|
|
15100
15325
|
function getStepResult(id) {
|
|
15101
|
-
const
|
|
15102
|
-
const row =
|
|
15326
|
+
const db3 = getDatabase();
|
|
15327
|
+
const row = db3.query("SELECT * FROM step_results WHERE id = ?").get(id);
|
|
15103
15328
|
return row ? stepResultFromRow(row) : null;
|
|
15104
15329
|
}
|
|
15105
15330
|
function updateStepResult(id, updates) {
|
|
15106
|
-
const
|
|
15331
|
+
const db3 = getDatabase();
|
|
15107
15332
|
const existing = getStepResult(id);
|
|
15108
15333
|
if (!existing)
|
|
15109
15334
|
return null;
|
|
@@ -15132,7 +15357,7 @@ function updateStepResult(id, updates) {
|
|
|
15132
15357
|
if (sets.length === 0)
|
|
15133
15358
|
return existing;
|
|
15134
15359
|
params.push(id);
|
|
15135
|
-
|
|
15360
|
+
db3.query(`UPDATE step_results SET ${sets.join(", ")} WHERE id = ?`).run(...params);
|
|
15136
15361
|
return getStepResult(id);
|
|
15137
15362
|
}
|
|
15138
15363
|
function stepResultFromRow(row) {
|
|
@@ -15157,18 +15382,18 @@ function stepResultFromRow(row) {
|
|
|
15157
15382
|
init_types();
|
|
15158
15383
|
init_database();
|
|
15159
15384
|
function getPersona(id) {
|
|
15160
|
-
const
|
|
15161
|
-
let row =
|
|
15385
|
+
const db3 = getDatabase();
|
|
15386
|
+
let row = db3.query("SELECT * FROM personas WHERE id = ?").get(id);
|
|
15162
15387
|
if (row)
|
|
15163
15388
|
return personaFromRow(row);
|
|
15164
|
-
row =
|
|
15389
|
+
row = db3.query("SELECT * FROM personas WHERE short_id = ?").get(id);
|
|
15165
15390
|
if (row)
|
|
15166
15391
|
return personaFromRow(row);
|
|
15167
15392
|
return null;
|
|
15168
15393
|
}
|
|
15169
15394
|
function savePersonaAuthCookies(id, cookies) {
|
|
15170
|
-
const
|
|
15171
|
-
|
|
15395
|
+
const db3 = getDatabase();
|
|
15396
|
+
db3.query("UPDATE personas SET auth_cookies = ?, updated_at = ? WHERE id = ?").run(JSON.stringify(cookies), now(), id);
|
|
15172
15397
|
}
|
|
15173
15398
|
|
|
15174
15399
|
// src/lib/runner.ts
|
|
@@ -15192,9 +15417,9 @@ function lookupFromVault(key) {
|
|
|
15192
15417
|
if (!existsSync9(vaultPath))
|
|
15193
15418
|
return null;
|
|
15194
15419
|
try {
|
|
15195
|
-
const
|
|
15196
|
-
const row =
|
|
15197
|
-
|
|
15420
|
+
const db3 = new Database2(vaultPath, { readonly: true });
|
|
15421
|
+
const row = db3.query("SELECT value FROM secrets WHERE key = ?").get(key);
|
|
15422
|
+
db3.close();
|
|
15198
15423
|
return row?.value ?? null;
|
|
15199
15424
|
} catch {
|
|
15200
15425
|
return null;
|
|
@@ -15458,21 +15683,21 @@ function fromRow(row) {
|
|
|
15458
15683
|
};
|
|
15459
15684
|
}
|
|
15460
15685
|
function createWebhook(input) {
|
|
15461
|
-
const
|
|
15686
|
+
const db3 = getDatabase();
|
|
15462
15687
|
const id = uuid();
|
|
15463
15688
|
const events = input.events ?? ["failed"];
|
|
15464
15689
|
const secret = input.secret ?? crypto.randomUUID().replace(/-/g, "");
|
|
15465
|
-
|
|
15690
|
+
db3.query(`
|
|
15466
15691
|
INSERT INTO webhooks (id, url, events, project_id, secret, active, created_at)
|
|
15467
15692
|
VALUES (?, ?, ?, ?, ?, 1, ?)
|
|
15468
15693
|
`).run(id, input.url, JSON.stringify(events), input.projectId ?? null, secret, now());
|
|
15469
15694
|
return getWebhook(id);
|
|
15470
15695
|
}
|
|
15471
15696
|
function getWebhook(id) {
|
|
15472
|
-
const
|
|
15473
|
-
const row =
|
|
15697
|
+
const db3 = getDatabase();
|
|
15698
|
+
const row = db3.query("SELECT * FROM webhooks WHERE id = ?").get(id);
|
|
15474
15699
|
if (!row) {
|
|
15475
|
-
const rows =
|
|
15700
|
+
const rows = db3.query("SELECT * FROM webhooks WHERE id LIKE ? || '%'").all(id);
|
|
15476
15701
|
if (rows.length === 1)
|
|
15477
15702
|
return fromRow(rows[0]);
|
|
15478
15703
|
return null;
|
|
@@ -15480,7 +15705,7 @@ function getWebhook(id) {
|
|
|
15480
15705
|
return fromRow(row);
|
|
15481
15706
|
}
|
|
15482
15707
|
function listWebhooks(projectId) {
|
|
15483
|
-
const
|
|
15708
|
+
const db3 = getDatabase();
|
|
15484
15709
|
let query = "SELECT * FROM webhooks WHERE active = 1";
|
|
15485
15710
|
const params = [];
|
|
15486
15711
|
if (projectId) {
|
|
@@ -15488,15 +15713,15 @@ function listWebhooks(projectId) {
|
|
|
15488
15713
|
params.push(projectId);
|
|
15489
15714
|
}
|
|
15490
15715
|
query += " ORDER BY created_at DESC";
|
|
15491
|
-
const rows =
|
|
15716
|
+
const rows = db3.query(query).all(...params);
|
|
15492
15717
|
return rows.map(fromRow);
|
|
15493
15718
|
}
|
|
15494
15719
|
function deleteWebhook(id) {
|
|
15495
|
-
const
|
|
15720
|
+
const db3 = getDatabase();
|
|
15496
15721
|
const webhook = getWebhook(id);
|
|
15497
15722
|
if (!webhook)
|
|
15498
15723
|
return false;
|
|
15499
|
-
|
|
15724
|
+
db3.query("DELETE FROM webhooks WHERE id = ?").run(webhook.id);
|
|
15500
15725
|
return true;
|
|
15501
15726
|
}
|
|
15502
15727
|
function signPayload(body, secret) {
|
|
@@ -15664,12 +15889,12 @@ function connectToTodos(options = {}) {
|
|
|
15664
15889
|
if (!existsSync10(dbPath)) {
|
|
15665
15890
|
throw new TodosConnectionError(`Todos database not found at ${dbPath}. Install @hasna/todos or set TODOS_DB_PATH.`);
|
|
15666
15891
|
}
|
|
15667
|
-
const
|
|
15668
|
-
|
|
15669
|
-
return
|
|
15892
|
+
const db3 = new Database3(dbPath, { readonly: options.readonly ?? true });
|
|
15893
|
+
db3.exec("PRAGMA foreign_keys = ON");
|
|
15894
|
+
return db3;
|
|
15670
15895
|
}
|
|
15671
15896
|
function pullTasks(options = {}) {
|
|
15672
|
-
const
|
|
15897
|
+
const db3 = connectToTodos({ readonly: true });
|
|
15673
15898
|
try {
|
|
15674
15899
|
let query = "SELECT id, short_id, title, description, status, priority, tags, project_id FROM tasks WHERE 1=1";
|
|
15675
15900
|
const params = [];
|
|
@@ -15684,14 +15909,14 @@ function pullTasks(options = {}) {
|
|
|
15684
15909
|
params.push(options.priority);
|
|
15685
15910
|
}
|
|
15686
15911
|
if (options.projectName) {
|
|
15687
|
-
const project =
|
|
15912
|
+
const project = db3.query("SELECT id FROM projects WHERE name = ?").get(options.projectName);
|
|
15688
15913
|
if (project) {
|
|
15689
15914
|
query += " AND project_id = ?";
|
|
15690
15915
|
params.push(project.id);
|
|
15691
15916
|
}
|
|
15692
15917
|
}
|
|
15693
15918
|
query += " ORDER BY CASE priority WHEN 'critical' THEN 0 WHEN 'high' THEN 1 WHEN 'medium' THEN 2 WHEN 'low' THEN 3 END";
|
|
15694
|
-
const tasks =
|
|
15919
|
+
const tasks = db3.query(query).all(...params);
|
|
15695
15920
|
if (options.tags && options.tags.length > 0) {
|
|
15696
15921
|
return tasks.filter((task) => {
|
|
15697
15922
|
const taskTags = JSON.parse(task.tags || "[]");
|
|
@@ -15700,7 +15925,7 @@ function pullTasks(options = {}) {
|
|
|
15700
15925
|
}
|
|
15701
15926
|
return tasks;
|
|
15702
15927
|
} finally {
|
|
15703
|
-
|
|
15928
|
+
db3.close();
|
|
15704
15929
|
}
|
|
15705
15930
|
}
|
|
15706
15931
|
function taskToScenarioInput(task, projectId) {
|
|
@@ -15752,15 +15977,15 @@ function markTodoDone(taskId) {
|
|
|
15752
15977
|
const dbPath = resolveTodosDbPath();
|
|
15753
15978
|
if (!existsSync10(dbPath))
|
|
15754
15979
|
return false;
|
|
15755
|
-
const
|
|
15980
|
+
const db3 = new Database3(dbPath);
|
|
15756
15981
|
try {
|
|
15757
|
-
const task =
|
|
15982
|
+
const task = db3.query("SELECT id, version FROM tasks WHERE id LIKE ? || '%'").get(taskId);
|
|
15758
15983
|
if (!task)
|
|
15759
15984
|
return false;
|
|
15760
|
-
|
|
15985
|
+
db3.query("UPDATE tasks SET status = 'completed', completed_at = datetime('now'), version = version + 1, updated_at = datetime('now') WHERE id = ? AND version = ?").run(task.id, task.version);
|
|
15761
15986
|
return true;
|
|
15762
15987
|
} finally {
|
|
15763
|
-
|
|
15988
|
+
db3.close();
|
|
15764
15989
|
}
|
|
15765
15990
|
}
|
|
15766
15991
|
|
|
@@ -15771,9 +15996,9 @@ async function createFailureTasks(run, failedResults, scenarios) {
|
|
|
15771
15996
|
const projectId = process.env["TESTERS_TODOS_PROJECT_ID"];
|
|
15772
15997
|
if (!projectId)
|
|
15773
15998
|
return { created: 0, skipped: 0 };
|
|
15774
|
-
let
|
|
15999
|
+
let db3 = null;
|
|
15775
16000
|
try {
|
|
15776
|
-
|
|
16001
|
+
db3 = connectToTodos({ readonly: false });
|
|
15777
16002
|
} catch {
|
|
15778
16003
|
return { created: 0, skipped: 0 };
|
|
15779
16004
|
}
|
|
@@ -15784,7 +16009,7 @@ async function createFailureTasks(run, failedResults, scenarios) {
|
|
|
15784
16009
|
for (const result of failedResults) {
|
|
15785
16010
|
const scenario = scenarioMap.get(result.scenarioId);
|
|
15786
16011
|
const title = `BUG: [testers] ${scenario?.name ?? result.scenarioId} failed`;
|
|
15787
|
-
const existing =
|
|
16012
|
+
const existing = db3.query("SELECT id FROM tasks WHERE title = ? AND status NOT IN ('completed', 'cancelled') LIMIT 1").get(title);
|
|
15788
16013
|
if (existing) {
|
|
15789
16014
|
skipped++;
|
|
15790
16015
|
continue;
|
|
@@ -15805,7 +16030,7 @@ async function createFailureTasks(run, failedResults, scenarios) {
|
|
|
15805
16030
|
].filter(Boolean).join(`
|
|
15806
16031
|
`);
|
|
15807
16032
|
try {
|
|
15808
|
-
|
|
16033
|
+
db3.query(`
|
|
15809
16034
|
INSERT INTO tasks (id, short_id, title, description, status, priority, tags, project_id, version, created_at, updated_at)
|
|
15810
16035
|
VALUES (?, ?, ?, ?, 'pending', 'high', ?, ?, 1, ?, ?)
|
|
15811
16036
|
`).run(id, `BUG-${id.slice(0, 6)}`, title, description, JSON.stringify(["bug", "testers", "auto-created"]), projectId, now2, now2);
|
|
@@ -15815,7 +16040,7 @@ async function createFailureTasks(run, failedResults, scenarios) {
|
|
|
15815
16040
|
}
|
|
15816
16041
|
}
|
|
15817
16042
|
} finally {
|
|
15818
|
-
|
|
16043
|
+
db3.close();
|
|
15819
16044
|
}
|
|
15820
16045
|
return { created, skipped };
|
|
15821
16046
|
}
|
|
@@ -15894,6 +16119,291 @@ async function notifyRunToConversations(run, results, options) {
|
|
|
15894
16119
|
} catch {}
|
|
15895
16120
|
}
|
|
15896
16121
|
|
|
16122
|
+
// src/lib/a11y-audit.ts
|
|
16123
|
+
async function runA11yAudit(page, options = {}) {
|
|
16124
|
+
const { level = "AA", rules, exclude = [] } = options;
|
|
16125
|
+
await page.addScriptTag({ url: "https://cdnjs.cloudflare.com/ajax/libs/axe-core/4.9.1/axe.min.js" });
|
|
16126
|
+
const config = {
|
|
16127
|
+
runOnly: {
|
|
16128
|
+
type: level === "AAA" ? "standard" : "tag",
|
|
16129
|
+
values: level === "AAA" ? undefined : [level, "best-practice"]
|
|
16130
|
+
}
|
|
16131
|
+
};
|
|
16132
|
+
if (rules && rules.length > 0) {
|
|
16133
|
+
config.rules = Object.fromEntries(rules.map((r) => [r, { enabled: true }]));
|
|
16134
|
+
}
|
|
16135
|
+
if (exclude.length > 0) {
|
|
16136
|
+
config.exclude = exclude;
|
|
16137
|
+
}
|
|
16138
|
+
const result = await page.evaluate(async (auditConfig) => {
|
|
16139
|
+
const axeResult = await window.axe.run(auditConfig);
|
|
16140
|
+
return axeResult;
|
|
16141
|
+
}, config);
|
|
16142
|
+
const violations = (result.violations ?? []).map((v) => ({
|
|
16143
|
+
id: v.id,
|
|
16144
|
+
impact: v.impact,
|
|
16145
|
+
description: v.description,
|
|
16146
|
+
help: v.help,
|
|
16147
|
+
helpUrl: v.helpUrl,
|
|
16148
|
+
nodes: (v.nodes ?? []).map((n) => ({
|
|
16149
|
+
html: n.html,
|
|
16150
|
+
target: n.target,
|
|
16151
|
+
failureSummary: n.failureSummary
|
|
16152
|
+
}))
|
|
16153
|
+
}));
|
|
16154
|
+
const passes = (result.passes ?? []).map((p) => ({
|
|
16155
|
+
id: p.id,
|
|
16156
|
+
description: p.description
|
|
16157
|
+
}));
|
|
16158
|
+
const incomplete = (result.incomplete ?? []).map((i) => ({
|
|
16159
|
+
id: i.id,
|
|
16160
|
+
description: i.description,
|
|
16161
|
+
impact: i.impact
|
|
16162
|
+
}));
|
|
16163
|
+
const criticalCount = violations.filter((v) => v.impact === "critical").length;
|
|
16164
|
+
const seriousCount = violations.filter((v) => v.impact === "serious").length;
|
|
16165
|
+
const moderateCount = violations.filter((v) => v.impact === "moderate").length;
|
|
16166
|
+
const minorCount = violations.filter((v) => v.impact === "minor").length;
|
|
16167
|
+
return {
|
|
16168
|
+
violations,
|
|
16169
|
+
passes,
|
|
16170
|
+
incomplete,
|
|
16171
|
+
url: page.url(),
|
|
16172
|
+
timestamp: new Date().toISOString(),
|
|
16173
|
+
totalViolations: violations.length,
|
|
16174
|
+
criticalCount,
|
|
16175
|
+
seriousCount,
|
|
16176
|
+
moderateCount,
|
|
16177
|
+
minorCount
|
|
16178
|
+
};
|
|
16179
|
+
}
|
|
16180
|
+
|
|
16181
|
+
// src/lib/assertions.ts
|
|
16182
|
+
async function evaluateAssertions(page, assertions, context = {}) {
|
|
16183
|
+
const results = [];
|
|
16184
|
+
for (const assertion of assertions) {
|
|
16185
|
+
try {
|
|
16186
|
+
const result = await evaluateOne(page, assertion, context);
|
|
16187
|
+
results.push(result);
|
|
16188
|
+
} catch (err) {
|
|
16189
|
+
results.push({
|
|
16190
|
+
assertion,
|
|
16191
|
+
passed: false,
|
|
16192
|
+
actual: "",
|
|
16193
|
+
error: err instanceof Error ? err.message : String(err)
|
|
16194
|
+
});
|
|
16195
|
+
}
|
|
16196
|
+
}
|
|
16197
|
+
return results;
|
|
16198
|
+
}
|
|
16199
|
+
async function evaluateOne(page, assertion, context) {
|
|
16200
|
+
switch (assertion.type) {
|
|
16201
|
+
case "visible": {
|
|
16202
|
+
const visible = await page.locator(assertion.selector).isVisible();
|
|
16203
|
+
return {
|
|
16204
|
+
assertion,
|
|
16205
|
+
passed: visible,
|
|
16206
|
+
actual: String(visible)
|
|
16207
|
+
};
|
|
16208
|
+
}
|
|
16209
|
+
case "not_visible": {
|
|
16210
|
+
const visible = await page.locator(assertion.selector).isVisible();
|
|
16211
|
+
return {
|
|
16212
|
+
assertion,
|
|
16213
|
+
passed: !visible,
|
|
16214
|
+
actual: String(visible)
|
|
16215
|
+
};
|
|
16216
|
+
}
|
|
16217
|
+
case "text_contains": {
|
|
16218
|
+
const text = await page.locator(assertion.selector).textContent() ?? "";
|
|
16219
|
+
const expected = String(assertion.expected ?? "");
|
|
16220
|
+
return {
|
|
16221
|
+
assertion,
|
|
16222
|
+
passed: text.includes(expected),
|
|
16223
|
+
actual: text
|
|
16224
|
+
};
|
|
16225
|
+
}
|
|
16226
|
+
case "text_equals": {
|
|
16227
|
+
const text = await page.locator(assertion.selector).textContent() ?? "";
|
|
16228
|
+
const expected = String(assertion.expected ?? "");
|
|
16229
|
+
return {
|
|
16230
|
+
assertion,
|
|
16231
|
+
passed: text.trim() === expected.trim(),
|
|
16232
|
+
actual: text
|
|
16233
|
+
};
|
|
16234
|
+
}
|
|
16235
|
+
case "element_count": {
|
|
16236
|
+
const count = await page.locator(assertion.selector).count();
|
|
16237
|
+
const expected = Number(assertion.expected ?? 0);
|
|
16238
|
+
return {
|
|
16239
|
+
assertion,
|
|
16240
|
+
passed: count === expected,
|
|
16241
|
+
actual: String(count)
|
|
16242
|
+
};
|
|
16243
|
+
}
|
|
16244
|
+
case "no_console_errors": {
|
|
16245
|
+
if (context.consoleErrors !== undefined) {
|
|
16246
|
+
const errors = context.consoleErrors.filter(Boolean);
|
|
16247
|
+
return {
|
|
16248
|
+
assertion,
|
|
16249
|
+
passed: errors.length === 0,
|
|
16250
|
+
actual: errors.length === 0 ? "No console errors captured" : errors.slice(0, 3).join(" | ")
|
|
16251
|
+
};
|
|
16252
|
+
}
|
|
16253
|
+
const errorElements = await page.locator('[role="alert"], .error, .error-message, [data-testid="error"]').count();
|
|
16254
|
+
return {
|
|
16255
|
+
assertion,
|
|
16256
|
+
passed: errorElements === 0,
|
|
16257
|
+
actual: `${errorElements} error element(s) found`
|
|
16258
|
+
};
|
|
16259
|
+
}
|
|
16260
|
+
case "no_a11y_violations": {
|
|
16261
|
+
try {
|
|
16262
|
+
const auditResult = await runA11yAudit(page);
|
|
16263
|
+
const hasIssues = auditResult.violations.length > 0;
|
|
16264
|
+
return {
|
|
16265
|
+
assertion,
|
|
16266
|
+
passed: !hasIssues,
|
|
16267
|
+
actual: hasIssues ? `${auditResult.totalViolations} violation(s): ${auditResult.violations.map((v) => v.id).join(", ")}` : "No accessibility violations found"
|
|
16268
|
+
};
|
|
16269
|
+
} catch (err) {
|
|
16270
|
+
return {
|
|
16271
|
+
assertion,
|
|
16272
|
+
passed: false,
|
|
16273
|
+
actual: "",
|
|
16274
|
+
error: err instanceof Error ? err.message : String(err)
|
|
16275
|
+
};
|
|
16276
|
+
}
|
|
16277
|
+
}
|
|
16278
|
+
case "url_contains": {
|
|
16279
|
+
const url = page.url();
|
|
16280
|
+
const expected = String(assertion.expected ?? "");
|
|
16281
|
+
return {
|
|
16282
|
+
assertion,
|
|
16283
|
+
passed: url.includes(expected),
|
|
16284
|
+
actual: url
|
|
16285
|
+
};
|
|
16286
|
+
}
|
|
16287
|
+
case "title_contains": {
|
|
16288
|
+
const title = await page.title();
|
|
16289
|
+
const expected = String(assertion.expected ?? "");
|
|
16290
|
+
return {
|
|
16291
|
+
assertion,
|
|
16292
|
+
passed: title.includes(expected),
|
|
16293
|
+
actual: title
|
|
16294
|
+
};
|
|
16295
|
+
}
|
|
16296
|
+
case "cookie_exists": {
|
|
16297
|
+
const cookieName = assertion.expected;
|
|
16298
|
+
const cookies = await page.context().cookies();
|
|
16299
|
+
const found = cookies.some((c) => c.name === cookieName);
|
|
16300
|
+
return {
|
|
16301
|
+
assertion,
|
|
16302
|
+
passed: found,
|
|
16303
|
+
actual: found ? `Cookie "${cookieName}" exists` : `Cookie "${cookieName}" not found`
|
|
16304
|
+
};
|
|
16305
|
+
}
|
|
16306
|
+
case "cookie_not_exists": {
|
|
16307
|
+
const cookieName = assertion.expected;
|
|
16308
|
+
const cookies = await page.context().cookies();
|
|
16309
|
+
const found = cookies.some((c) => c.name === cookieName);
|
|
16310
|
+
return {
|
|
16311
|
+
assertion,
|
|
16312
|
+
passed: !found,
|
|
16313
|
+
actual: found ? `Cookie "${cookieName}" found (unexpected)` : `Cookie "${cookieName}" does not exist`
|
|
16314
|
+
};
|
|
16315
|
+
}
|
|
16316
|
+
case "cookie_value": {
|
|
16317
|
+
const [cookieName, expectedValue] = assertion.expected.split("=", 2);
|
|
16318
|
+
const cookies = await page.context().cookies();
|
|
16319
|
+
const cookie = cookies.find((c) => c.name === cookieName);
|
|
16320
|
+
const actualValue = cookie?.value ?? "";
|
|
16321
|
+
return {
|
|
16322
|
+
assertion,
|
|
16323
|
+
passed: actualValue === expectedValue,
|
|
16324
|
+
actual: cookie ? `${cookieName}=${actualValue}` : `Cookie "${cookieName}" not found`
|
|
16325
|
+
};
|
|
16326
|
+
}
|
|
16327
|
+
case "local_storage_exists": {
|
|
16328
|
+
const key = assertion.expected;
|
|
16329
|
+
const value = await page.evaluate((k) => localStorage.getItem(k), key);
|
|
16330
|
+
return {
|
|
16331
|
+
assertion,
|
|
16332
|
+
passed: value !== null,
|
|
16333
|
+
actual: value !== null ? `Key "${key}" exists with value "${value}"` : `Key "${key}" not found in localStorage`
|
|
16334
|
+
};
|
|
16335
|
+
}
|
|
16336
|
+
case "local_storage_not_exists": {
|
|
16337
|
+
const key = assertion.expected;
|
|
16338
|
+
const value = await page.evaluate((k) => localStorage.getItem(k), key);
|
|
16339
|
+
return {
|
|
16340
|
+
assertion,
|
|
16341
|
+
passed: value === null,
|
|
16342
|
+
actual: value !== null ? `Key "${key}" exists (unexpected)` : `Key "${key}" does not exist in localStorage`
|
|
16343
|
+
};
|
|
16344
|
+
}
|
|
16345
|
+
case "local_storage_value": {
|
|
16346
|
+
const [lsKey, expectedValue] = assertion.expected.split("=", 2);
|
|
16347
|
+
const value = await page.evaluate((k) => localStorage.getItem(k), lsKey ?? "");
|
|
16348
|
+
return {
|
|
16349
|
+
assertion,
|
|
16350
|
+
passed: value === expectedValue,
|
|
16351
|
+
actual: value !== null ? `${lsKey}=${value}` : `Key "${lsKey}" not found in localStorage`
|
|
16352
|
+
};
|
|
16353
|
+
}
|
|
16354
|
+
case "session_storage_value": {
|
|
16355
|
+
const [ssKey, expectedValue] = assertion.expected.split("=", 2);
|
|
16356
|
+
const value = await page.evaluate((k) => sessionStorage.getItem(k), ssKey ?? "");
|
|
16357
|
+
return {
|
|
16358
|
+
assertion,
|
|
16359
|
+
passed: value === expectedValue,
|
|
16360
|
+
actual: value !== null ? `${ssKey}=${value}` : `Key "${ssKey}" not found in sessionStorage`
|
|
16361
|
+
};
|
|
16362
|
+
}
|
|
16363
|
+
case "session_storage_not_exists": {
|
|
16364
|
+
const key = assertion.expected;
|
|
16365
|
+
const value = await page.evaluate((k) => sessionStorage.getItem(k), key);
|
|
16366
|
+
return {
|
|
16367
|
+
assertion,
|
|
16368
|
+
passed: value === null,
|
|
16369
|
+
actual: value !== null ? `Key "${key}" exists (unexpected)` : `Key "${key}" does not exist in sessionStorage`
|
|
16370
|
+
};
|
|
16371
|
+
}
|
|
16372
|
+
default: {
|
|
16373
|
+
return {
|
|
16374
|
+
assertion,
|
|
16375
|
+
passed: false,
|
|
16376
|
+
actual: "",
|
|
16377
|
+
error: `Unknown assertion type: ${assertion.type}`
|
|
16378
|
+
};
|
|
16379
|
+
}
|
|
16380
|
+
}
|
|
16381
|
+
}
|
|
16382
|
+
function allAssertionsPassed(results) {
|
|
16383
|
+
return results.every((r) => r.passed);
|
|
16384
|
+
}
|
|
16385
|
+
function formatAssertionResults(results) {
|
|
16386
|
+
if (results.length === 0)
|
|
16387
|
+
return "No assertions.";
|
|
16388
|
+
const lines = [];
|
|
16389
|
+
for (const r of results) {
|
|
16390
|
+
const icon = r.passed ? "PASS" : "FAIL";
|
|
16391
|
+
const desc = r.assertion.description || `${r.assertion.type}${r.assertion.selector ? ` ${r.assertion.selector}` : ""}`;
|
|
16392
|
+
let line = ` [${icon}] ${desc}`;
|
|
16393
|
+
if (!r.passed) {
|
|
16394
|
+
line += ` (actual: ${r.actual})`;
|
|
16395
|
+
if (r.error)
|
|
16396
|
+
line += ` \u2014 ${r.error}`;
|
|
16397
|
+
}
|
|
16398
|
+
lines.push(line);
|
|
16399
|
+
}
|
|
16400
|
+
const passed = results.filter((r) => r.passed).length;
|
|
16401
|
+
lines.push(`
|
|
16402
|
+
${passed}/${results.length} assertions passed.`);
|
|
16403
|
+
return lines.join(`
|
|
16404
|
+
`);
|
|
16405
|
+
}
|
|
16406
|
+
|
|
15897
16407
|
// src/lib/runner.ts
|
|
15898
16408
|
var eventHandler = null;
|
|
15899
16409
|
function onRunEvent(handler) {
|
|
@@ -15903,6 +16413,54 @@ function emit(event) {
|
|
|
15903
16413
|
if (eventHandler)
|
|
15904
16414
|
eventHandler(event);
|
|
15905
16415
|
}
|
|
16416
|
+
function assertionDescription(result) {
|
|
16417
|
+
return result.assertion.description || `${result.assertion.type}${result.assertion.selector ? ` ${result.assertion.selector}` : ""}`;
|
|
16418
|
+
}
|
|
16419
|
+
function summarizeAssertionResult(result) {
|
|
16420
|
+
const description = assertionDescription(result);
|
|
16421
|
+
if (result.passed)
|
|
16422
|
+
return description;
|
|
16423
|
+
const suffix = result.error ? `; ${result.error}` : "";
|
|
16424
|
+
return `${description} (actual: ${result.actual}${suffix})`;
|
|
16425
|
+
}
|
|
16426
|
+
async function applyStructuredAssertionsToResult(input) {
|
|
16427
|
+
const assertions = input.scenario.assertions ?? [];
|
|
16428
|
+
if (assertions.length === 0) {
|
|
16429
|
+
return {
|
|
16430
|
+
status: input.status,
|
|
16431
|
+
reasoning: input.reasoning,
|
|
16432
|
+
assertionsPassed: [],
|
|
16433
|
+
assertionsFailed: [],
|
|
16434
|
+
assertionResults: []
|
|
16435
|
+
};
|
|
16436
|
+
}
|
|
16437
|
+
const results = await evaluateAssertions(input.page, assertions, {
|
|
16438
|
+
consoleErrors: input.consoleErrors
|
|
16439
|
+
});
|
|
16440
|
+
const assertionsPassed = results.filter((r) => r.passed).map(summarizeAssertionResult);
|
|
16441
|
+
const assertionsFailed = results.filter((r) => !r.passed).map(summarizeAssertionResult);
|
|
16442
|
+
const assertionResults = results.map((result) => ({
|
|
16443
|
+
type: result.assertion.type,
|
|
16444
|
+
description: assertionDescription(result),
|
|
16445
|
+
passed: result.passed,
|
|
16446
|
+
actual: result.actual,
|
|
16447
|
+
...result.error ? { error: result.error } : {}
|
|
16448
|
+
}));
|
|
16449
|
+
const assertionsOk = allAssertionsPassed(results);
|
|
16450
|
+
const status = assertionsOk || input.status !== "passed" ? input.status : "failed";
|
|
16451
|
+
const assertionHeading = assertionsOk ? "Structured assertions passed:" : "Structured assertions failed:";
|
|
16452
|
+
const reasoningParts = [input.reasoning, `${assertionHeading}
|
|
16453
|
+
${formatAssertionResults(results)}`].map((part) => part.trim()).filter(Boolean);
|
|
16454
|
+
return {
|
|
16455
|
+
status,
|
|
16456
|
+
reasoning: reasoningParts.join(`
|
|
16457
|
+
|
|
16458
|
+
`),
|
|
16459
|
+
assertionsPassed,
|
|
16460
|
+
assertionsFailed,
|
|
16461
|
+
assertionResults
|
|
16462
|
+
};
|
|
16463
|
+
}
|
|
15906
16464
|
function withTimeout(promise, ms, label) {
|
|
15907
16465
|
return new Promise((resolve, reject) => {
|
|
15908
16466
|
const warningAt = Math.floor(ms * 0.8);
|
|
@@ -16073,6 +16631,7 @@ async function runSingleScenario(scenario, runId, options) {
|
|
|
16073
16631
|
model,
|
|
16074
16632
|
runId,
|
|
16075
16633
|
sessionId: result.id,
|
|
16634
|
+
baseUrl: options.url,
|
|
16076
16635
|
maxTurns: effectiveOptions.minimal ? 10 : 30,
|
|
16077
16636
|
a11y: effectiveOptions.a11y,
|
|
16078
16637
|
persona: persona ? {
|
|
@@ -16155,27 +16714,46 @@ async function runSingleScenario(scenario, runId, options) {
|
|
|
16155
16714
|
closeSession(result.id);
|
|
16156
16715
|
const lightpandaNote = options.engine === "lightpanda" ? " (Running with Lightpanda \u2014 no screenshots)" : options.engine === "bun" ? " (Running with Bun.WebView \u2014 native, ~11x faster)" : "";
|
|
16157
16716
|
const networkMeta = networkErrors.length > 0 ? { networkErrors: networkErrors.slice(0, 20) } : {};
|
|
16158
|
-
|
|
16717
|
+
const baseReasoning = agentResult.reasoning ? agentResult.reasoning + lightpandaNote : lightpandaNote || "";
|
|
16718
|
+
const assertionOutcome = await applyStructuredAssertionsToResult({
|
|
16719
|
+
page,
|
|
16720
|
+
scenario,
|
|
16721
|
+
consoleErrors,
|
|
16159
16722
|
status: agentResult.status,
|
|
16160
|
-
reasoning:
|
|
16723
|
+
reasoning: baseReasoning
|
|
16724
|
+
});
|
|
16725
|
+
const structuredAssertionMeta = assertionOutcome.assertionResults.length > 0 ? {
|
|
16726
|
+
structuredAssertions: {
|
|
16727
|
+
passed: assertionOutcome.assertionsPassed,
|
|
16728
|
+
failed: assertionOutcome.assertionsFailed,
|
|
16729
|
+
results: assertionOutcome.assertionResults
|
|
16730
|
+
}
|
|
16731
|
+
} : {};
|
|
16732
|
+
let updatedResult = updateResult(result.id, {
|
|
16733
|
+
status: assertionOutcome.status,
|
|
16734
|
+
reasoning: assertionOutcome.reasoning || undefined,
|
|
16161
16735
|
stepsCompleted: agentResult.stepsCompleted,
|
|
16162
16736
|
durationMs: Date.now() - new Date(result.createdAt).getTime(),
|
|
16163
16737
|
tokensUsed: agentResult.tokensUsed,
|
|
16164
16738
|
costCents: estimateCost(model, agentResult.tokensUsed),
|
|
16165
|
-
metadata: {
|
|
16739
|
+
metadata: {
|
|
16740
|
+
consoleLogs,
|
|
16741
|
+
...networkErrors.length > 0 ? networkMeta : {},
|
|
16742
|
+
...structuredAssertionMeta
|
|
16743
|
+
}
|
|
16166
16744
|
});
|
|
16167
|
-
if (
|
|
16168
|
-
const failureAnalysis = analyzeFailure(null,
|
|
16745
|
+
if (assertionOutcome.status === "failed" || assertionOutcome.status === "error") {
|
|
16746
|
+
const failureAnalysis = analyzeFailure(null, assertionOutcome.reasoning ?? null);
|
|
16169
16747
|
if (failureAnalysis) {
|
|
16170
16748
|
updatedResult = updateResult(result.id, { failureAnalysis });
|
|
16171
16749
|
}
|
|
16172
16750
|
}
|
|
16173
|
-
if (
|
|
16751
|
+
if (assertionOutcome.status === "passed") {
|
|
16174
16752
|
try {
|
|
16175
16753
|
updateScenarioPassedCache(scenario.id, options.url);
|
|
16176
16754
|
} catch {}
|
|
16177
16755
|
}
|
|
16178
|
-
const eventType =
|
|
16756
|
+
const eventType = assertionOutcome.status === "passed" ? "scenario:pass" : "scenario:fail";
|
|
16179
16757
|
emit({ type: eventType, scenarioId: scenario.id, scenarioName: scenario.name, resultId: result.id, runId });
|
|
16180
16758
|
return updatedResult;
|
|
16181
16759
|
} catch (error) {
|
|
@@ -16200,7 +16778,8 @@ async function runSingleScenario(scenario, runId, options) {
|
|
|
16200
16778
|
} finally {
|
|
16201
16779
|
if (harPath) {
|
|
16202
16780
|
try {
|
|
16203
|
-
|
|
16781
|
+
const existing = getResult(result.id);
|
|
16782
|
+
updateResult(result.id, { metadata: { ...existing?.metadata ?? {}, harPath } });
|
|
16204
16783
|
} catch {}
|
|
16205
16784
|
}
|
|
16206
16785
|
if (browser) {
|
|
@@ -16372,22 +16951,31 @@ async function runBatch(scenarios, options) {
|
|
|
16372
16951
|
}
|
|
16373
16952
|
return { run: finalRun, results };
|
|
16374
16953
|
}
|
|
16375
|
-
|
|
16376
|
-
|
|
16954
|
+
function findScenarioInList(scenarios, id) {
|
|
16955
|
+
return scenarios.find((scenario) => scenario.id === id || scenario.shortId === id || scenario.id.startsWith(id)) ?? null;
|
|
16956
|
+
}
|
|
16957
|
+
function resolveScenariosForRun(options) {
|
|
16377
16958
|
if (options.scenarioIds && options.scenarioIds.length > 0) {
|
|
16378
|
-
const
|
|
16379
|
-
|
|
16380
|
-
|
|
16381
|
-
|
|
16382
|
-
|
|
16959
|
+
const scoped = listScenarios({ projectId: options.projectId });
|
|
16960
|
+
const resolved = [];
|
|
16961
|
+
const seen = new Set;
|
|
16962
|
+
for (const id of options.scenarioIds) {
|
|
16963
|
+
const scenario = findScenarioInList(scoped, id) ?? getScenario(id);
|
|
16964
|
+
if (scenario && !seen.has(scenario.id)) {
|
|
16965
|
+
resolved.push(scenario);
|
|
16966
|
+
seen.add(scenario.id);
|
|
16967
|
+
}
|
|
16383
16968
|
}
|
|
16384
|
-
|
|
16385
|
-
scenarios = listScenarios({
|
|
16386
|
-
projectId: options.projectId,
|
|
16387
|
-
tags: options.tags,
|
|
16388
|
-
priority: options.priority
|
|
16389
|
-
});
|
|
16969
|
+
return resolved;
|
|
16390
16970
|
}
|
|
16971
|
+
return listScenarios({
|
|
16972
|
+
projectId: options.projectId,
|
|
16973
|
+
tags: options.tags,
|
|
16974
|
+
priority: options.priority
|
|
16975
|
+
});
|
|
16976
|
+
}
|
|
16977
|
+
async function runByFilter(options) {
|
|
16978
|
+
const scenarios = resolveScenariosForRun(options);
|
|
16391
16979
|
if (scenarios.length === 0) {
|
|
16392
16980
|
const config = loadConfig();
|
|
16393
16981
|
const model = resolveModel2(options.model ?? config.defaultModel);
|
|
@@ -16400,17 +16988,7 @@ async function runByFilter(options) {
|
|
|
16400
16988
|
function startRunAsync(options) {
|
|
16401
16989
|
const config = loadConfig();
|
|
16402
16990
|
const model = resolveModel2(options.model ?? config.defaultModel);
|
|
16403
|
-
|
|
16404
|
-
if (options.scenarioIds && options.scenarioIds.length > 0) {
|
|
16405
|
-
const all = listScenarios({ projectId: options.projectId });
|
|
16406
|
-
scenarios = all.filter((s) => options.scenarioIds.includes(s.id) || options.scenarioIds.includes(s.shortId));
|
|
16407
|
-
} else {
|
|
16408
|
-
scenarios = listScenarios({
|
|
16409
|
-
projectId: options.projectId,
|
|
16410
|
-
tags: options.tags,
|
|
16411
|
-
priority: options.priority
|
|
16412
|
-
});
|
|
16413
|
-
}
|
|
16991
|
+
const scenarios = resolveScenariosForRun(options);
|
|
16414
16992
|
if (!options.skipBudgetCheck) {
|
|
16415
16993
|
const cap = options.maxCostCents ?? config.defaultMaxCostCents;
|
|
16416
16994
|
if (cap !== undefined && cap > 0 && scenarios.length > 0) {
|
|
@@ -16495,6 +17073,170 @@ function estimateCost(model, tokens) {
|
|
|
16495
17073
|
const costPer1M = costs[model] ?? 0.5;
|
|
16496
17074
|
return tokens / 1e6 * costPer1M * 100;
|
|
16497
17075
|
}
|
|
17076
|
+
// src/lib/workflow-runner.ts
|
|
17077
|
+
init_database();
|
|
17078
|
+
import { mkdtempSync, rmSync, writeFileSync as writeFileSync3 } from "fs";
|
|
17079
|
+
import { tmpdir } from "os";
|
|
17080
|
+
import { join as join14 } from "path";
|
|
17081
|
+
function buildWorkflowRunPlan(workflow, options) {
|
|
17082
|
+
const runOptions = {
|
|
17083
|
+
url: options.url,
|
|
17084
|
+
model: options.model,
|
|
17085
|
+
headed: options.headed,
|
|
17086
|
+
parallel: options.parallel,
|
|
17087
|
+
timeout: options.timeout ?? workflow.execution.timeoutMs,
|
|
17088
|
+
projectId: workflow.projectId ?? undefined,
|
|
17089
|
+
scenarioIds: workflow.scenarioFilter.scenarioIds,
|
|
17090
|
+
tags: workflow.scenarioFilter.tags,
|
|
17091
|
+
priority: workflow.scenarioFilter.priority,
|
|
17092
|
+
personaIds: workflow.personaIds.length > 0 ? workflow.personaIds : undefined
|
|
17093
|
+
};
|
|
17094
|
+
return {
|
|
17095
|
+
workflow,
|
|
17096
|
+
runOptions,
|
|
17097
|
+
sandbox: workflow.execution.target === "sandbox" ? buildSandboxPlan(workflow, workflow.execution, runOptions) : null
|
|
17098
|
+
};
|
|
17099
|
+
}
|
|
17100
|
+
async function runTestingWorkflow(workflowId, options, dependencies = {}) {
|
|
17101
|
+
const workflow = getTestingWorkflow(workflowId);
|
|
17102
|
+
if (!workflow)
|
|
17103
|
+
throw new Error(`Testing workflow not found: ${workflowId}`);
|
|
17104
|
+
if (!workflow.enabled)
|
|
17105
|
+
throw new Error(`Testing workflow is disabled: ${workflow.name}`);
|
|
17106
|
+
validatePersonaIds(workflow);
|
|
17107
|
+
const plan = buildWorkflowRunPlan(workflow, options);
|
|
17108
|
+
if (options.dryRun)
|
|
17109
|
+
return { run: null, results: [], plan };
|
|
17110
|
+
if (workflow.execution.target === "sandbox") {
|
|
17111
|
+
const sandboxResult = await runViaSandbox(plan, dependencies);
|
|
17112
|
+
return { run: null, results: [], plan, sandboxResult };
|
|
17113
|
+
}
|
|
17114
|
+
const runLocal = dependencies.runByFilter ?? runByFilter;
|
|
17115
|
+
const { run, results } = await runLocal(plan.runOptions);
|
|
17116
|
+
return { run, results, plan };
|
|
17117
|
+
}
|
|
17118
|
+
function createWorkflowDatabaseBundle(workflow, plan) {
|
|
17119
|
+
if (!plan.sandbox)
|
|
17120
|
+
throw new Error(`Workflow is not configured for sandbox execution: ${workflow.name}`);
|
|
17121
|
+
const localDir = mkdtempSync(join14(tmpdir(), `testers-workflow-${workflow.id.slice(0, 8)}-`));
|
|
17122
|
+
writeFileSync3(join14(localDir, "testers.db"), getDatabase().serialize());
|
|
17123
|
+
return {
|
|
17124
|
+
localDir,
|
|
17125
|
+
remoteDir: plan.sandbox.stateRemoteDir,
|
|
17126
|
+
cleanup: () => rmSync(localDir, { recursive: true, force: true })
|
|
17127
|
+
};
|
|
17128
|
+
}
|
|
17129
|
+
function validatePersonaIds(workflow) {
|
|
17130
|
+
for (const personaId of workflow.personaIds) {
|
|
17131
|
+
if (!getPersona(personaId)) {
|
|
17132
|
+
throw new Error(`Persona not found for workflow ${workflow.name}: ${personaId}`);
|
|
17133
|
+
}
|
|
17134
|
+
}
|
|
17135
|
+
}
|
|
17136
|
+
function buildSandboxPlan(workflow, execution, runOptions) {
|
|
17137
|
+
const remoteDir = execution.sandboxRemoteDir ?? `/tmp/testers-workflow-${workflow.id.slice(0, 8)}`;
|
|
17138
|
+
const stateRemoteDir = `${remoteDir.replace(/\/+$/, "")}/.testers-state`;
|
|
17139
|
+
return {
|
|
17140
|
+
provider: execution.provider,
|
|
17141
|
+
image: execution.sandboxImage,
|
|
17142
|
+
name: `testers-${workflow.id.slice(0, 8)}`,
|
|
17143
|
+
remoteDir,
|
|
17144
|
+
stateRemoteDir,
|
|
17145
|
+
cleanup: execution.sandboxCleanup ?? "delete",
|
|
17146
|
+
timeoutMs: execution.timeoutMs,
|
|
17147
|
+
env: execution.env,
|
|
17148
|
+
command: buildSandboxCommand({
|
|
17149
|
+
runOptions,
|
|
17150
|
+
remoteDir,
|
|
17151
|
+
dbPath: `${stateRemoteDir}/testers.db`,
|
|
17152
|
+
setupCommand: execution.setupCommand,
|
|
17153
|
+
packageSpec: execution.packageSpec ?? "@hasna/testers"
|
|
17154
|
+
})
|
|
17155
|
+
};
|
|
17156
|
+
}
|
|
17157
|
+
function buildSandboxCommand(input) {
|
|
17158
|
+
const args = [
|
|
17159
|
+
"bunx",
|
|
17160
|
+
input.packageSpec,
|
|
17161
|
+
"run",
|
|
17162
|
+
input.runOptions.url,
|
|
17163
|
+
...input.runOptions.scenarioIds?.length ? ["--scenario", input.runOptions.scenarioIds.join(",")] : [],
|
|
17164
|
+
...input.runOptions.tags?.length ? input.runOptions.tags.flatMap((tag) => ["--tag", tag]) : [],
|
|
17165
|
+
...input.runOptions.priority ? ["--priority", input.runOptions.priority] : [],
|
|
17166
|
+
...input.runOptions.projectId ? ["--project", input.runOptions.projectId] : [],
|
|
17167
|
+
...input.runOptions.model ? ["--model", input.runOptions.model] : [],
|
|
17168
|
+
...input.runOptions.headed ? ["--headed"] : [],
|
|
17169
|
+
...input.runOptions.parallel ? ["--parallel", String(input.runOptions.parallel)] : [],
|
|
17170
|
+
...input.runOptions.timeout ? ["--timeout", String(input.runOptions.timeout)] : [],
|
|
17171
|
+
...input.runOptions.personaIds?.length ? ["--persona", input.runOptions.personaIds.join(",")] : [],
|
|
17172
|
+
"--no-auto-generate",
|
|
17173
|
+
"--json"
|
|
17174
|
+
];
|
|
17175
|
+
return [
|
|
17176
|
+
"set -euo pipefail",
|
|
17177
|
+
`mkdir -p ${shellQuote(input.remoteDir)}`,
|
|
17178
|
+
`cd ${shellQuote(input.remoteDir)}`,
|
|
17179
|
+
input.setupCommand,
|
|
17180
|
+
`HASNA_TESTERS_DB_PATH=${shellQuote(input.dbPath)} ${args.map(shellQuote).join(" ")}`
|
|
17181
|
+
].filter(Boolean).join(`
|
|
17182
|
+
`);
|
|
17183
|
+
}
|
|
17184
|
+
async function runViaSandbox(plan, dependencies) {
|
|
17185
|
+
if (!plan.sandbox)
|
|
17186
|
+
throw new Error("Workflow does not have a sandbox plan");
|
|
17187
|
+
const sandboxes = await resolveSandboxesRuntime(dependencies);
|
|
17188
|
+
const createBundle = dependencies.createDatabaseBundle ?? createWorkflowDatabaseBundle;
|
|
17189
|
+
const bundle = createBundle(plan.workflow, plan);
|
|
17190
|
+
try {
|
|
17191
|
+
const raw = await sandboxes.runCommandInSandbox({
|
|
17192
|
+
command: plan.sandbox.command,
|
|
17193
|
+
provider: plan.sandbox.provider,
|
|
17194
|
+
name: plan.sandbox.name,
|
|
17195
|
+
image: plan.sandbox.image,
|
|
17196
|
+
sandboxTimeout: plan.sandbox.timeoutMs,
|
|
17197
|
+
commandTimeoutMs: plan.sandbox.timeoutMs,
|
|
17198
|
+
projectId: plan.workflow.projectId ?? undefined,
|
|
17199
|
+
config: {
|
|
17200
|
+
source: "testers",
|
|
17201
|
+
workflowId: plan.workflow.id,
|
|
17202
|
+
workflowName: plan.workflow.name
|
|
17203
|
+
},
|
|
17204
|
+
sandboxEnvVars: plan.sandbox.env,
|
|
17205
|
+
cleanup: plan.sandbox.cleanup,
|
|
17206
|
+
upload: {
|
|
17207
|
+
localDir: bundle.localDir,
|
|
17208
|
+
remoteDir: bundle.remoteDir
|
|
17209
|
+
}
|
|
17210
|
+
});
|
|
17211
|
+
const exitCode = raw.result.exit_code ?? raw.result.exitCode ?? 0;
|
|
17212
|
+
const stdout = raw.result.stdout ?? "";
|
|
17213
|
+
const stderr = raw.result.stderr ?? "";
|
|
17214
|
+
if (exitCode !== 0) {
|
|
17215
|
+
throw new Error(`Sandbox workflow execution failed (${exitCode}): ${stderr || stdout}`);
|
|
17216
|
+
}
|
|
17217
|
+
return {
|
|
17218
|
+
sandboxId: raw.sandbox.id,
|
|
17219
|
+
sessionId: raw.session.id,
|
|
17220
|
+
exitCode,
|
|
17221
|
+
stdout,
|
|
17222
|
+
stderr,
|
|
17223
|
+
cleanup: raw.cleanup
|
|
17224
|
+
};
|
|
17225
|
+
} finally {
|
|
17226
|
+
bundle.cleanup?.();
|
|
17227
|
+
}
|
|
17228
|
+
}
|
|
17229
|
+
async function resolveSandboxesRuntime(dependencies) {
|
|
17230
|
+
if (dependencies.sandboxes)
|
|
17231
|
+
return dependencies.sandboxes;
|
|
17232
|
+
if (dependencies.createSandboxesSDK)
|
|
17233
|
+
return dependencies.createSandboxesSDK();
|
|
17234
|
+
const mod = await import("@hasna/sandboxes");
|
|
17235
|
+
return mod.createSandboxesSDK();
|
|
17236
|
+
}
|
|
17237
|
+
function shellQuote(value) {
|
|
17238
|
+
return `'${value.replaceAll("'", `'"'"'`)}'`;
|
|
17239
|
+
}
|
|
16498
17240
|
// src/lib/reporter.ts
|
|
16499
17241
|
init_database();
|
|
16500
17242
|
function useEmoji() {
|
|
@@ -16668,9 +17410,9 @@ function formatRunList(runs) {
|
|
|
16668
17410
|
`);
|
|
16669
17411
|
}
|
|
16670
17412
|
function getScenarioRunStats(scenarioId) {
|
|
16671
|
-
const
|
|
16672
|
-
const lastRow =
|
|
16673
|
-
const statsRow =
|
|
17413
|
+
const db3 = getDatabase();
|
|
17414
|
+
const lastRow = db3.query("SELECT status FROM results WHERE scenario_id = ? ORDER BY created_at DESC LIMIT 1").get(scenarioId);
|
|
17415
|
+
const statsRow = db3.query("SELECT COUNT(*) as total, SUM(CASE WHEN status = 'passed' THEN 1 ELSE 0 END) as passed FROM results WHERE scenario_id = ?").get(scenarioId);
|
|
16674
17416
|
return {
|
|
16675
17417
|
lastStatus: lastRow ? lastRow.status : null,
|
|
16676
17418
|
passRate: statsRow && statsRow.total > 0 ? `${statsRow.passed}/${statsRow.total}` : "\u2014"
|
|
@@ -16960,10 +17702,10 @@ class Scheduler {
|
|
|
16960
17702
|
}
|
|
16961
17703
|
// src/lib/init.ts
|
|
16962
17704
|
init_paths();
|
|
16963
|
-
import { existsSync as existsSync11, readFileSync as readFileSync3, writeFileSync as
|
|
16964
|
-
import { join as
|
|
17705
|
+
import { existsSync as existsSync11, readFileSync as readFileSync3, writeFileSync as writeFileSync4, mkdirSync as mkdirSync9 } from "fs";
|
|
17706
|
+
import { join as join15, basename } from "path";
|
|
16965
17707
|
function detectFramework(dir) {
|
|
16966
|
-
const pkgPath =
|
|
17708
|
+
const pkgPath = join15(dir, "package.json");
|
|
16967
17709
|
if (!existsSync11(pkgPath))
|
|
16968
17710
|
return null;
|
|
16969
17711
|
let pkg;
|
|
@@ -17191,7 +17933,7 @@ function initProject(options) {
|
|
|
17191
17933
|
}
|
|
17192
17934
|
}).filter((s) => s !== null);
|
|
17193
17935
|
const configDir = getTestersDir();
|
|
17194
|
-
const configPath =
|
|
17936
|
+
const configPath = join15(configDir, "config.json");
|
|
17195
17937
|
if (!existsSync11(configDir)) {
|
|
17196
17938
|
mkdirSync9(configDir, { recursive: true });
|
|
17197
17939
|
}
|
|
@@ -17202,7 +17944,7 @@ function initProject(options) {
|
|
|
17202
17944
|
} catch {}
|
|
17203
17945
|
}
|
|
17204
17946
|
config.activeProject = project.id;
|
|
17205
|
-
|
|
17947
|
+
writeFileSync4(configPath, JSON.stringify(config, null, 2), "utf-8");
|
|
17206
17948
|
return { project, scenarios, framework, url };
|
|
17207
17949
|
}
|
|
17208
17950
|
// src/lib/smoke.ts
|
|
@@ -17630,28 +18372,28 @@ function fromRow2(row) {
|
|
|
17630
18372
|
};
|
|
17631
18373
|
}
|
|
17632
18374
|
function createAuthPreset(input) {
|
|
17633
|
-
const
|
|
18375
|
+
const db3 = getDatabase();
|
|
17634
18376
|
const id = uuid();
|
|
17635
18377
|
const timestamp = now();
|
|
17636
|
-
|
|
18378
|
+
db3.query(`
|
|
17637
18379
|
INSERT INTO auth_presets (id, name, email, password, login_path, metadata, created_at)
|
|
17638
18380
|
VALUES (?, ?, ?, ?, ?, '{}', ?)
|
|
17639
18381
|
`).run(id, input.name, input.email, input.password, input.loginPath ?? "/login", timestamp);
|
|
17640
18382
|
return getAuthPreset(input.name);
|
|
17641
18383
|
}
|
|
17642
18384
|
function getAuthPreset(name) {
|
|
17643
|
-
const
|
|
17644
|
-
const row =
|
|
18385
|
+
const db3 = getDatabase();
|
|
18386
|
+
const row = db3.query("SELECT * FROM auth_presets WHERE name = ?").get(name);
|
|
17645
18387
|
return row ? fromRow2(row) : null;
|
|
17646
18388
|
}
|
|
17647
18389
|
function listAuthPresets() {
|
|
17648
|
-
const
|
|
17649
|
-
const rows =
|
|
18390
|
+
const db3 = getDatabase();
|
|
18391
|
+
const rows = db3.query("SELECT * FROM auth_presets ORDER BY created_at DESC").all();
|
|
17650
18392
|
return rows.map(fromRow2);
|
|
17651
18393
|
}
|
|
17652
18394
|
function deleteAuthPreset(name) {
|
|
17653
|
-
const
|
|
17654
|
-
const result =
|
|
18395
|
+
const db3 = getDatabase();
|
|
18396
|
+
const result = db3.query("DELETE FROM auth_presets WHERE name = ?").run(name);
|
|
17655
18397
|
return result.changes > 0;
|
|
17656
18398
|
}
|
|
17657
18399
|
// src/lib/report.ts
|
|
@@ -17947,12 +18689,12 @@ async function startWatcher(options) {
|
|
|
17947
18689
|
}
|
|
17948
18690
|
// src/lib/repo-discovery.ts
|
|
17949
18691
|
init_paths();
|
|
17950
|
-
import { existsSync as existsSync13, readFileSync as readFileSync5, readdirSync as readdirSync3, statSync, writeFileSync as
|
|
18692
|
+
import { existsSync as existsSync13, readFileSync as readFileSync5, readdirSync as readdirSync3, statSync, writeFileSync as writeFileSync5, mkdirSync as mkdirSync10, unlinkSync } from "fs";
|
|
17951
18693
|
import { createHash } from "crypto";
|
|
17952
|
-
import { join as
|
|
18694
|
+
import { join as join16, resolve as resolve2, relative as relative2 } from "path";
|
|
17953
18695
|
function getCacheDir() {
|
|
17954
18696
|
const testersDir = getTestersDir();
|
|
17955
|
-
const cacheDir =
|
|
18697
|
+
const cacheDir = join16(testersDir, "repo-index");
|
|
17956
18698
|
if (!existsSync13(cacheDir)) {
|
|
17957
18699
|
mkdirSync10(cacheDir, { recursive: true });
|
|
17958
18700
|
}
|
|
@@ -17962,11 +18704,11 @@ function pathHash(repoPath) {
|
|
|
17962
18704
|
return createHash("sha256").update(repoPath).digest("hex").slice(0, 16);
|
|
17963
18705
|
}
|
|
17964
18706
|
function getCachePath(repoPath) {
|
|
17965
|
-
return
|
|
18707
|
+
return join16(getCacheDir(), `${pathHash(repoPath)}.json`);
|
|
17966
18708
|
}
|
|
17967
18709
|
function isCacheStale(cached, repoPath) {
|
|
17968
18710
|
for (const spec of cached.specs) {
|
|
17969
|
-
const fullPath =
|
|
18711
|
+
const fullPath = join16(repoPath, spec.file);
|
|
17970
18712
|
if (!existsSync13(fullPath))
|
|
17971
18713
|
return true;
|
|
17972
18714
|
try {
|
|
@@ -17978,11 +18720,11 @@ function isCacheStale(cached, repoPath) {
|
|
|
17978
18720
|
}
|
|
17979
18721
|
}
|
|
17980
18722
|
if (cached.configPath) {
|
|
17981
|
-
const configFullPath =
|
|
18723
|
+
const configFullPath = join16(repoPath, cached.configPath);
|
|
17982
18724
|
if (!existsSync13(configFullPath))
|
|
17983
18725
|
return true;
|
|
17984
18726
|
try {
|
|
17985
|
-
|
|
18727
|
+
statSync(configFullPath);
|
|
17986
18728
|
const age = Date.now() - new Date(cached.snapshotAt).getTime();
|
|
17987
18729
|
if (age > 3600000)
|
|
17988
18730
|
return true;
|
|
@@ -18005,14 +18747,14 @@ function loadCache(repoPath) {
|
|
|
18005
18747
|
}
|
|
18006
18748
|
function saveCache(snapshot) {
|
|
18007
18749
|
const cachePath = getCachePath(snapshot.repoPath);
|
|
18008
|
-
|
|
18750
|
+
writeFileSync5(cachePath, JSON.stringify(snapshot, null, 2), "utf-8");
|
|
18009
18751
|
}
|
|
18010
18752
|
function detectPackageManager(repoPath) {
|
|
18011
18753
|
const result = {
|
|
18012
|
-
npm: existsSync13(
|
|
18013
|
-
yarn: existsSync13(
|
|
18014
|
-
pnpm: existsSync13(
|
|
18015
|
-
bun: existsSync13(
|
|
18754
|
+
npm: existsSync13(join16(repoPath, "package-lock.json")),
|
|
18755
|
+
yarn: existsSync13(join16(repoPath, "yarn.lock")),
|
|
18756
|
+
pnpm: existsSync13(join16(repoPath, "pnpm-lock.yaml")),
|
|
18757
|
+
bun: existsSync13(join16(repoPath, "bun.lockb")) || existsSync13(join16(repoPath, "bun.lock")),
|
|
18016
18758
|
preferred: "npm"
|
|
18017
18759
|
};
|
|
18018
18760
|
if (result.bun)
|
|
@@ -18026,7 +18768,7 @@ function detectPackageManager(repoPath) {
|
|
|
18026
18768
|
return result;
|
|
18027
18769
|
}
|
|
18028
18770
|
function detectDevScripts(repoPath) {
|
|
18029
|
-
const pkgPath =
|
|
18771
|
+
const pkgPath = join16(repoPath, "package.json");
|
|
18030
18772
|
if (!existsSync13(pkgPath)) {
|
|
18031
18773
|
return { dev: null, test: null, seed: null, build: null };
|
|
18032
18774
|
}
|
|
@@ -18053,7 +18795,7 @@ function findPlaywrightConfig(repoPath) {
|
|
|
18053
18795
|
"playwright-ct.config.js"
|
|
18054
18796
|
];
|
|
18055
18797
|
for (const name of candidates) {
|
|
18056
|
-
if (existsSync13(
|
|
18798
|
+
if (existsSync13(join16(repoPath, name)))
|
|
18057
18799
|
return name;
|
|
18058
18800
|
}
|
|
18059
18801
|
return null;
|
|
@@ -18062,7 +18804,7 @@ function extractTestGlobPatterns(configPath, repoPath) {
|
|
|
18062
18804
|
if (!configPath) {
|
|
18063
18805
|
return ["**/*.spec.ts", "**/*.spec.js", "**/*.test.ts", "**/*.test.js", "**/e2e/**/*.ts", "**/e2e/**/*.js", "**/tests/**/*.ts", "**/tests/**/*.js"];
|
|
18064
18806
|
}
|
|
18065
|
-
const fullPath =
|
|
18807
|
+
const fullPath = join16(repoPath, configPath);
|
|
18066
18808
|
let content;
|
|
18067
18809
|
try {
|
|
18068
18810
|
content = readFileSync5(fullPath, "utf-8");
|
|
@@ -18073,8 +18815,9 @@ function extractTestGlobPatterns(configPath, repoPath) {
|
|
|
18073
18815
|
const testDirMatch = content.match(/testDir\s*[:=]\s*['"`]([^'"`]+)['"`]/);
|
|
18074
18816
|
const testDir = testDirMatch?.[1];
|
|
18075
18817
|
const testMatchArray = content.match(/testMatch\s*[:=]\s*\[([^\]]+)\]/);
|
|
18076
|
-
|
|
18077
|
-
|
|
18818
|
+
const testMatchBody = testMatchArray?.[1];
|
|
18819
|
+
if (testMatchBody) {
|
|
18820
|
+
const items = testMatchBody.match(/['"`]([^'"`]+)['"`]/g);
|
|
18078
18821
|
if (items) {
|
|
18079
18822
|
for (const item of items) {
|
|
18080
18823
|
patterns.push(item.replace(/['"`]/g, ""));
|
|
@@ -18082,8 +18825,9 @@ function extractTestGlobPatterns(configPath, repoPath) {
|
|
|
18082
18825
|
}
|
|
18083
18826
|
}
|
|
18084
18827
|
const testMatchSingle = content.match(/testMatch\s*[:=]\s*['"`]([^'"`]+)['"`]/);
|
|
18085
|
-
|
|
18086
|
-
|
|
18828
|
+
const singleTestMatch = testMatchSingle?.[1];
|
|
18829
|
+
if (singleTestMatch) {
|
|
18830
|
+
patterns.push(singleTestMatch);
|
|
18087
18831
|
}
|
|
18088
18832
|
if (testDir && patterns.length === 0) {
|
|
18089
18833
|
patterns.push(`${testDir}/**/*.spec.ts`, `${testDir}/**/*.test.ts`, `${testDir}/**/*.spec.js`, `${testDir}/**/*.test.js`);
|
|
@@ -18109,7 +18853,7 @@ function findSpecFiles(repoPath, globPatterns) {
|
|
|
18109
18853
|
for (const pattern of globPatterns) {
|
|
18110
18854
|
const dirsToSearch = ["", ".", "tests", "e2e", "test", "__tests__", "specs", "src"];
|
|
18111
18855
|
for (const dir of dirsToSearch) {
|
|
18112
|
-
const searchDir = dir ?
|
|
18856
|
+
const searchDir = dir ? join16(repoPath, dir) : repoPath;
|
|
18113
18857
|
if (!existsSync13(searchDir))
|
|
18114
18858
|
continue;
|
|
18115
18859
|
try {
|
|
@@ -18143,7 +18887,7 @@ function walkDir(dir) {
|
|
|
18143
18887
|
try {
|
|
18144
18888
|
const entries = readdirSync3(dir, { withFileTypes: true });
|
|
18145
18889
|
for (const entry of entries) {
|
|
18146
|
-
const fullPath =
|
|
18890
|
+
const fullPath = join16(dir, entry.name);
|
|
18147
18891
|
if (entry.isDirectory()) {
|
|
18148
18892
|
if (entry.name === "node_modules" || entry.name === ".git")
|
|
18149
18893
|
continue;
|
|
@@ -18161,7 +18905,7 @@ function matchesGlob(filePath, pattern) {
|
|
|
18161
18905
|
return new RegExp(regex).test(filePath);
|
|
18162
18906
|
}
|
|
18163
18907
|
function detectSuggestedUrl(repoPath) {
|
|
18164
|
-
const pkgPath =
|
|
18908
|
+
const pkgPath = join16(repoPath, "package.json");
|
|
18165
18909
|
if (!existsSync13(pkgPath))
|
|
18166
18910
|
return null;
|
|
18167
18911
|
try {
|
|
@@ -18181,10 +18925,10 @@ function detectSuggestedUrl(repoPath) {
|
|
|
18181
18925
|
return null;
|
|
18182
18926
|
}
|
|
18183
18927
|
function checkPlaywrightBrowserInstalled(repoPath) {
|
|
18184
|
-
const cacheDir =
|
|
18928
|
+
const cacheDir = join16(repoPath, "node_modules", ".cache", "ms-playwright");
|
|
18185
18929
|
if (existsSync13(cacheDir))
|
|
18186
18930
|
return true;
|
|
18187
|
-
const globalCache =
|
|
18931
|
+
const globalCache = join16(repoPath, ".cache", "ms-playwright");
|
|
18188
18932
|
if (existsSync13(globalCache))
|
|
18189
18933
|
return true;
|
|
18190
18934
|
return false;
|
|
@@ -18201,7 +18945,7 @@ function getInstallCommand(pm) {
|
|
|
18201
18945
|
return "bun install";
|
|
18202
18946
|
}
|
|
18203
18947
|
}
|
|
18204
|
-
function getPlaywrightInstallCommand(
|
|
18948
|
+
function getPlaywrightInstallCommand(_pm) {
|
|
18205
18949
|
return "npx playwright install";
|
|
18206
18950
|
}
|
|
18207
18951
|
function discoverRepo(opts) {
|
|
@@ -18216,7 +18960,7 @@ function discoverRepo(opts) {
|
|
|
18216
18960
|
let configRaw = null;
|
|
18217
18961
|
if (configPath) {
|
|
18218
18962
|
try {
|
|
18219
|
-
configRaw = readFileSync5(
|
|
18963
|
+
configRaw = readFileSync5(join16(repoPath, configPath), "utf-8");
|
|
18220
18964
|
} catch {
|
|
18221
18965
|
configRaw = null;
|
|
18222
18966
|
}
|
|
@@ -18225,7 +18969,7 @@ function discoverRepo(opts) {
|
|
|
18225
18969
|
const specs = findSpecFiles(repoPath, globPatterns);
|
|
18226
18970
|
const packageManager = detectPackageManager(repoPath);
|
|
18227
18971
|
const devScripts = detectDevScripts(repoPath);
|
|
18228
|
-
const playwrightInstalled = existsSync13(
|
|
18972
|
+
const playwrightInstalled = existsSync13(join16(repoPath, "node_modules", "playwright")) || existsSync13(join16(repoPath, "node_modules", "@playwright", "test"));
|
|
18229
18973
|
const browsersInstalled = checkPlaywrightBrowserInstalled(repoPath);
|
|
18230
18974
|
const configExists = configPath !== null;
|
|
18231
18975
|
const specsFound = specs.length > 0;
|
|
@@ -18294,7 +19038,7 @@ function clearDiscoveryCache(repoPath) {
|
|
|
18294
19038
|
} else {
|
|
18295
19039
|
for (const file of readdirSync3(cacheDir)) {
|
|
18296
19040
|
if (file.endsWith(".json")) {
|
|
18297
|
-
unlinkSync(
|
|
19041
|
+
unlinkSync(join16(cacheDir, file));
|
|
18298
19042
|
}
|
|
18299
19043
|
}
|
|
18300
19044
|
}
|
|
@@ -18317,10 +19061,10 @@ init_runs();
|
|
|
18317
19061
|
init_database();
|
|
18318
19062
|
init_paths();
|
|
18319
19063
|
import { execSync as execSync2 } from "child_process";
|
|
18320
|
-
import { existsSync as existsSync14, mkdirSync as mkdirSync11, writeFileSync as
|
|
18321
|
-
import { join as
|
|
19064
|
+
import { existsSync as existsSync14, mkdirSync as mkdirSync11, writeFileSync as writeFileSync6 } from "fs";
|
|
19065
|
+
import { join as join17 } from "path";
|
|
18322
19066
|
function resolvePlaywrightCmd(repoPath) {
|
|
18323
|
-
const localPw =
|
|
19067
|
+
const localPw = join17(repoPath, "node_modules", ".bin", "playwright");
|
|
18324
19068
|
if (existsSync14(localPw)) {
|
|
18325
19069
|
return [localPw, "test"];
|
|
18326
19070
|
}
|
|
@@ -18339,7 +19083,7 @@ function buildPlaywrightArgs(specFiles, extraArgs = []) {
|
|
|
18339
19083
|
}
|
|
18340
19084
|
function runPlaywright(repoPath, workingDir, specFiles, extraArgs, timeoutMs) {
|
|
18341
19085
|
const cmd = resolvePlaywrightCmd(repoPath);
|
|
18342
|
-
const args = buildPlaywrightArgs(specFiles, extraArgs
|
|
19086
|
+
const args = buildPlaywrightArgs(specFiles, extraArgs);
|
|
18343
19087
|
const startTime = Date.now();
|
|
18344
19088
|
try {
|
|
18345
19089
|
const result = execSync2(`${cmd.join(" ")} ${args.join(" ")}`, {
|
|
@@ -18367,7 +19111,7 @@ function runPlaywright(repoPath, workingDir, specFiles, extraArgs, timeoutMs) {
|
|
|
18367
19111
|
};
|
|
18368
19112
|
}
|
|
18369
19113
|
}
|
|
18370
|
-
function parsePlaywrightJsonOutput(stdout,
|
|
19114
|
+
function parsePlaywrightJsonOutput(stdout, _stderr) {
|
|
18371
19115
|
const testResults = [];
|
|
18372
19116
|
try {
|
|
18373
19117
|
const obj = JSON.parse(stdout);
|
|
@@ -18472,19 +19216,21 @@ async function runRepoTests(opts) {
|
|
|
18472
19216
|
const workingDir = opts.snapshot.workingDir;
|
|
18473
19217
|
const repoPath = snapshot.repoPath;
|
|
18474
19218
|
const url = opts.url ?? snapshot.suggestedUrl ?? "http://localhost:3000";
|
|
18475
|
-
const
|
|
19219
|
+
const initialRun = createRun({
|
|
18476
19220
|
projectId: opts.projectId,
|
|
18477
19221
|
url,
|
|
18478
19222
|
model: opts.model ?? "repo-native",
|
|
18479
19223
|
headed: false,
|
|
18480
|
-
parallel: 1
|
|
18481
|
-
|
|
19224
|
+
parallel: 1
|
|
19225
|
+
});
|
|
19226
|
+
const run = updateRun(initialRun.id, {
|
|
19227
|
+
metadata: JSON.stringify({
|
|
18482
19228
|
runType: "repo-native",
|
|
18483
19229
|
repoPath,
|
|
18484
19230
|
configPath: snapshot.configPath,
|
|
18485
19231
|
cacheKey: snapshot.cacheKey,
|
|
18486
19232
|
label: opts.label
|
|
18487
|
-
}
|
|
19233
|
+
})
|
|
18488
19234
|
});
|
|
18489
19235
|
const specResults = [];
|
|
18490
19236
|
const startTime = Date.now();
|
|
@@ -18496,12 +19242,12 @@ async function runRepoTests(opts) {
|
|
|
18496
19242
|
specResults.push(result);
|
|
18497
19243
|
const resultId = uuid();
|
|
18498
19244
|
const timestamp = now();
|
|
18499
|
-
const
|
|
18500
|
-
|
|
19245
|
+
const db3 = getDatabase();
|
|
19246
|
+
db3.exec("PRAGMA foreign_keys = OFF");
|
|
18501
19247
|
try {
|
|
18502
19248
|
const reasoning = result.status === "passed" ? "All tests passed" : (result.error ?? "").slice(0, 500) || null;
|
|
18503
19249
|
const errorStr = result.status !== "passed" ? result.error ?? null : null;
|
|
18504
|
-
|
|
19250
|
+
db3.query(`
|
|
18505
19251
|
INSERT INTO results (id, run_id, scenario_id, status, reasoning, error, steps_completed, steps_total, duration_ms, model, tokens_used, cost_cents, metadata, created_at, persona_id, persona_name)
|
|
18506
19252
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 0, 0, ?, ?, NULL, NULL)
|
|
18507
19253
|
`).run(resultId, run.id, "__repo__", result.status, reasoning, errorStr, result.testResults.filter((t) => t.status === "passed").length, result.testResults.length || 1, result.durationMs, "repo-native", JSON.stringify({
|
|
@@ -18510,14 +19256,14 @@ async function runRepoTests(opts) {
|
|
|
18510
19256
|
testResults: result.testResults
|
|
18511
19257
|
}), timestamp);
|
|
18512
19258
|
} finally {
|
|
18513
|
-
|
|
19259
|
+
db3.exec("PRAGMA foreign_keys = ON");
|
|
18514
19260
|
}
|
|
18515
19261
|
const resultRecord = { id: resultId };
|
|
18516
19262
|
if (result.stdout || result.stderr) {
|
|
18517
|
-
const reportersDir =
|
|
19263
|
+
const reportersDir = join17(getTestersDir(), "repo-run-output");
|
|
18518
19264
|
mkdirSync11(reportersDir, { recursive: true });
|
|
18519
|
-
const outputFile =
|
|
18520
|
-
|
|
19265
|
+
const outputFile = join17(reportersDir, `${resultRecord.id}.log`);
|
|
19266
|
+
writeFileSync6(outputFile, `=== stdout ===
|
|
18521
19267
|
${result.stdout}
|
|
18522
19268
|
|
|
18523
19269
|
=== stderr ===
|
|
@@ -19122,46 +19868,46 @@ async function postGitHubComment(run, results, options) {
|
|
|
19122
19868
|
// src/db/sessions.ts
|
|
19123
19869
|
init_database();
|
|
19124
19870
|
function createSession(input) {
|
|
19125
|
-
const
|
|
19871
|
+
const db3 = getDatabase();
|
|
19126
19872
|
const id = input.sessionId ?? uuid();
|
|
19127
19873
|
const timestamp = now();
|
|
19128
|
-
|
|
19874
|
+
db3.query(`
|
|
19129
19875
|
INSERT INTO sessions (id, tab_id, url, title, entries, entry_count, error_count, console_count, nav_count, status, start_time, end_time, created_at)
|
|
19130
19876
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
19131
19877
|
`).run(id, input.tabId, input.url ?? null, input.title ?? null, input.entries, input.entryCount, input.errorCount ?? 0, input.consoleCount ?? 0, input.navCount ?? 0, input.status, input.startTime, input.endTime ?? null, timestamp);
|
|
19132
19878
|
return getSession(id);
|
|
19133
19879
|
}
|
|
19134
19880
|
function getSession(id) {
|
|
19135
|
-
const
|
|
19136
|
-
let row =
|
|
19881
|
+
const db3 = getDatabase();
|
|
19882
|
+
let row = db3.query("SELECT * FROM sessions WHERE id = ?").get(id);
|
|
19137
19883
|
if (row)
|
|
19138
19884
|
return sessionFromRow(row);
|
|
19139
19885
|
const fullId = resolvePartialId("sessions", id);
|
|
19140
19886
|
if (fullId) {
|
|
19141
|
-
row =
|
|
19887
|
+
row = db3.query("SELECT * FROM sessions WHERE id = ?").get(fullId);
|
|
19142
19888
|
if (row)
|
|
19143
19889
|
return sessionFromRow(row);
|
|
19144
19890
|
}
|
|
19145
19891
|
return null;
|
|
19146
19892
|
}
|
|
19147
19893
|
function listSessions(limit = 50, offset = 0) {
|
|
19148
|
-
const
|
|
19149
|
-
const rows =
|
|
19894
|
+
const db3 = getDatabase();
|
|
19895
|
+
const rows = db3.query("SELECT * FROM sessions ORDER BY created_at DESC LIMIT ? OFFSET ?").all(limit, offset);
|
|
19150
19896
|
return rows.map(sessionFromRow);
|
|
19151
19897
|
}
|
|
19152
19898
|
function deleteSession(id) {
|
|
19153
|
-
const
|
|
19154
|
-
const result =
|
|
19899
|
+
const db3 = getDatabase();
|
|
19900
|
+
const result = db3.query("DELETE FROM sessions WHERE id = ?").run(id);
|
|
19155
19901
|
return result.changes > 0;
|
|
19156
19902
|
}
|
|
19157
19903
|
function searchSessions(query, limit = 20) {
|
|
19158
|
-
const
|
|
19159
|
-
const rows =
|
|
19904
|
+
const db3 = getDatabase();
|
|
19905
|
+
const rows = db3.query("SELECT * FROM sessions WHERE url LIKE ? OR title LIKE ? ORDER BY created_at DESC LIMIT ?").all(`%${query}%`, `%${query}%`, limit);
|
|
19160
19906
|
return rows.map(sessionFromRow);
|
|
19161
19907
|
}
|
|
19162
19908
|
function countSessions() {
|
|
19163
|
-
const
|
|
19164
|
-
const row =
|
|
19909
|
+
const db3 = getDatabase();
|
|
19910
|
+
const row = db3.query("SELECT COUNT(*) as count FROM sessions").get();
|
|
19165
19911
|
return row.count;
|
|
19166
19912
|
}
|
|
19167
19913
|
function sessionFromRow(row) {
|
|
@@ -19185,6 +19931,7 @@ export {
|
|
|
19185
19931
|
writeScenarioMeta,
|
|
19186
19932
|
writeRunMeta,
|
|
19187
19933
|
uuid,
|
|
19934
|
+
updateTestingWorkflow,
|
|
19188
19935
|
updateSchedule,
|
|
19189
19936
|
updateScenario,
|
|
19190
19937
|
updateRun,
|
|
@@ -19202,6 +19949,7 @@ export {
|
|
|
19202
19949
|
screenshotFromRow,
|
|
19203
19950
|
scheduleFromRow,
|
|
19204
19951
|
scenarioFromRow,
|
|
19952
|
+
runTestingWorkflow,
|
|
19205
19953
|
runSmoke,
|
|
19206
19954
|
runSingleScenario,
|
|
19207
19955
|
runRepoTests,
|
|
@@ -19233,6 +19981,7 @@ export {
|
|
|
19233
19981
|
loginWithAuthConfig,
|
|
19234
19982
|
loadConfig,
|
|
19235
19983
|
listWebhooks,
|
|
19984
|
+
listTestingWorkflows,
|
|
19236
19985
|
listTemplateNames,
|
|
19237
19986
|
listSessions,
|
|
19238
19987
|
listScreenshots,
|
|
@@ -19255,6 +20004,7 @@ export {
|
|
|
19255
20004
|
imageToBase64,
|
|
19256
20005
|
getWebhook,
|
|
19257
20006
|
getTransitiveDependencies,
|
|
20007
|
+
getTestingWorkflow,
|
|
19258
20008
|
getTemplate,
|
|
19259
20009
|
getStarterScenarios,
|
|
19260
20010
|
getSession,
|
|
@@ -19311,13 +20061,16 @@ export {
|
|
|
19311
20061
|
diffRuns,
|
|
19312
20062
|
detectFramework,
|
|
19313
20063
|
deleteWebhook,
|
|
20064
|
+
deleteTestingWorkflow,
|
|
19314
20065
|
deleteSession,
|
|
19315
20066
|
deleteSchedule,
|
|
19316
20067
|
deleteScenario,
|
|
19317
20068
|
deleteRun,
|
|
19318
20069
|
deleteFlow,
|
|
19319
20070
|
deleteAuthPreset,
|
|
20071
|
+
createWorkflowDatabaseBundle,
|
|
19320
20072
|
createWebhook,
|
|
20073
|
+
createTestingWorkflow,
|
|
19321
20074
|
createSession,
|
|
19322
20075
|
createScreenshot,
|
|
19323
20076
|
createSchedule,
|
|
@@ -19336,6 +20089,7 @@ export {
|
|
|
19336
20089
|
closeBrowser,
|
|
19337
20090
|
clearDiscoveryCache,
|
|
19338
20091
|
checkBudget,
|
|
20092
|
+
buildWorkflowRunPlan,
|
|
19339
20093
|
agentFromRow,
|
|
19340
20094
|
addDependency,
|
|
19341
20095
|
VersionConflictError,
|