@hasna/testers 0.0.10 → 0.0.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dashboard/dist/assets/{index-DvYdwJK-.css → index-DyXKnBM8.css} +1 -1
- package/dashboard/dist/assets/index-jNG_Nd_Q.js +49 -0
- package/dashboard/dist/index.html +2 -2
- package/dist/cli/index.js +254 -15
- package/dist/db/runs.d.ts.map +1 -1
- package/dist/db/scenarios.d.ts +4 -0
- package/dist/db/scenarios.d.ts.map +1 -1
- package/dist/index.js +45 -4
- package/dist/lib/costs.d.ts +12 -0
- package/dist/lib/costs.d.ts.map +1 -1
- package/dist/lib/reporter.d.ts +2 -1
- package/dist/lib/reporter.d.ts.map +1 -1
- package/dist/lib/runner.d.ts +4 -1
- package/dist/lib/runner.d.ts.map +1 -1
- package/dist/mcp/index.js +375 -170
- package/dist/server/index.js +85 -3
- package/dist/types/index.d.ts +5 -0
- package/dist/types/index.d.ts.map +1 -1
- package/package.json +1 -1
- package/dashboard/dist/assets/index-RV9LMdfY.js +0 -49
|
@@ -4,8 +4,8 @@
|
|
|
4
4
|
<meta charset="UTF-8" />
|
|
5
5
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
|
6
6
|
<title>Testers Dashboard</title>
|
|
7
|
-
<script type="module" crossorigin src="/assets/index-
|
|
8
|
-
<link rel="stylesheet" crossorigin href="/assets/index-
|
|
7
|
+
<script type="module" crossorigin src="/assets/index-jNG_Nd_Q.js"></script>
|
|
8
|
+
<link rel="stylesheet" crossorigin href="/assets/index-DyXKnBM8.css">
|
|
9
9
|
</head>
|
|
10
10
|
<body>
|
|
11
11
|
<div id="root"></div>
|
package/dist/cli/index.js
CHANGED
|
@@ -2560,7 +2560,10 @@ function listScenarios(filter) {
|
|
|
2560
2560
|
if (conditions.length > 0) {
|
|
2561
2561
|
sql += " WHERE " + conditions.join(" AND ");
|
|
2562
2562
|
}
|
|
2563
|
-
|
|
2563
|
+
const sortField = filter?.sort ?? "date";
|
|
2564
|
+
const sortDir = filter?.desc === false ? "ASC" : "DESC";
|
|
2565
|
+
const orderByCol = sortField === "name" ? "name" : sortField === "priority" ? "CASE priority WHEN 'critical' THEN 0 WHEN 'high' THEN 1 WHEN 'medium' THEN 2 WHEN 'low' THEN 3 ELSE 4 END" : "created_at";
|
|
2566
|
+
sql += ` ORDER BY ${orderByCol} ${sortDir}`;
|
|
2564
2567
|
if (filter?.limit) {
|
|
2565
2568
|
sql += " LIMIT ?";
|
|
2566
2569
|
params.push(filter.limit);
|
|
@@ -2708,7 +2711,10 @@ function listRuns(filter) {
|
|
|
2708
2711
|
if (conditions.length > 0) {
|
|
2709
2712
|
sql += " WHERE " + conditions.join(" AND ");
|
|
2710
2713
|
}
|
|
2711
|
-
|
|
2714
|
+
const sortField = filter?.sort ?? "date";
|
|
2715
|
+
const sortDir = filter?.desc === false ? "ASC" : "DESC";
|
|
2716
|
+
const orderByCol = sortField === "duration" ? "(CASE WHEN finished_at IS NULL THEN NULL ELSE (julianday(finished_at) - julianday(started_at)) * 86400000 END)" : sortField === "cost" ? "(SELECT COALESCE(SUM(cost_cents), 0) FROM results WHERE run_id = runs.id)" : "started_at";
|
|
2717
|
+
sql += ` ORDER BY ${orderByCol} ${sortDir}`;
|
|
2712
2718
|
if (filter?.limit) {
|
|
2713
2719
|
sql += " LIMIT ?";
|
|
2714
2720
|
params.push(filter.limit);
|
|
@@ -3407,7 +3413,7 @@ import chalk5 from "chalk";
|
|
|
3407
3413
|
// package.json
|
|
3408
3414
|
var package_default = {
|
|
3409
3415
|
name: "@hasna/testers",
|
|
3410
|
-
version: "0.0.
|
|
3416
|
+
version: "0.0.12",
|
|
3411
3417
|
description: "AI-powered QA testing CLI \u2014 spawns cheap AI agents to test web apps with headless browsers",
|
|
3412
3418
|
type: "module",
|
|
3413
3419
|
main: "dist/index.js",
|
|
@@ -4741,14 +4747,26 @@ function emit(event) {
|
|
|
4741
4747
|
}
|
|
4742
4748
|
function withTimeout(promise, ms, label) {
|
|
4743
4749
|
return new Promise((resolve, reject) => {
|
|
4750
|
+
const warningAt = Math.floor(ms * 0.8);
|
|
4751
|
+
const warningTimer = setTimeout(() => {
|
|
4752
|
+
emit({
|
|
4753
|
+
type: "scenario:timeout_warning",
|
|
4754
|
+
scenarioName: label,
|
|
4755
|
+
timeoutMs: ms,
|
|
4756
|
+
elapsedMs: warningAt
|
|
4757
|
+
});
|
|
4758
|
+
}, warningAt);
|
|
4744
4759
|
const timer = setTimeout(() => {
|
|
4760
|
+
clearTimeout(warningTimer);
|
|
4745
4761
|
reject(new Error(`Scenario '${label}' timed out after ${ms}ms. Try: testers run --timeout ${ms * 2} or simplify the scenario steps.`));
|
|
4746
4762
|
}, ms);
|
|
4747
4763
|
promise.then((val) => {
|
|
4748
4764
|
clearTimeout(timer);
|
|
4765
|
+
clearTimeout(warningTimer);
|
|
4749
4766
|
resolve(val);
|
|
4750
4767
|
}, (err) => {
|
|
4751
4768
|
clearTimeout(timer);
|
|
4769
|
+
clearTimeout(warningTimer);
|
|
4752
4770
|
reject(err);
|
|
4753
4771
|
});
|
|
4754
4772
|
});
|
|
@@ -4777,6 +4795,7 @@ async function runSingleScenario(scenario, runId, options) {
|
|
|
4777
4795
|
const targetUrl = scenario.targetPath ? `${options.url.replace(/\/$/, "")}${scenario.targetPath}` : options.url;
|
|
4778
4796
|
const scenarioTimeout = scenario.timeoutMs ?? options.timeout ?? config.browser.timeout ?? 60000;
|
|
4779
4797
|
await page.goto(targetUrl, { timeout: Math.min(scenarioTimeout, 30000) });
|
|
4798
|
+
const stepStartTimes = new Map;
|
|
4780
4799
|
const agentResult = await withTimeout(runAgentLoop({
|
|
4781
4800
|
client,
|
|
4782
4801
|
page,
|
|
@@ -4786,6 +4805,16 @@ async function runSingleScenario(scenario, runId, options) {
|
|
|
4786
4805
|
runId,
|
|
4787
4806
|
maxTurns: 30,
|
|
4788
4807
|
onStep: (stepEvent) => {
|
|
4808
|
+
let stepDurationMs;
|
|
4809
|
+
if (stepEvent.type === "tool_call") {
|
|
4810
|
+
stepStartTimes.set(stepEvent.stepNumber, Date.now());
|
|
4811
|
+
} else if (stepEvent.type === "tool_result") {
|
|
4812
|
+
const startTime = stepStartTimes.get(stepEvent.stepNumber);
|
|
4813
|
+
if (startTime !== undefined) {
|
|
4814
|
+
stepDurationMs = Date.now() - startTime;
|
|
4815
|
+
stepStartTimes.delete(stepEvent.stepNumber);
|
|
4816
|
+
}
|
|
4817
|
+
}
|
|
4789
4818
|
emit({
|
|
4790
4819
|
type: `step:${stepEvent.type}`,
|
|
4791
4820
|
scenarioId: scenario.id,
|
|
@@ -4795,7 +4824,8 @@ async function runSingleScenario(scenario, runId, options) {
|
|
|
4795
4824
|
toolInput: stepEvent.toolInput,
|
|
4796
4825
|
toolResult: stepEvent.toolResult,
|
|
4797
4826
|
thinking: stepEvent.thinking,
|
|
4798
|
-
stepNumber: stepEvent.stepNumber
|
|
4827
|
+
stepNumber: stepEvent.stepNumber,
|
|
4828
|
+
stepDurationMs
|
|
4799
4829
|
});
|
|
4800
4830
|
}
|
|
4801
4831
|
}), scenarioTimeout, scenario.name);
|
|
@@ -5062,13 +5092,24 @@ init_database();
|
|
|
5062
5092
|
function useEmoji() {
|
|
5063
5093
|
return !process.env["NO_COLOR"] && process.argv.indexOf("--no-color") === -1;
|
|
5064
5094
|
}
|
|
5065
|
-
function formatTerminal(run, results) {
|
|
5095
|
+
function formatTerminal(run, results, options) {
|
|
5066
5096
|
const lines = [];
|
|
5097
|
+
const failedOnly = options?.failedOnly ?? false;
|
|
5067
5098
|
lines.push("");
|
|
5068
5099
|
lines.push(chalk.bold(` Run ${run.id.slice(0, 8)} \u2014 ${run.url}`));
|
|
5069
5100
|
lines.push(chalk.dim(` Model: ${run.model} | Parallel: ${run.parallel} | Headed: ${run.headed ? "yes" : "no"}`));
|
|
5070
5101
|
lines.push("");
|
|
5102
|
+
if (failedOnly) {
|
|
5103
|
+
const passedCount = results.filter((r) => r.status === "passed").length;
|
|
5104
|
+
if (passedCount > 0) {
|
|
5105
|
+
lines.push(chalk.dim(` (${passedCount} passed scenario${passedCount !== 1 ? "s" : ""} hidden \u2014 use without --failed-only to see all)`));
|
|
5106
|
+
lines.push("");
|
|
5107
|
+
}
|
|
5108
|
+
}
|
|
5071
5109
|
for (const result of results) {
|
|
5110
|
+
if (failedOnly && result.status !== "failed" && result.status !== "error") {
|
|
5111
|
+
continue;
|
|
5112
|
+
}
|
|
5072
5113
|
const scenario = getScenario(result.scenarioId);
|
|
5073
5114
|
const name = scenario ? `${scenario.shortId}: ${scenario.name}` : result.scenarioId.slice(0, 8);
|
|
5074
5115
|
const screenshots = listScreenshots(result.id);
|
|
@@ -6438,6 +6479,53 @@ function getCostSummary(options) {
|
|
|
6438
6479
|
estimatedMonthlyCents
|
|
6439
6480
|
};
|
|
6440
6481
|
}
|
|
6482
|
+
function getCostsByScenario(options) {
|
|
6483
|
+
const db2 = getDatabase();
|
|
6484
|
+
const period = options?.period ?? "month";
|
|
6485
|
+
const projectId = options?.projectId;
|
|
6486
|
+
const dateFilter = getDateFilter(period);
|
|
6487
|
+
const projectFilter = projectId ? "AND ru.project_id = ?" : "";
|
|
6488
|
+
const projectParams = projectId ? [projectId] : [];
|
|
6489
|
+
const rows = db2.query(`SELECT
|
|
6490
|
+
r.scenario_id,
|
|
6491
|
+
COALESCE(s.name, r.scenario_id) as name,
|
|
6492
|
+
COUNT(DISTINCT r.run_id) as run_count,
|
|
6493
|
+
COALESCE(SUM(r.cost_cents), 0) as total_cost_cents
|
|
6494
|
+
FROM results r
|
|
6495
|
+
JOIN runs ru ON r.run_id = ru.id
|
|
6496
|
+
LEFT JOIN scenarios s ON r.scenario_id = s.id
|
|
6497
|
+
WHERE 1=1 ${dateFilter} ${projectFilter}
|
|
6498
|
+
GROUP BY r.scenario_id
|
|
6499
|
+
ORDER BY total_cost_cents DESC`).all(...projectParams);
|
|
6500
|
+
return rows.map((row) => ({
|
|
6501
|
+
scenarioId: row.scenario_id,
|
|
6502
|
+
name: row.name,
|
|
6503
|
+
runCount: row.run_count,
|
|
6504
|
+
totalCostCents: row.total_cost_cents,
|
|
6505
|
+
avgCostPerRunCents: row.run_count > 0 ? row.total_cost_cents / row.run_count : 0
|
|
6506
|
+
}));
|
|
6507
|
+
}
|
|
6508
|
+
function formatCostsByScenarioTerminal(rows, period) {
|
|
6509
|
+
const lines = [];
|
|
6510
|
+
lines.push("");
|
|
6511
|
+
lines.push(chalk4.bold(` Cost by Scenario (${period})`));
|
|
6512
|
+
lines.push("");
|
|
6513
|
+
if (rows.length === 0) {
|
|
6514
|
+
lines.push(chalk4.dim(" No cost data found."));
|
|
6515
|
+
lines.push("");
|
|
6516
|
+
return lines.join(`
|
|
6517
|
+
`);
|
|
6518
|
+
}
|
|
6519
|
+
lines.push(` ${"Scenario".padEnd(40)} ${"Runs".padEnd(8)} ${"Total Cost".padEnd(14)} Avg/Run`);
|
|
6520
|
+
lines.push(` ${"\u2500".repeat(40)} ${"\u2500".repeat(8)} ${"\u2500".repeat(14)} ${"\u2500".repeat(10)}`);
|
|
6521
|
+
for (const row of rows) {
|
|
6522
|
+
const label = row.name.length > 38 ? row.name.slice(0, 35) + "..." : row.name;
|
|
6523
|
+
lines.push(` ${label.padEnd(40)} ${String(row.runCount).padEnd(8)} ${formatDollars(row.totalCostCents).padEnd(14)} ${formatDollars(row.avgCostPerRunCents)}`);
|
|
6524
|
+
}
|
|
6525
|
+
lines.push("");
|
|
6526
|
+
return lines.join(`
|
|
6527
|
+
`);
|
|
6528
|
+
}
|
|
6441
6529
|
function checkBudget(estimatedCostCents) {
|
|
6442
6530
|
const budget = loadBudgetConfig();
|
|
6443
6531
|
if (estimatedCostCents > budget.maxPerRunCents) {
|
|
@@ -7188,12 +7276,15 @@ program2.command("add [name]").alias("create").description("Create a new test sc
|
|
|
7188
7276
|
process.exit(1);
|
|
7189
7277
|
}
|
|
7190
7278
|
});
|
|
7191
|
-
program2.command("list").alias("ls").description("List test scenarios").option("-t, --tag <tag>", "Filter by tag").option("-p, --priority <level>", "Filter by priority").option("--project <id>", "Filter by project ID").option("-l, --limit <n>", "Limit results", "50").option("--offset <n>", "Skip first N results", "0").option("--json", "Output as JSON", false).action((opts) => {
|
|
7279
|
+
program2.command("list").alias("ls").description("List test scenarios").option("-t, --tag <tag>", "Filter by tag").option("-p, --priority <level>", "Filter by priority").option("--project <id>", "Filter by project ID").option("--search <text>", "Filter by name or description (case-insensitive substring match)").option("--sort <field>", "Sort field: date, priority, name (default: date)").option("--asc", "Sort ascending instead of descending", false).option("-l, --limit <n>", "Limit results", "50").option("--offset <n>", "Skip first N results", "0").option("--json", "Output as JSON", false).action((opts) => {
|
|
7192
7280
|
try {
|
|
7193
7281
|
const scenarios = listScenarios({
|
|
7194
7282
|
tags: opts.tag ? [opts.tag] : undefined,
|
|
7195
7283
|
priority: opts.priority,
|
|
7196
7284
|
projectId: opts.project,
|
|
7285
|
+
search: opts.search,
|
|
7286
|
+
sort: opts.sort,
|
|
7287
|
+
desc: !opts.asc,
|
|
7197
7288
|
limit: parseInt(opts.limit, 10),
|
|
7198
7289
|
offset: parseInt(opts.offset, 10) || undefined
|
|
7199
7290
|
});
|
|
@@ -7248,7 +7339,13 @@ program2.command("show <id>").description("Show scenario details").option("--jso
|
|
|
7248
7339
|
program2.command("update <id>").description("Update a scenario").option("-n, --name <name>", "New name").option("-d, --description <text>", "New description").option("-s, --steps <step>", "Replace steps (repeatable)", (val, acc) => {
|
|
7249
7340
|
acc.push(val);
|
|
7250
7341
|
return acc;
|
|
7251
|
-
}, []).option("-t, --tag <tag>", "Replace tags (repeatable)", (val, acc) => {
|
|
7342
|
+
}, []).option("-t, --tag <tag>", "Replace all tags (repeatable)", (val, acc) => {
|
|
7343
|
+
acc.push(val);
|
|
7344
|
+
return acc;
|
|
7345
|
+
}, []).option("--tag-add <tag>", "Add a tag to existing tags (repeatable)", (val, acc) => {
|
|
7346
|
+
acc.push(val);
|
|
7347
|
+
return acc;
|
|
7348
|
+
}, []).option("--tag-remove <tag>", "Remove a tag from existing tags (repeatable)", (val, acc) => {
|
|
7252
7349
|
acc.push(val);
|
|
7253
7350
|
return acc;
|
|
7254
7351
|
}, []).option("-p, --priority <level>", "New priority").option("-m, --model <model>", "New model").action((id, opts) => {
|
|
@@ -7258,15 +7355,29 @@ program2.command("update <id>").description("Update a scenario").option("-n, --n
|
|
|
7258
7355
|
logError(chalk5.red(`Scenario not found: ${id}`));
|
|
7259
7356
|
process.exit(1);
|
|
7260
7357
|
}
|
|
7358
|
+
let newTags;
|
|
7359
|
+
if (opts.tag.length > 0) {
|
|
7360
|
+
newTags = opts.tag;
|
|
7361
|
+
} else if (opts.tagAdd.length > 0 || opts.tagRemove.length > 0) {
|
|
7362
|
+
const existing = new Set(scenario.tags);
|
|
7363
|
+
for (const t of opts.tagAdd)
|
|
7364
|
+
existing.add(t);
|
|
7365
|
+
for (const t of opts.tagRemove)
|
|
7366
|
+
existing.delete(t);
|
|
7367
|
+
newTags = [...existing];
|
|
7368
|
+
}
|
|
7261
7369
|
const updated = updateScenario(scenario.id, {
|
|
7262
7370
|
name: opts.name,
|
|
7263
7371
|
description: opts.description,
|
|
7264
7372
|
steps: opts.steps.length > 0 ? opts.steps : undefined,
|
|
7265
|
-
tags:
|
|
7373
|
+
tags: newTags,
|
|
7266
7374
|
priority: opts.priority,
|
|
7267
7375
|
model: opts.model
|
|
7268
7376
|
}, scenario.version);
|
|
7269
7377
|
log(chalk5.green(`Updated scenario ${chalk5.bold(updated.shortId)}: ${updated.name}`));
|
|
7378
|
+
if (newTags !== undefined) {
|
|
7379
|
+
log(chalk5.dim(` Tags: [${updated.tags.join(", ")}]`));
|
|
7380
|
+
}
|
|
7270
7381
|
} catch (error) {
|
|
7271
7382
|
logError(chalk5.red(`Error: ${error instanceof Error ? error.message : String(error)}`));
|
|
7272
7383
|
process.exit(1);
|
|
@@ -7314,7 +7425,7 @@ program2.command("delete <id>").description("Delete a scenario").option("-y, --y
|
|
|
7314
7425
|
program2.command("run [url] [description]").alias("test").description("Run test scenarios against a URL").option("-t, --tag <tag>", "Filter by tag (repeatable)", (val, acc) => {
|
|
7315
7426
|
acc.push(val);
|
|
7316
7427
|
return acc;
|
|
7317
|
-
}, []).option("-s, --scenario <id>", "Run specific scenario ID").option("-p, --priority <level>", "Filter by priority").option("--headed", "Run browser in headed mode", false).option("-m, --model <model>", "AI model to use").option("--parallel <n>", "Number of parallel browsers", "1").option("--json", "Output results as JSON", false).option("-o, --output <filepath>", "Write JSON results to file").option("--timeout <ms>", "Timeout in milliseconds").option("--from-todos", "Import scenarios from todos before running", false).option("--project <id>", "Project ID").option("-b, --background", "Start run in background and return immediately", false).option("--browser <engine>", "Browser engine: playwright or lightpanda", "playwright").option("--env <name>", "Use a named environment for the URL").option("--dry-run", "Print what would run without launching browser", false).option("--retry <n>", "Retry failed scenarios up to n times", "0").action(async (urlArg, description, opts) => {
|
|
7428
|
+
}, []).option("-s, --scenario <id>", "Run specific scenario ID").option("-p, --priority <level>", "Filter by priority").option("--headed", "Run browser in headed mode", false).option("-m, --model <model>", "AI model to use").option("--parallel <n>", "Number of parallel browsers", "1").option("--json", "Output results as JSON", false).option("-o, --output <filepath>", "Write JSON results to file").option("--timeout <ms>", "Timeout in milliseconds").option("--from-todos", "Import scenarios from todos before running", false).option("--project <id>", "Project ID").option("-b, --background", "Start run in background and return immediately", false).option("--browser <engine>", "Browser engine: playwright or lightpanda", "playwright").option("--env <name>", "Use a named environment for the URL").option("--dry-run", "Print what would run without launching browser", false).option("--retry <n>", "Retry failed scenarios up to n times", "0").option("--verbose", "Show per-step timing and full tool results", false).option("--watch-results", "When used with --background, poll and display live results table until run completes", false).option("--failed-only", "Only show failed/error scenarios in output (passed count shown as summary)", false).action(async (urlArg, description, opts) => {
|
|
7318
7429
|
try {
|
|
7319
7430
|
const projectId = resolveProject(opts.project);
|
|
7320
7431
|
let url = urlArg;
|
|
@@ -7420,10 +7531,53 @@ program2.command("run [url] [description]").alias("test").description("Run test
|
|
|
7420
7531
|
log(chalk5.green(`Run started in background: ${chalk5.bold(runId.slice(0, 8))}`));
|
|
7421
7532
|
log(chalk5.dim(` Scenarios: ${scenarioCount}`));
|
|
7422
7533
|
log(chalk5.dim(` URL: ${url}`));
|
|
7534
|
+
if (opts.watchResults) {
|
|
7535
|
+
log(chalk5.dim(` Watching results (polling every 3s)...`));
|
|
7536
|
+
log("");
|
|
7537
|
+
const POLL_INTERVAL = 3000;
|
|
7538
|
+
const DONE_STATUSES = new Set(["passed", "failed", "cancelled"]);
|
|
7539
|
+
const renderTable = () => {
|
|
7540
|
+
const run2 = getRun(runId);
|
|
7541
|
+
if (!run2)
|
|
7542
|
+
return;
|
|
7543
|
+
const results2 = getResultsByRun(runId);
|
|
7544
|
+
const statusIcon = run2.status === "passed" ? chalk5.green("PASS") : run2.status === "failed" ? chalk5.red("FAIL") : chalk5.blue("RUN ");
|
|
7545
|
+
process.stdout.write(`\r ${statusIcon} ${run2.passed} passed ${run2.failed} failed ${run2.total - run2.passed - run2.failed} running (${results2.length}/${run2.total})
|
|
7546
|
+
`);
|
|
7547
|
+
for (const r of results2) {
|
|
7548
|
+
const scenario = getScenario(r.scenarioId);
|
|
7549
|
+
const name = scenario ? scenario.name : r.scenarioId.slice(0, 8);
|
|
7550
|
+
const icon = r.status === "passed" ? chalk5.green("\u2713") : r.status === "failed" ? chalk5.red("\u2717") : r.status === "error" ? chalk5.yellow("!") : chalk5.blue("\u2026");
|
|
7551
|
+
const dur = r.durationMs > 0 ? chalk5.dim(` ${(r.durationMs / 1000).toFixed(1)}s`) : "";
|
|
7552
|
+
process.stdout.write(` ${icon} ${name}${dur}
|
|
7553
|
+
`);
|
|
7554
|
+
}
|
|
7555
|
+
};
|
|
7556
|
+
await new Promise((resolve2) => {
|
|
7557
|
+
const poll = setInterval(() => {
|
|
7558
|
+
const run2 = getRun(runId);
|
|
7559
|
+
if (!run2)
|
|
7560
|
+
return;
|
|
7561
|
+
renderTable();
|
|
7562
|
+
if (DONE_STATUSES.has(run2.status)) {
|
|
7563
|
+
clearInterval(poll);
|
|
7564
|
+
resolve2();
|
|
7565
|
+
}
|
|
7566
|
+
}, POLL_INTERVAL);
|
|
7567
|
+
});
|
|
7568
|
+
const finalRun = getRun(runId);
|
|
7569
|
+
if (finalRun) {
|
|
7570
|
+
log("");
|
|
7571
|
+
const results2 = getResultsByRun(runId);
|
|
7572
|
+
log(formatTerminal(finalRun, results2));
|
|
7573
|
+
}
|
|
7574
|
+
process.exit(finalRun ? getExitCode(finalRun) : 0);
|
|
7575
|
+
}
|
|
7423
7576
|
log(chalk5.dim(` Check progress: testers results ${runId.slice(0, 8)}`));
|
|
7424
7577
|
process.exit(0);
|
|
7425
7578
|
}
|
|
7426
7579
|
if (!opts.json && !opts.output) {
|
|
7580
|
+
const verbose = !!opts.verbose;
|
|
7427
7581
|
onRunEvent((event) => {
|
|
7428
7582
|
switch (event.type) {
|
|
7429
7583
|
case "scenario:start":
|
|
@@ -7433,6 +7587,12 @@ program2.command("run [url] [description]").alias("test").description("Run test
|
|
|
7433
7587
|
log(chalk5.blue(` [start] ${event.scenarioName ?? event.scenarioId}`));
|
|
7434
7588
|
}
|
|
7435
7589
|
break;
|
|
7590
|
+
case "scenario:timeout_warning": {
|
|
7591
|
+
const elapsedS = ((event.elapsedMs ?? 0) / 1000).toFixed(0);
|
|
7592
|
+
const totalS = ((event.timeoutMs ?? 0) / 1000).toFixed(0);
|
|
7593
|
+
log(chalk5.yellow(` \u26A0\uFE0F Scenario '${event.scenarioName}' at 80% timeout (${elapsedS}s/${totalS}s) \u2014 still running`));
|
|
7594
|
+
break;
|
|
7595
|
+
}
|
|
7436
7596
|
case "step:thinking":
|
|
7437
7597
|
if (event.thinking) {
|
|
7438
7598
|
const preview = event.thinking.length > 120 ? event.thinking.slice(0, 120) + "..." : event.thinking;
|
|
@@ -7446,8 +7606,9 @@ program2.command("run [url] [description]").alias("test").description("Run test
|
|
|
7446
7606
|
if (event.toolName === "report_result") {
|
|
7447
7607
|
log(chalk5.bold(` [result] ${event.toolResult}`));
|
|
7448
7608
|
} else {
|
|
7609
|
+
const durationStr = verbose && event.stepDurationMs !== undefined ? chalk5.dim(`[${(event.stepDurationMs / 1000).toFixed(1)}s] `) : "";
|
|
7449
7610
|
const resultPreview = (event.toolResult ?? "").length > 100 ? (event.toolResult ?? "").slice(0, 100) + "..." : event.toolResult ?? "";
|
|
7450
|
-
log(chalk5.dim(` [done] ${resultPreview}`));
|
|
7611
|
+
log(chalk5.dim(` [done] ${durationStr}${resultPreview}`));
|
|
7451
7612
|
}
|
|
7452
7613
|
break;
|
|
7453
7614
|
case "screenshot:captured":
|
|
@@ -7496,7 +7657,7 @@ program2.command("run [url] [description]").alias("test").description("Run test
|
|
|
7496
7657
|
log(jsonOutput);
|
|
7497
7658
|
}
|
|
7498
7659
|
} else {
|
|
7499
|
-
log(formatTerminal(run2, results2));
|
|
7660
|
+
log(formatTerminal(run2, results2, { failedOnly: opts.failedOnly }));
|
|
7500
7661
|
}
|
|
7501
7662
|
process.exit(getExitCode(run2));
|
|
7502
7663
|
}
|
|
@@ -7529,7 +7690,7 @@ program2.command("run [url] [description]").alias("test").description("Run test
|
|
|
7529
7690
|
log(jsonOutput);
|
|
7530
7691
|
}
|
|
7531
7692
|
} else {
|
|
7532
|
-
log(formatTerminal(run, results));
|
|
7693
|
+
log(formatTerminal(run, results, { failedOnly: opts.failedOnly }));
|
|
7533
7694
|
}
|
|
7534
7695
|
process.exit(getExitCode(run));
|
|
7535
7696
|
} catch (error) {
|
|
@@ -7537,10 +7698,12 @@ program2.command("run [url] [description]").alias("test").description("Run test
|
|
|
7537
7698
|
process.exit(1);
|
|
7538
7699
|
}
|
|
7539
7700
|
});
|
|
7540
|
-
program2.command("runs").description("List past test runs").option("--status <status>", "Filter by status").option("-l, --limit <n>", "Limit results", "20").option("--offset <n>", "Skip first N results", "0").option("--json", "Output as JSON", false).action((opts) => {
|
|
7701
|
+
program2.command("runs").description("List past test runs").option("--status <status>", "Filter by status").option("--sort <field>", "Sort field: date, duration, cost (default: date)").option("--asc", "Sort ascending instead of descending", false).option("-l, --limit <n>", "Limit results", "20").option("--offset <n>", "Skip first N results", "0").option("--json", "Output as JSON", false).action((opts) => {
|
|
7541
7702
|
try {
|
|
7542
7703
|
const runs = listRuns({
|
|
7543
7704
|
status: opts.status,
|
|
7705
|
+
sort: opts.sort,
|
|
7706
|
+
desc: !opts.asc,
|
|
7544
7707
|
limit: parseInt(opts.limit, 10),
|
|
7545
7708
|
offset: parseInt(opts.offset, 10) || undefined
|
|
7546
7709
|
});
|
|
@@ -7663,6 +7826,71 @@ program2.command("import <dir>").description("Import markdown test files as scen
|
|
|
7663
7826
|
process.exit(1);
|
|
7664
7827
|
}
|
|
7665
7828
|
});
|
|
7829
|
+
program2.command("export [format]").description("Export scenarios as JSON (default) or markdown files").option("-o, --output <path>", "Output file (JSON) or directory (markdown)").option("-t, --tag <tag>", "Filter by tag").option("-p, --priority <level>", "Filter by priority").option("--project <id>", "Filter by project ID").action((format, opts) => {
|
|
7830
|
+
try {
|
|
7831
|
+
const fmt = (format ?? "json").toLowerCase();
|
|
7832
|
+
if (fmt !== "json" && fmt !== "markdown") {
|
|
7833
|
+
logError(chalk5.red(`Unknown format: ${fmt}. Supported: json, markdown`));
|
|
7834
|
+
process.exit(1);
|
|
7835
|
+
}
|
|
7836
|
+
const projectId = resolveProject(opts.project);
|
|
7837
|
+
const scenarios = listScenarios({
|
|
7838
|
+
tags: opts.tag ? [opts.tag] : undefined,
|
|
7839
|
+
priority: opts.priority,
|
|
7840
|
+
projectId
|
|
7841
|
+
});
|
|
7842
|
+
if (scenarios.length === 0) {
|
|
7843
|
+
log(chalk5.dim("No scenarios found to export."));
|
|
7844
|
+
return;
|
|
7845
|
+
}
|
|
7846
|
+
if (fmt === "json") {
|
|
7847
|
+
const outputPath = opts.output ?? "testers-export.json";
|
|
7848
|
+
const data = JSON.stringify(scenarios, null, 2);
|
|
7849
|
+
writeFileSync3(outputPath, data, "utf-8");
|
|
7850
|
+
log(chalk5.green(`Exported ${scenarios.length} scenario(s) to ${resolve(outputPath)}`));
|
|
7851
|
+
return;
|
|
7852
|
+
}
|
|
7853
|
+
const outputDir = opts.output ?? ".";
|
|
7854
|
+
if (!existsSync8(outputDir)) {
|
|
7855
|
+
mkdirSync4(outputDir, { recursive: true });
|
|
7856
|
+
}
|
|
7857
|
+
for (const s of scenarios) {
|
|
7858
|
+
const lines = [];
|
|
7859
|
+
lines.push(`# ${s.name}`);
|
|
7860
|
+
lines.push("");
|
|
7861
|
+
if (s.description && s.description !== s.name) {
|
|
7862
|
+
lines.push(s.description);
|
|
7863
|
+
lines.push("");
|
|
7864
|
+
}
|
|
7865
|
+
if (s.tags.length > 0) {
|
|
7866
|
+
lines.push(`**Tags:** ${s.tags.join(", ")}`);
|
|
7867
|
+
}
|
|
7868
|
+
lines.push(`**Priority:** ${s.priority}`);
|
|
7869
|
+
if (s.targetPath) {
|
|
7870
|
+
lines.push(`**Path:** ${s.targetPath}`);
|
|
7871
|
+
}
|
|
7872
|
+
lines.push("");
|
|
7873
|
+
if (s.steps.length > 0) {
|
|
7874
|
+
lines.push("## Steps");
|
|
7875
|
+
lines.push("");
|
|
7876
|
+
for (const step of s.steps) {
|
|
7877
|
+
lines.push(`- [ ] ${step}`);
|
|
7878
|
+
}
|
|
7879
|
+
lines.push("");
|
|
7880
|
+
}
|
|
7881
|
+
const safeFilename = s.name.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "").slice(0, 80);
|
|
7882
|
+
const filePath = join6(outputDir, `${s.shortId}-${safeFilename}.md`);
|
|
7883
|
+
writeFileSync3(filePath, lines.join(`
|
|
7884
|
+
`), "utf-8");
|
|
7885
|
+
log(chalk5.dim(` ${s.shortId}: ${s.name} \u2192 ${filePath}`));
|
|
7886
|
+
}
|
|
7887
|
+
log(chalk5.green(`
|
|
7888
|
+
Exported ${scenarios.length} scenario(s) as markdown to ${resolve(outputDir)}`));
|
|
7889
|
+
} catch (error) {
|
|
7890
|
+
logError(chalk5.red(`Error: ${error instanceof Error ? error.message : String(error)}`));
|
|
7891
|
+
process.exit(1);
|
|
7892
|
+
}
|
|
7893
|
+
});
|
|
7666
7894
|
program2.command("config").description("Show current configuration").action(() => {
|
|
7667
7895
|
try {
|
|
7668
7896
|
const config = loadConfig();
|
|
@@ -8275,9 +8503,20 @@ authCmd.command("delete <name>").description("Delete an auth preset").action((na
|
|
|
8275
8503
|
process.exit(1);
|
|
8276
8504
|
}
|
|
8277
8505
|
});
|
|
8278
|
-
program2.command("costs").description("Show cost tracking and budget status").option("--project <id>", "Project ID").option("--period <period>", "Time period", "month").option("--json", "JSON output", false).option("--csv", "CSV output", false).action((opts) => {
|
|
8506
|
+
program2.command("costs").description("Show cost tracking and budget status").option("--project <id>", "Project ID").option("--period <period>", "Time period: day, week, month, all (default: month)", "month").option("--by-scenario", "Group cost breakdown by scenario, sorted by total cost", false).option("--json", "JSON output", false).option("--csv", "CSV output", false).action((opts) => {
|
|
8279
8507
|
try {
|
|
8280
|
-
const
|
|
8508
|
+
const projectId = resolveProject(opts.project);
|
|
8509
|
+
const period = opts.period;
|
|
8510
|
+
if (opts.byScenario) {
|
|
8511
|
+
const rows = getCostsByScenario({ projectId, period });
|
|
8512
|
+
if (opts.json) {
|
|
8513
|
+
log(JSON.stringify(rows, null, 2));
|
|
8514
|
+
} else {
|
|
8515
|
+
log(formatCostsByScenarioTerminal(rows, period));
|
|
8516
|
+
}
|
|
8517
|
+
return;
|
|
8518
|
+
}
|
|
8519
|
+
const summary = getCostSummary({ projectId, period });
|
|
8281
8520
|
if (opts.csv) {
|
|
8282
8521
|
log(formatCostsCsv(summary));
|
|
8283
8522
|
} else if (opts.json) {
|
package/dist/db/runs.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"runs.d.ts","sourceRoot":"","sources":["../../src/db/runs.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,GAAG,EACR,KAAK,MAAM,EACX,KAAK,cAAc,EACnB,KAAK,SAAS,EAEf,MAAM,mBAAmB,CAAC;AAG3B,wBAAgB,SAAS,CAAC,KAAK,EAAE,cAAc,GAAG;IAAE,KAAK,EAAE,MAAM,CAAA;CAAE,GAAG,GAAG,CAoBxE;AAED,wBAAgB,MAAM,CAAC,EAAE,EAAE,MAAM,GAAG,GAAG,GAAG,IAAI,CAc7C;AAED,wBAAgB,QAAQ,CAAC,MAAM,CAAC,EAAE,SAAS,GAAG,GAAG,EAAE,
|
|
1
|
+
{"version":3,"file":"runs.d.ts","sourceRoot":"","sources":["../../src/db/runs.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,GAAG,EACR,KAAK,MAAM,EACX,KAAK,cAAc,EACnB,KAAK,SAAS,EAEf,MAAM,mBAAmB,CAAC;AAG3B,wBAAgB,SAAS,CAAC,KAAK,EAAE,cAAc,GAAG;IAAE,KAAK,EAAE,MAAM,CAAA;CAAE,GAAG,GAAG,CAoBxE;AAED,wBAAgB,MAAM,CAAC,EAAE,EAAE,MAAM,GAAG,GAAG,GAAG,IAAI,CAc7C;AAED,wBAAgB,QAAQ,CAAC,MAAM,CAAC,EAAE,SAAS,GAAG,GAAG,EAAE,CAwClD;AAED,wBAAgB,SAAS,CAAC,MAAM,CAAC,EAAE,SAAS,GAAG,MAAM,CAmBpD;AAED,wBAAgB,SAAS,CAAC,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,CAAC,MAAM,CAAC,GAAG,GAAG,CAmEnE;AAED,wBAAgB,SAAS,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAO7C"}
|
package/dist/db/scenarios.d.ts
CHANGED
|
@@ -5,5 +5,9 @@ export declare function getScenarioByShortId(shortId: string): Scenario | null;
|
|
|
5
5
|
export declare function listScenarios(filter?: ScenarioFilter): Scenario[];
|
|
6
6
|
export declare function updateScenario(id: string, input: UpdateScenarioInput, version: number): Scenario;
|
|
7
7
|
export declare function countScenarios(filter?: ScenarioFilter): number;
|
|
8
|
+
export interface StaleScenario extends Scenario {
|
|
9
|
+
lastRunAt: string | null;
|
|
10
|
+
}
|
|
11
|
+
export declare function findStaleScenarios(days: number): StaleScenario[];
|
|
8
12
|
export declare function deleteScenario(id: string): boolean;
|
|
9
13
|
//# sourceMappingURL=scenarios.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"scenarios.d.ts","sourceRoot":"","sources":["../../src/db/scenarios.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,QAAQ,EAEb,KAAK,mBAAmB,EACxB,KAAK,mBAAmB,EACxB,KAAK,cAAc,EAGpB,MAAM,mBAAmB,CAAC;AAsB3B,wBAAgB,cAAc,CAAC,KAAK,EAAE,mBAAmB,GAAG,QAAQ,CA8BnE;AAED,wBAAgB,WAAW,CAAC,EAAE,EAAE,MAAM,GAAG,QAAQ,GAAG,IAAI,CAmBvD;AAED,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,MAAM,GAAG,QAAQ,GAAG,IAAI,CAIrE;AAED,wBAAgB,aAAa,CAAC,MAAM,CAAC,EAAE,cAAc,GAAG,QAAQ,EAAE,
|
|
1
|
+
{"version":3,"file":"scenarios.d.ts","sourceRoot":"","sources":["../../src/db/scenarios.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,QAAQ,EAEb,KAAK,mBAAmB,EACxB,KAAK,mBAAmB,EACxB,KAAK,cAAc,EAGpB,MAAM,mBAAmB,CAAC;AAsB3B,wBAAgB,cAAc,CAAC,KAAK,EAAE,mBAAmB,GAAG,QAAQ,CA8BnE;AAED,wBAAgB,WAAW,CAAC,EAAE,EAAE,MAAM,GAAG,QAAQ,GAAG,IAAI,CAmBvD;AAED,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,MAAM,GAAG,QAAQ,GAAG,IAAI,CAIrE;AAED,wBAAgB,aAAa,CAAC,MAAM,CAAC,EAAE,cAAc,GAAG,QAAQ,EAAE,CAqDjE;AAED,wBAAgB,cAAc,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,mBAAmB,EAAE,OAAO,EAAE,MAAM,GAAG,QAAQ,CAoFhG;AAED,wBAAgB,cAAc,CAAC,MAAM,CAAC,EAAE,cAAc,GAAG,MAAM,CA8B9D;AAED,MAAM,WAAW,aAAc,SAAQ,QAAQ;IAC7C,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;CAC1B;AAED,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,MAAM,GAAG,aAAa,EAAE,CAmBhE;AAED,wBAAgB,cAAc,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAOlD"}
|
package/dist/index.js
CHANGED
|
@@ -555,7 +555,10 @@ function listRuns(filter) {
|
|
|
555
555
|
if (conditions.length > 0) {
|
|
556
556
|
sql += " WHERE " + conditions.join(" AND ");
|
|
557
557
|
}
|
|
558
|
-
|
|
558
|
+
const sortField = filter?.sort ?? "date";
|
|
559
|
+
const sortDir = filter?.desc === false ? "ASC" : "DESC";
|
|
560
|
+
const orderByCol = sortField === "duration" ? "(CASE WHEN finished_at IS NULL THEN NULL ELSE (julianday(finished_at) - julianday(started_at)) * 86400000 END)" : sortField === "cost" ? "(SELECT COALESCE(SUM(cost_cents), 0) FROM results WHERE run_id = runs.id)" : "started_at";
|
|
561
|
+
sql += ` ORDER BY ${orderByCol} ${sortDir}`;
|
|
559
562
|
if (filter?.limit) {
|
|
560
563
|
sql += " LIMIT ?";
|
|
561
564
|
params.push(filter.limit);
|
|
@@ -1052,7 +1055,10 @@ function listScenarios(filter) {
|
|
|
1052
1055
|
if (conditions.length > 0) {
|
|
1053
1056
|
sql += " WHERE " + conditions.join(" AND ");
|
|
1054
1057
|
}
|
|
1055
|
-
|
|
1058
|
+
const sortField = filter?.sort ?? "date";
|
|
1059
|
+
const sortDir = filter?.desc === false ? "ASC" : "DESC";
|
|
1060
|
+
const orderByCol = sortField === "name" ? "name" : sortField === "priority" ? "CASE priority WHEN 'critical' THEN 0 WHEN 'high' THEN 1 WHEN 'medium' THEN 2 WHEN 'low' THEN 3 ELSE 4 END" : "created_at";
|
|
1061
|
+
sql += ` ORDER BY ${orderByCol} ${sortDir}`;
|
|
1056
1062
|
if (filter?.limit) {
|
|
1057
1063
|
sql += " LIMIT ?";
|
|
1058
1064
|
params.push(filter.limit);
|
|
@@ -2727,14 +2733,26 @@ function emit(event) {
|
|
|
2727
2733
|
}
|
|
2728
2734
|
function withTimeout(promise, ms, label) {
|
|
2729
2735
|
return new Promise((resolve, reject) => {
|
|
2736
|
+
const warningAt = Math.floor(ms * 0.8);
|
|
2737
|
+
const warningTimer = setTimeout(() => {
|
|
2738
|
+
emit({
|
|
2739
|
+
type: "scenario:timeout_warning",
|
|
2740
|
+
scenarioName: label,
|
|
2741
|
+
timeoutMs: ms,
|
|
2742
|
+
elapsedMs: warningAt
|
|
2743
|
+
});
|
|
2744
|
+
}, warningAt);
|
|
2730
2745
|
const timer = setTimeout(() => {
|
|
2746
|
+
clearTimeout(warningTimer);
|
|
2731
2747
|
reject(new Error(`Scenario '${label}' timed out after ${ms}ms. Try: testers run --timeout ${ms * 2} or simplify the scenario steps.`));
|
|
2732
2748
|
}, ms);
|
|
2733
2749
|
promise.then((val) => {
|
|
2734
2750
|
clearTimeout(timer);
|
|
2751
|
+
clearTimeout(warningTimer);
|
|
2735
2752
|
resolve(val);
|
|
2736
2753
|
}, (err) => {
|
|
2737
2754
|
clearTimeout(timer);
|
|
2755
|
+
clearTimeout(warningTimer);
|
|
2738
2756
|
reject(err);
|
|
2739
2757
|
});
|
|
2740
2758
|
});
|
|
@@ -2763,6 +2781,7 @@ async function runSingleScenario(scenario, runId, options) {
|
|
|
2763
2781
|
const targetUrl = scenario.targetPath ? `${options.url.replace(/\/$/, "")}${scenario.targetPath}` : options.url;
|
|
2764
2782
|
const scenarioTimeout = scenario.timeoutMs ?? options.timeout ?? config.browser.timeout ?? 60000;
|
|
2765
2783
|
await page.goto(targetUrl, { timeout: Math.min(scenarioTimeout, 30000) });
|
|
2784
|
+
const stepStartTimes = new Map;
|
|
2766
2785
|
const agentResult = await withTimeout(runAgentLoop({
|
|
2767
2786
|
client,
|
|
2768
2787
|
page,
|
|
@@ -2772,6 +2791,16 @@ async function runSingleScenario(scenario, runId, options) {
|
|
|
2772
2791
|
runId,
|
|
2773
2792
|
maxTurns: 30,
|
|
2774
2793
|
onStep: (stepEvent) => {
|
|
2794
|
+
let stepDurationMs;
|
|
2795
|
+
if (stepEvent.type === "tool_call") {
|
|
2796
|
+
stepStartTimes.set(stepEvent.stepNumber, Date.now());
|
|
2797
|
+
} else if (stepEvent.type === "tool_result") {
|
|
2798
|
+
const startTime = stepStartTimes.get(stepEvent.stepNumber);
|
|
2799
|
+
if (startTime !== undefined) {
|
|
2800
|
+
stepDurationMs = Date.now() - startTime;
|
|
2801
|
+
stepStartTimes.delete(stepEvent.stepNumber);
|
|
2802
|
+
}
|
|
2803
|
+
}
|
|
2775
2804
|
emit({
|
|
2776
2805
|
type: `step:${stepEvent.type}`,
|
|
2777
2806
|
scenarioId: scenario.id,
|
|
@@ -2781,7 +2810,8 @@ async function runSingleScenario(scenario, runId, options) {
|
|
|
2781
2810
|
toolInput: stepEvent.toolInput,
|
|
2782
2811
|
toolResult: stepEvent.toolResult,
|
|
2783
2812
|
thinking: stepEvent.thinking,
|
|
2784
|
-
stepNumber: stepEvent.stepNumber
|
|
2813
|
+
stepNumber: stepEvent.stepNumber,
|
|
2814
|
+
stepDurationMs
|
|
2785
2815
|
});
|
|
2786
2816
|
}
|
|
2787
2817
|
}), scenarioTimeout, scenario.name);
|
|
@@ -3534,13 +3564,24 @@ init_database();
|
|
|
3534
3564
|
function useEmoji() {
|
|
3535
3565
|
return !process.env["NO_COLOR"] && process.argv.indexOf("--no-color") === -1;
|
|
3536
3566
|
}
|
|
3537
|
-
function formatTerminal(run, results) {
|
|
3567
|
+
function formatTerminal(run, results, options) {
|
|
3538
3568
|
const lines = [];
|
|
3569
|
+
const failedOnly = options?.failedOnly ?? false;
|
|
3539
3570
|
lines.push("");
|
|
3540
3571
|
lines.push(source_default.bold(` Run ${run.id.slice(0, 8)} \u2014 ${run.url}`));
|
|
3541
3572
|
lines.push(source_default.dim(` Model: ${run.model} | Parallel: ${run.parallel} | Headed: ${run.headed ? "yes" : "no"}`));
|
|
3542
3573
|
lines.push("");
|
|
3574
|
+
if (failedOnly) {
|
|
3575
|
+
const passedCount = results.filter((r) => r.status === "passed").length;
|
|
3576
|
+
if (passedCount > 0) {
|
|
3577
|
+
lines.push(source_default.dim(` (${passedCount} passed scenario${passedCount !== 1 ? "s" : ""} hidden \u2014 use without --failed-only to see all)`));
|
|
3578
|
+
lines.push("");
|
|
3579
|
+
}
|
|
3580
|
+
}
|
|
3543
3581
|
for (const result of results) {
|
|
3582
|
+
if (failedOnly && result.status !== "failed" && result.status !== "error") {
|
|
3583
|
+
continue;
|
|
3584
|
+
}
|
|
3544
3585
|
const scenario = getScenario(result.scenarioId);
|
|
3545
3586
|
const name = scenario ? `${scenario.shortId}: ${scenario.name}` : result.scenarioId.slice(0, 8);
|
|
3546
3587
|
const screenshots = listScreenshots(result.id);
|
package/dist/lib/costs.d.ts
CHANGED
|
@@ -27,6 +27,18 @@ export declare function getCostSummary(options?: {
|
|
|
27
27
|
projectId?: string;
|
|
28
28
|
period?: "day" | "week" | "month" | "all";
|
|
29
29
|
}): CostSummary;
|
|
30
|
+
export interface ScenarioCostRow {
|
|
31
|
+
scenarioId: string;
|
|
32
|
+
name: string;
|
|
33
|
+
runCount: number;
|
|
34
|
+
totalCostCents: number;
|
|
35
|
+
avgCostPerRunCents: number;
|
|
36
|
+
}
|
|
37
|
+
export declare function getCostsByScenario(options?: {
|
|
38
|
+
projectId?: string;
|
|
39
|
+
period?: "day" | "week" | "month" | "all";
|
|
40
|
+
}): ScenarioCostRow[];
|
|
41
|
+
export declare function formatCostsByScenarioTerminal(rows: ScenarioCostRow[], period: string): string;
|
|
30
42
|
export declare function checkBudget(estimatedCostCents: number): {
|
|
31
43
|
allowed: boolean;
|
|
32
44
|
warning?: string;
|
package/dist/lib/costs.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"costs.d.ts","sourceRoot":"","sources":["../../src/lib/costs.ts"],"names":[],"mappings":"AAMA,MAAM,WAAW,WAAW;IAC1B,MAAM,EAAE,MAAM,CAAC;IACf,cAAc,EAAE,MAAM,CAAC;IACvB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC7E,UAAU,EAAE,KAAK,CAAC;QAAE,UAAU,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACzG,aAAa,EAAE,MAAM,CAAC;IACtB,qBAAqB,EAAE,MAAM,CAAC;CAC/B;AAED,MAAM,WAAW,YAAY;IAC3B,cAAc,EAAE,MAAM,CAAC;IACvB,cAAc,EAAE,MAAM,CAAC;IACvB,aAAa,EAAE,MAAM,CAAC;CACvB;AA0CD,wBAAgB,cAAc,CAAC,OAAO,CAAC,EAAE;IACvC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,KAAK,GAAG,MAAM,GAAG,OAAO,GAAG,KAAK,CAAC;CAC3C,GAAG,WAAW,CAyFd;AAED,wBAAgB,WAAW,CAAC,kBAAkB,EAAE,MAAM,GAAG;IAAE,OAAO,EAAE,OAAO,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAA;CAAE,CA+B9F;AAcD,wBAAgB,mBAAmB,CAAC,OAAO,EAAE,WAAW,GAAG,MAAM,CAyChE;AAED,wBAAgB,eAAe,CAAC,OAAO,EAAE,WAAW,GAAG,MAAM,CAE5D;AAED,wBAAgB,cAAc,CAAC,OAAO,EAAE,WAAW,GAAG,MAAM,CAS3D"}
|
|
1
|
+
{"version":3,"file":"costs.d.ts","sourceRoot":"","sources":["../../src/lib/costs.ts"],"names":[],"mappings":"AAMA,MAAM,WAAW,WAAW;IAC1B,MAAM,EAAE,MAAM,CAAC;IACf,cAAc,EAAE,MAAM,CAAC;IACvB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC7E,UAAU,EAAE,KAAK,CAAC;QAAE,UAAU,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACzG,aAAa,EAAE,MAAM,CAAC;IACtB,qBAAqB,EAAE,MAAM,CAAC;CAC/B;AAED,MAAM,WAAW,YAAY;IAC3B,cAAc,EAAE,MAAM,CAAC;IACvB,cAAc,EAAE,MAAM,CAAC;IACvB,aAAa,EAAE,MAAM,CAAC;CACvB;AA0CD,wBAAgB,cAAc,CAAC,OAAO,CAAC,EAAE;IACvC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,KAAK,GAAG,MAAM,GAAG,OAAO,GAAG,KAAK,CAAC;CAC3C,GAAG,WAAW,CAyFd;AAID,MAAM,WAAW,eAAe;IAC9B,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,cAAc,EAAE,MAAM,CAAC;IACvB,kBAAkB,EAAE,MAAM,CAAC;CAC5B;AAED,wBAAgB,kBAAkB,CAAC,OAAO,CAAC,EAAE;IAC3C,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,KAAK,GAAG,MAAM,GAAG,OAAO,GAAG,KAAK,CAAC;CAC3C,GAAG,eAAe,EAAE,CAoCpB;AAED,wBAAgB,6BAA6B,CAAC,IAAI,EAAE,eAAe,EAAE,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAyB7F;AAED,wBAAgB,WAAW,CAAC,kBAAkB,EAAE,MAAM,GAAG;IAAE,OAAO,EAAE,OAAO,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAA;CAAE,CA+B9F;AAcD,wBAAgB,mBAAmB,CAAC,OAAO,EAAE,WAAW,GAAG,MAAM,CAyChE;AAED,wBAAgB,eAAe,CAAC,OAAO,EAAE,WAAW,GAAG,MAAM,CAE5D;AAED,wBAAgB,cAAc,CAAC,OAAO,EAAE,WAAW,GAAG,MAAM,CAS3D"}
|
package/dist/lib/reporter.d.ts
CHANGED
|
@@ -2,8 +2,9 @@ import type { Run, Result, Screenshot } from "../types/index.js";
|
|
|
2
2
|
export interface ReportOptions {
|
|
3
3
|
json?: boolean;
|
|
4
4
|
verbose?: boolean;
|
|
5
|
+
failedOnly?: boolean;
|
|
5
6
|
}
|
|
6
|
-
export declare function formatTerminal(run: Run, results: Result[]): string;
|
|
7
|
+
export declare function formatTerminal(run: Run, results: Result[], options?: ReportOptions): string;
|
|
7
8
|
export declare function formatSummary(run: Run): string;
|
|
8
9
|
export declare function formatActionableSummary(run: Run, results: Result[]): string;
|
|
9
10
|
export declare function formatJSON(run: Run, results: Result[]): string;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"reporter.d.ts","sourceRoot":"","sources":["../../src/lib/reporter.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAWjE,MAAM,WAAW,aAAa;IAC5B,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,OAAO,CAAC,EAAE,OAAO,CAAC;
|
|
1
|
+
{"version":3,"file":"reporter.d.ts","sourceRoot":"","sources":["../../src/lib/reporter.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAWjE,MAAM,WAAW,aAAa;IAC5B,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,UAAU,CAAC,EAAE,OAAO,CAAC;CACtB;AAED,wBAAgB,cAAc,CAAC,GAAG,EAAE,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,OAAO,CAAC,EAAE,aAAa,GAAG,MAAM,CAmE3F;AAED,wBAAgB,aAAa,CAAC,GAAG,EAAE,GAAG,GAAG,MAAM,CAU9C;AAED,wBAAgB,uBAAuB,CAAC,GAAG,EAAE,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,MAAM,CA4B3E;AAED,wBAAgB,UAAU,CAAC,GAAG,EAAE,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,MAAM,CAoD9D;AAED,wBAAgB,WAAW,CAAC,GAAG,EAAE,GAAG,GAAG,MAAM,CAI5C;AAED,wBAAgB,aAAa,CAAC,IAAI,EAAE,GAAG,EAAE,GAAG,MAAM,CA6BjD;AAED,MAAM,WAAW,gBAAgB;IAC/B,UAAU,EAAE,QAAQ,GAAG,QAAQ,GAAG,OAAO,GAAG,SAAS,GAAG,IAAI,CAAC;IAC7D,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,wBAAgB,mBAAmB,CAAC,UAAU,EAAE,MAAM,GAAG,gBAAgB,CAiBxE;AAED,wBAAgB,kBAAkB,CAAC,SAAS,EAAE,KAAK,CAAC;IAAE,EAAE,CAAC,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,EAAE,CAAA;CAAE,CAAC,GAAG,MAAM,CAwC7I;AAED,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,UAAU,EAAE,GAAG,MAAM,CAqCpF"}
|