executable-stories-formatters 0.12.0 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -0
- package/dist/adapters.d.cts +1 -1
- package/dist/adapters.d.ts +1 -1
- package/dist/cli.js +565 -8
- package/dist/cli.js.map +1 -1
- package/dist/{index-mrT6-JSt.d.cts → index-CXrzCk9p.d.cts} +1 -1
- package/dist/{index-mrT6-JSt.d.ts → index-CXrzCk9p.d.ts} +1 -1
- package/dist/index.cjs +423 -2
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +233 -4
- package/dist/index.d.ts +233 -4
- package/dist/index.js +415 -2
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
- package/schemas/raw-run.schema.json +9 -0
package/dist/cli.js
CHANGED
|
@@ -191,6 +191,11 @@ var raw_run_schema_default = {
|
|
|
191
191
|
},
|
|
192
192
|
description: "Ticket/issue references. Each item is either a string ID or an object with id and optional url."
|
|
193
193
|
},
|
|
194
|
+
covers: {
|
|
195
|
+
type: "array",
|
|
196
|
+
items: { type: "string" },
|
|
197
|
+
description: "Product-code paths/globs this scenario exercises (project-root-relative). Used to map code changes to at-risk scenarios."
|
|
198
|
+
},
|
|
194
199
|
meta: {
|
|
195
200
|
type: "object",
|
|
196
201
|
description: "User-defined metadata for this story."
|
|
@@ -512,6 +517,10 @@ var raw_run_schema_default = {
|
|
|
512
517
|
minimum: 0,
|
|
513
518
|
description: "Step index (0-based)."
|
|
514
519
|
},
|
|
520
|
+
stepId: {
|
|
521
|
+
type: "string",
|
|
522
|
+
description: "Stable step ID when the framework provides one (correlates with StoryStep.id)."
|
|
523
|
+
},
|
|
515
524
|
title: {
|
|
516
525
|
type: "string",
|
|
517
526
|
description: "Step title/description."
|
|
@@ -1290,14 +1299,14 @@ var CucumberJsonFormatter = class {
|
|
|
1290
1299
|
duration: 0
|
|
1291
1300
|
};
|
|
1292
1301
|
}
|
|
1293
|
-
const
|
|
1302
|
+
const statusMap2 = {
|
|
1294
1303
|
passed: "passed",
|
|
1295
1304
|
failed: "failed",
|
|
1296
1305
|
skipped: "skipped",
|
|
1297
1306
|
pending: "pending"
|
|
1298
1307
|
};
|
|
1299
1308
|
const stepResult = {
|
|
1300
|
-
status:
|
|
1309
|
+
status: statusMap2[result.status] ?? "undefined",
|
|
1301
1310
|
// Duration in nanoseconds (Cucumber uses nanoseconds)
|
|
1302
1311
|
duration: result.durationMs * 1e6
|
|
1303
1312
|
};
|
|
@@ -16446,6 +16455,132 @@ function escapePipe(value) {
|
|
|
16446
16455
|
return value.replace(/\|/g, "\\|");
|
|
16447
16456
|
}
|
|
16448
16457
|
|
|
16458
|
+
// src/formatters/traceability-matrix.ts
|
|
16459
|
+
var TraceabilityMatrixFormatter = class {
|
|
16460
|
+
format(run) {
|
|
16461
|
+
const matrix = toTraceabilityMatrix(run);
|
|
16462
|
+
const lines = [];
|
|
16463
|
+
lines.push("# Traceability Matrix");
|
|
16464
|
+
lines.push("");
|
|
16465
|
+
lines.push(`Generated: ${matrix.generatedAt}`);
|
|
16466
|
+
lines.push(`Run: ${matrix.run.startedAt} to ${matrix.run.finishedAt}`);
|
|
16467
|
+
if (matrix.run.branch) lines.push(`Branch: ${matrix.run.branch}`);
|
|
16468
|
+
if (matrix.run.gitSha) lines.push(`Commit: ${matrix.run.gitSha}`);
|
|
16469
|
+
lines.push("");
|
|
16470
|
+
lines.push("| Requirements | Verified | Failing | Scenarios | Untraced |");
|
|
16471
|
+
lines.push("| ---: | ---: | ---: | ---: | ---: |");
|
|
16472
|
+
lines.push(
|
|
16473
|
+
`| ${matrix.summary.requirements} | ${matrix.summary.requirementsVerified} | ${matrix.summary.requirementsFailing} | ${matrix.summary.scenarios} | ${matrix.summary.untracedScenarios} |`
|
|
16474
|
+
);
|
|
16475
|
+
lines.push("");
|
|
16476
|
+
for (const req of matrix.requirements) {
|
|
16477
|
+
const heading2 = req.url ? `[${req.ticket}](${req.url})` : req.ticket;
|
|
16478
|
+
lines.push(`## ${heading2}`);
|
|
16479
|
+
lines.push("");
|
|
16480
|
+
lines.push(`Status: ${renderRequirementStatus(req.status)}`);
|
|
16481
|
+
if (req.covers.length > 0) {
|
|
16482
|
+
lines.push(`Covers: ${req.covers.map((path16) => `\`${path16}\``).join(", ")}`);
|
|
16483
|
+
}
|
|
16484
|
+
lines.push("");
|
|
16485
|
+
lines.push("| Status | Scenario | Source | Covers |");
|
|
16486
|
+
lines.push("| --- | --- | --- | --- |");
|
|
16487
|
+
for (const scenario of req.scenarios) {
|
|
16488
|
+
const source = `${scenario.sourceFile}:${scenario.sourceLine}`;
|
|
16489
|
+
const covers = scenario.covers.length > 0 ? scenario.covers.map((path16) => `\`${path16}\``).join(", ") : "";
|
|
16490
|
+
lines.push(`| ${scenario.status} | ${escapePipe2(scenario.title)} | \`${source}\` | ${covers} |`);
|
|
16491
|
+
}
|
|
16492
|
+
lines.push("");
|
|
16493
|
+
}
|
|
16494
|
+
if (matrix.untraced.length > 0) {
|
|
16495
|
+
lines.push("## Untraced scenarios");
|
|
16496
|
+
lines.push("");
|
|
16497
|
+
lines.push("Behavior with no requirement link. Add a `ticket` to each so it appears against a requirement.");
|
|
16498
|
+
lines.push("");
|
|
16499
|
+
lines.push("| Status | Scenario | Source |");
|
|
16500
|
+
lines.push("| --- | --- | --- |");
|
|
16501
|
+
for (const scenario of matrix.untraced) {
|
|
16502
|
+
const source = `${scenario.sourceFile}:${scenario.sourceLine}`;
|
|
16503
|
+
lines.push(`| ${scenario.status} | ${escapePipe2(scenario.title)} | \`${source}\` |`);
|
|
16504
|
+
}
|
|
16505
|
+
lines.push("");
|
|
16506
|
+
}
|
|
16507
|
+
return lines.join("\n").trimEnd();
|
|
16508
|
+
}
|
|
16509
|
+
};
|
|
16510
|
+
function toTraceabilityMatrix(run) {
|
|
16511
|
+
const sorted = [...run.testCases].sort((a, b) => a.id.localeCompare(b.id));
|
|
16512
|
+
const byTicket = /* @__PURE__ */ new Map();
|
|
16513
|
+
const untraced = [];
|
|
16514
|
+
for (const tc of sorted) {
|
|
16515
|
+
const tickets = tc.story.tickets ?? [];
|
|
16516
|
+
if (tickets.length === 0) {
|
|
16517
|
+
untraced.push({
|
|
16518
|
+
id: tc.id,
|
|
16519
|
+
title: tc.story.scenario,
|
|
16520
|
+
status: tc.status,
|
|
16521
|
+
sourceFile: tc.sourceFile,
|
|
16522
|
+
sourceLine: tc.sourceLine
|
|
16523
|
+
});
|
|
16524
|
+
continue;
|
|
16525
|
+
}
|
|
16526
|
+
for (const ticket of tickets) {
|
|
16527
|
+
const entry = byTicket.get(ticket.id) ?? { url: ticket.url, cases: [] };
|
|
16528
|
+
if (!entry.url && ticket.url) entry.url = ticket.url;
|
|
16529
|
+
entry.cases.push(tc);
|
|
16530
|
+
byTicket.set(ticket.id, entry);
|
|
16531
|
+
}
|
|
16532
|
+
}
|
|
16533
|
+
const requirements = [...byTicket.entries()].sort(([a], [b]) => a.localeCompare(b)).map(([ticket, entry]) => {
|
|
16534
|
+
const scenarios = entry.cases.map((tc) => ({
|
|
16535
|
+
id: tc.id,
|
|
16536
|
+
title: tc.story.scenario,
|
|
16537
|
+
status: tc.status,
|
|
16538
|
+
sourceFile: tc.sourceFile,
|
|
16539
|
+
sourceLine: tc.sourceLine,
|
|
16540
|
+
covers: tc.story.covers ?? []
|
|
16541
|
+
}));
|
|
16542
|
+
const covers = [...new Set(scenarios.flatMap((s) => s.covers))].sort();
|
|
16543
|
+
return { ticket, url: entry.url, status: requirementStatus(entry.cases), scenarios, covers };
|
|
16544
|
+
});
|
|
16545
|
+
return {
|
|
16546
|
+
schemaVersion: "1.0",
|
|
16547
|
+
generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
16548
|
+
run: {
|
|
16549
|
+
startedAt: new Date(run.startedAtMs).toISOString(),
|
|
16550
|
+
finishedAt: new Date(run.finishedAtMs).toISOString(),
|
|
16551
|
+
gitSha: run.gitSha,
|
|
16552
|
+
branch: run.ci?.branch
|
|
16553
|
+
},
|
|
16554
|
+
summary: {
|
|
16555
|
+
requirements: requirements.length,
|
|
16556
|
+
requirementsVerified: requirements.filter((r) => r.status === "verified").length,
|
|
16557
|
+
requirementsFailing: requirements.filter((r) => r.status === "failing").length,
|
|
16558
|
+
scenarios: run.testCases.length,
|
|
16559
|
+
untracedScenarios: untraced.length
|
|
16560
|
+
},
|
|
16561
|
+
requirements,
|
|
16562
|
+
untraced
|
|
16563
|
+
};
|
|
16564
|
+
}
|
|
16565
|
+
function requirementStatus(cases) {
|
|
16566
|
+
if (cases.some((tc) => tc.status === "failed")) return "failing";
|
|
16567
|
+
if (cases.some((tc) => tc.status === "passed")) return "verified";
|
|
16568
|
+
return "incomplete";
|
|
16569
|
+
}
|
|
16570
|
+
function renderRequirementStatus(status) {
|
|
16571
|
+
switch (status) {
|
|
16572
|
+
case "verified":
|
|
16573
|
+
return "verified (all scenarios passed)";
|
|
16574
|
+
case "failing":
|
|
16575
|
+
return "failing (a scenario failed)";
|
|
16576
|
+
default:
|
|
16577
|
+
return "incomplete (no scenario passed yet)";
|
|
16578
|
+
}
|
|
16579
|
+
}
|
|
16580
|
+
function escapePipe2(value) {
|
|
16581
|
+
return value.replace(/\|/g, "\\|");
|
|
16582
|
+
}
|
|
16583
|
+
|
|
16449
16584
|
// src/formatters/cucumber-messages/synthesize-feature.ts
|
|
16450
16585
|
function extractFeatureName(testCases, uri) {
|
|
16451
16586
|
for (const tc of testCases) {
|
|
@@ -20262,6 +20397,280 @@ function collectDocKinds(testCase) {
|
|
|
20262
20397
|
return [...kinds].sort();
|
|
20263
20398
|
}
|
|
20264
20399
|
|
|
20400
|
+
// src/scenario-failure.ts
|
|
20401
|
+
function failingScenarioMessage(tc) {
|
|
20402
|
+
const failingStep = tc.stepResults.find((s) => s.status === "failed" && s.errorMessage);
|
|
20403
|
+
return failingStep?.errorMessage ?? tc.errorMessage;
|
|
20404
|
+
}
|
|
20405
|
+
|
|
20406
|
+
// src/check.ts
|
|
20407
|
+
var ICON_PASS = "\u2713";
|
|
20408
|
+
var ICON_FAIL = "\u2717";
|
|
20409
|
+
var ICON_SKIP = "\u2298";
|
|
20410
|
+
var ICON_PENDING = "\u23F3";
|
|
20411
|
+
var ICON_WARN = "\u26A0";
|
|
20412
|
+
function buildCheck(args, _deps = {}) {
|
|
20413
|
+
const { testCases, baseline } = args;
|
|
20414
|
+
const summary = {
|
|
20415
|
+
total: testCases.length,
|
|
20416
|
+
passed: testCases.filter((tc) => tc.status === "passed").length,
|
|
20417
|
+
failed: testCases.filter((tc) => tc.status === "failed").length,
|
|
20418
|
+
skipped: testCases.filter((tc) => tc.status === "skipped").length,
|
|
20419
|
+
pending: testCases.filter((tc) => tc.status === "pending").length
|
|
20420
|
+
};
|
|
20421
|
+
let regressed = 0;
|
|
20422
|
+
let fixed = 0;
|
|
20423
|
+
if (baseline) {
|
|
20424
|
+
for (const tc of testCases) {
|
|
20425
|
+
const before = baseline.get(tc.id);
|
|
20426
|
+
if (before === "passed" && tc.status === "failed") regressed += 1;
|
|
20427
|
+
if (before === "failed" && tc.status === "passed") fixed += 1;
|
|
20428
|
+
}
|
|
20429
|
+
}
|
|
20430
|
+
const failures = testCases.filter((tc) => tc.status === "failed").map((tc) => toFailure(tc, baseline)).sort((a, b) => {
|
|
20431
|
+
if (a.regressed !== b.regressed) return a.regressed ? -1 : 1;
|
|
20432
|
+
return a.location.localeCompare(b.location);
|
|
20433
|
+
});
|
|
20434
|
+
return {
|
|
20435
|
+
summary,
|
|
20436
|
+
failures,
|
|
20437
|
+
regressed,
|
|
20438
|
+
fixed,
|
|
20439
|
+
comparedToBaseline: baseline !== void 0
|
|
20440
|
+
};
|
|
20441
|
+
}
|
|
20442
|
+
function toFailure(tc, baseline) {
|
|
20443
|
+
const failedIndexes = new Set(
|
|
20444
|
+
tc.stepResults.filter((s) => s.status === "failed").map((s) => s.index)
|
|
20445
|
+
);
|
|
20446
|
+
const steps = tc.story.steps.map((step, index) => ({
|
|
20447
|
+
keyword: step.keyword,
|
|
20448
|
+
text: step.text,
|
|
20449
|
+
failed: failedIndexes.has(index)
|
|
20450
|
+
}));
|
|
20451
|
+
return {
|
|
20452
|
+
id: tc.id,
|
|
20453
|
+
scenario: tc.story.scenario,
|
|
20454
|
+
location: `${tc.sourceFile}:${tc.sourceLine}`,
|
|
20455
|
+
steps,
|
|
20456
|
+
errorMessage: failingScenarioMessage(tc),
|
|
20457
|
+
covers: tc.story.covers ?? [],
|
|
20458
|
+
tickets: (tc.story.tickets ?? []).map((t) => t.id),
|
|
20459
|
+
regressed: baseline?.get(tc.id) === "passed"
|
|
20460
|
+
};
|
|
20461
|
+
}
|
|
20462
|
+
function renderCheck(report, format) {
|
|
20463
|
+
return format === "json" ? JSON.stringify(report, null, 2) : renderCheckText(report);
|
|
20464
|
+
}
|
|
20465
|
+
function renderCheckText(report) {
|
|
20466
|
+
const { summary, failures } = report;
|
|
20467
|
+
const headlineParts = [`${ICON_PASS} ${summary.passed} passed`];
|
|
20468
|
+
if (summary.failed > 0) headlineParts.push(`${ICON_FAIL} ${summary.failed} failed`);
|
|
20469
|
+
if (summary.skipped > 0) headlineParts.push(`${ICON_SKIP} ${summary.skipped} skipped`);
|
|
20470
|
+
if (summary.pending > 0) headlineParts.push(`${ICON_PENDING} ${summary.pending} pending`);
|
|
20471
|
+
const headline = `${headlineParts.join(" ")} (${summary.total} scenarios)`;
|
|
20472
|
+
if (failures.length === 0) {
|
|
20473
|
+
const lines2 = [headline];
|
|
20474
|
+
if (report.comparedToBaseline && report.fixed > 0) {
|
|
20475
|
+
lines2.push(`${ICON_PASS} ${report.fixed} fixed since baseline.`);
|
|
20476
|
+
}
|
|
20477
|
+
lines2.push("All scenarios green.");
|
|
20478
|
+
return lines2.join("\n");
|
|
20479
|
+
}
|
|
20480
|
+
const lines = [headline, ""];
|
|
20481
|
+
for (const f of failures) {
|
|
20482
|
+
lines.push(`${ICON_FAIL} ${f.scenario}${f.regressed ? " (regressed)" : ""}`);
|
|
20483
|
+
lines.push(` ${f.location}`);
|
|
20484
|
+
for (const step of f.steps) {
|
|
20485
|
+
const marker = step.failed ? ` ${ICON_FAIL} ` : " ";
|
|
20486
|
+
lines.push(`${marker}${step.keyword} ${step.text}`);
|
|
20487
|
+
}
|
|
20488
|
+
if (f.errorMessage) {
|
|
20489
|
+
const firstLine = f.errorMessage.split("\n")[0];
|
|
20490
|
+
lines.push(` \u2192 ${firstLine}`);
|
|
20491
|
+
}
|
|
20492
|
+
if (f.covers.length > 0) {
|
|
20493
|
+
lines.push(` covers: ${f.covers.join(", ")}`);
|
|
20494
|
+
}
|
|
20495
|
+
if (f.tickets.length > 0) {
|
|
20496
|
+
lines.push(` ticket: ${f.tickets.join(", ")}`);
|
|
20497
|
+
}
|
|
20498
|
+
lines.push("");
|
|
20499
|
+
}
|
|
20500
|
+
if (report.comparedToBaseline) {
|
|
20501
|
+
if (report.regressed > 0) {
|
|
20502
|
+
lines.push(`${ICON_WARN} ${report.regressed} regressed since baseline (was passing).`);
|
|
20503
|
+
}
|
|
20504
|
+
if (report.fixed > 0) {
|
|
20505
|
+
lines.push(`${ICON_PASS} ${report.fixed} fixed since baseline.`);
|
|
20506
|
+
}
|
|
20507
|
+
if (report.regressed === 0 && report.fixed === 0) {
|
|
20508
|
+
lines.push("No status changes vs. baseline.");
|
|
20509
|
+
}
|
|
20510
|
+
}
|
|
20511
|
+
return lines.join("\n").trimEnd();
|
|
20512
|
+
}
|
|
20513
|
+
|
|
20514
|
+
// src/goal.ts
|
|
20515
|
+
var ACTIVE = ["passed", "failed"];
|
|
20516
|
+
function buildGoal(args, _deps = {}) {
|
|
20517
|
+
const { run, baseline } = args;
|
|
20518
|
+
const cases = run.testCases;
|
|
20519
|
+
const selectors = [
|
|
20520
|
+
...args.requireTags.map((tag) => ({ label: `tag:${tag}`, match: (tc) => tc.tags.includes(tag) })),
|
|
20521
|
+
...args.requireTickets.map((id) => ({ label: `ticket:${id}`, match: (tc) => (tc.story.tickets ?? []).some((t) => t.id === id) })),
|
|
20522
|
+
...args.requireScenarios.map((sel) => ({ label: `scenario:${sel}`, match: (tc) => tc.id === sel || tc.story.scenario === sel }))
|
|
20523
|
+
];
|
|
20524
|
+
const requirements = selectors.length === 0 ? [evaluate("all scenarios", cases)] : selectors.map((s) => evaluate(s.label, cases.filter(s.match)));
|
|
20525
|
+
const regressions = [];
|
|
20526
|
+
if (baseline && args.enforceNoRegressions) {
|
|
20527
|
+
const before = statusMap(baseline);
|
|
20528
|
+
for (const tc of cases) {
|
|
20529
|
+
if (before.get(tc.id) === "passed" && tc.status === "failed") {
|
|
20530
|
+
regressions.push({ id: tc.id, title: tc.story.scenario });
|
|
20531
|
+
}
|
|
20532
|
+
}
|
|
20533
|
+
}
|
|
20534
|
+
const violations = [];
|
|
20535
|
+
if (baseline && args.enforceRatchet) {
|
|
20536
|
+
const current = new Map(cases.map((tc) => [tc.id, tc]));
|
|
20537
|
+
for (const base of baseline.testCases) {
|
|
20538
|
+
const now = current.get(base.id);
|
|
20539
|
+
if (!now) {
|
|
20540
|
+
violations.push({ id: base.id, title: base.story.scenario, kind: "removed", detail: "scenario no longer present" });
|
|
20541
|
+
continue;
|
|
20542
|
+
}
|
|
20543
|
+
if (ACTIVE.includes(base.status) && (now.status === "skipped" || now.status === "pending")) {
|
|
20544
|
+
violations.push({ id: base.id, title: base.story.scenario, kind: "disabled", detail: `${base.status} -> ${now.status}` });
|
|
20545
|
+
}
|
|
20546
|
+
const baseSteps = base.story.steps.length;
|
|
20547
|
+
const nowSteps = now.story.steps.length;
|
|
20548
|
+
if (nowSteps < baseSteps) {
|
|
20549
|
+
violations.push({ id: base.id, title: base.story.scenario, kind: "weakened", detail: `${baseSteps} steps -> ${nowSteps} steps` });
|
|
20550
|
+
}
|
|
20551
|
+
}
|
|
20552
|
+
}
|
|
20553
|
+
const met = requirements.every((r) => r.met) && regressions.length === 0 && violations.length === 0;
|
|
20554
|
+
return {
|
|
20555
|
+
met,
|
|
20556
|
+
requirements,
|
|
20557
|
+
regressions,
|
|
20558
|
+
regressionsEnforced: Boolean(baseline && args.enforceNoRegressions),
|
|
20559
|
+
ratchet: { enforced: Boolean(baseline && args.enforceRatchet), violations }
|
|
20560
|
+
};
|
|
20561
|
+
}
|
|
20562
|
+
function evaluate(selector, matched) {
|
|
20563
|
+
const passed = matched.filter((tc) => tc.status === "passed").length;
|
|
20564
|
+
const failing = matched.filter((tc) => tc.status !== "passed").map((tc) => tc.story.scenario);
|
|
20565
|
+
return {
|
|
20566
|
+
selector,
|
|
20567
|
+
matched: matched.length,
|
|
20568
|
+
passed,
|
|
20569
|
+
failing,
|
|
20570
|
+
met: matched.length > 0 && failing.length === 0
|
|
20571
|
+
};
|
|
20572
|
+
}
|
|
20573
|
+
function statusMap(run) {
|
|
20574
|
+
return new Map(run.testCases.map((tc) => [tc.id, tc.status]));
|
|
20575
|
+
}
|
|
20576
|
+
function renderGoal(report, format) {
|
|
20577
|
+
if (format === "json") return JSON.stringify(report, null, 2);
|
|
20578
|
+
const lines = [`GOAL: ${report.met ? "met" : "not met"}`];
|
|
20579
|
+
for (const req of report.requirements) {
|
|
20580
|
+
if (req.matched === 0) {
|
|
20581
|
+
lines.push(` ${req.selector}: no matching scenario (no proof)`);
|
|
20582
|
+
continue;
|
|
20583
|
+
}
|
|
20584
|
+
const tail = req.failing.length > 0 ? ` (${req.failing.length} failing)` : "";
|
|
20585
|
+
lines.push(` ${req.selector}: ${req.passed}/${req.matched} scenarios pass${tail}`);
|
|
20586
|
+
}
|
|
20587
|
+
if (report.regressionsEnforced) {
|
|
20588
|
+
if (report.regressions.length === 0) {
|
|
20589
|
+
lines.push(" regressions: 0");
|
|
20590
|
+
} else {
|
|
20591
|
+
lines.push(` regressions: ${report.regressions.length} (${report.regressions.map((r) => r.title).join(", ")})`);
|
|
20592
|
+
}
|
|
20593
|
+
}
|
|
20594
|
+
if (report.ratchet.enforced) {
|
|
20595
|
+
if (report.ratchet.violations.length === 0) {
|
|
20596
|
+
lines.push(" ratchet: clean (0 scenarios removed/weakened)");
|
|
20597
|
+
} else {
|
|
20598
|
+
lines.push(` ratchet: ${report.ratchet.violations.length} removed/weakened`);
|
|
20599
|
+
for (const v of report.ratchet.violations) {
|
|
20600
|
+
lines.push(` ${v.kind}: ${v.title} (${v.detail})`);
|
|
20601
|
+
}
|
|
20602
|
+
}
|
|
20603
|
+
}
|
|
20604
|
+
return lines.join("\n");
|
|
20605
|
+
}
|
|
20606
|
+
|
|
20607
|
+
// src/triage.ts
|
|
20608
|
+
function buildTriage(args, _deps = {}) {
|
|
20609
|
+
const { testCases, baseline } = args;
|
|
20610
|
+
const failing = testCases.filter((tc) => tc.status === "failed");
|
|
20611
|
+
const ranked = failing.map((tc) => {
|
|
20612
|
+
const regressed = baseline?.get(tc.id) === "passed";
|
|
20613
|
+
return {
|
|
20614
|
+
tc,
|
|
20615
|
+
regressed,
|
|
20616
|
+
covers: tc.story.covers ?? []
|
|
20617
|
+
};
|
|
20618
|
+
}).sort((a, b) => {
|
|
20619
|
+
if (a.regressed !== b.regressed) return a.regressed ? -1 : 1;
|
|
20620
|
+
const la = `${a.tc.sourceFile}:${a.tc.sourceLine}`;
|
|
20621
|
+
const lb = `${b.tc.sourceFile}:${b.tc.sourceLine}`;
|
|
20622
|
+
return la.localeCompare(lb);
|
|
20623
|
+
});
|
|
20624
|
+
const items = ranked.map((entry, index) => ({
|
|
20625
|
+
rank: index + 1,
|
|
20626
|
+
id: entry.tc.id,
|
|
20627
|
+
scenario: entry.tc.story.scenario,
|
|
20628
|
+
status: entry.tc.status,
|
|
20629
|
+
location: `${entry.tc.sourceFile}:${entry.tc.sourceLine}`,
|
|
20630
|
+
covers: entry.covers,
|
|
20631
|
+
tickets: (entry.tc.story.tickets ?? []).map((t) => t.id),
|
|
20632
|
+
errorMessage: failingScenarioMessage(entry.tc),
|
|
20633
|
+
regressed: entry.regressed,
|
|
20634
|
+
reason: entry.regressed ? "regression" : "failing"
|
|
20635
|
+
}));
|
|
20636
|
+
return {
|
|
20637
|
+
total: testCases.length,
|
|
20638
|
+
failing: failing.length,
|
|
20639
|
+
regressions: items.filter((i) => i.regressed).length,
|
|
20640
|
+
needsCovers: items.filter((i) => i.covers.length === 0).length,
|
|
20641
|
+
items
|
|
20642
|
+
};
|
|
20643
|
+
}
|
|
20644
|
+
function renderTriage(report, format) {
|
|
20645
|
+
if (format === "json") return JSON.stringify(report, null, 2);
|
|
20646
|
+
if (report.items.length === 0) {
|
|
20647
|
+
return "Nothing to triage. No failing scenarios.";
|
|
20648
|
+
}
|
|
20649
|
+
const header = report.regressions > 0 ? `${report.items.length} items to triage (${report.regressions} regression${report.regressions === 1 ? "" : "s"})` : `${report.items.length} items to triage`;
|
|
20650
|
+
const lines = [header, ""];
|
|
20651
|
+
for (const item of report.items) {
|
|
20652
|
+
const tag = item.regressed ? "[regression] " : "";
|
|
20653
|
+
lines.push(`${item.rank}. ${tag}${item.scenario}`);
|
|
20654
|
+
lines.push(` ${item.location}`);
|
|
20655
|
+
if (item.errorMessage) {
|
|
20656
|
+
lines.push(` \u2192 ${item.errorMessage.split("\n")[0]}`);
|
|
20657
|
+
}
|
|
20658
|
+
if (item.covers.length > 0) {
|
|
20659
|
+
lines.push(` fix: ${item.covers.join(", ")}`);
|
|
20660
|
+
} else {
|
|
20661
|
+
lines.push(" fix: (no covers declared \u2014 add `covers` to route this to code)");
|
|
20662
|
+
}
|
|
20663
|
+
if (item.tickets.length > 0) {
|
|
20664
|
+
lines.push(` ticket: ${item.tickets.join(", ")}`);
|
|
20665
|
+
}
|
|
20666
|
+
lines.push("");
|
|
20667
|
+
}
|
|
20668
|
+
if (report.needsCovers > 0) {
|
|
20669
|
+
lines.push(`${report.needsCovers} failing scenario(s) have no covers and can't be routed to code automatically.`);
|
|
20670
|
+
}
|
|
20671
|
+
return lines.join("\n").trimEnd();
|
|
20672
|
+
}
|
|
20673
|
+
|
|
20265
20674
|
// src/review/conventions.ts
|
|
20266
20675
|
var CHANGE_TAG_PREFIX = "change:";
|
|
20267
20676
|
var AUDIENCE_TAG_PREFIX = "audience:";
|
|
@@ -21063,6 +21472,7 @@ var FORMAT_EXTENSIONS = {
|
|
|
21063
21472
|
"behavior-manifest-json": ".behavior-manifest.json",
|
|
21064
21473
|
markdown: ".md",
|
|
21065
21474
|
"release-manifest": ".release-manifest.md",
|
|
21475
|
+
"traceability-matrix": ".traceability-matrix.md",
|
|
21066
21476
|
html: ".html",
|
|
21067
21477
|
"cucumber-html": ".cucumber.html",
|
|
21068
21478
|
junit: ".junit.xml",
|
|
@@ -21489,6 +21899,10 @@ var ReportGenerator = class {
|
|
|
21489
21899
|
const formatter = new ReleaseManifestFormatter();
|
|
21490
21900
|
return formatter.format(run);
|
|
21491
21901
|
}
|
|
21902
|
+
case "traceability-matrix": {
|
|
21903
|
+
const formatter = new TraceabilityMatrixFormatter();
|
|
21904
|
+
return formatter.format(run);
|
|
21905
|
+
}
|
|
21492
21906
|
case "story-report-json": {
|
|
21493
21907
|
const formatter = new StoryReportJsonFormatter({
|
|
21494
21908
|
pretty: this.options.storyReportJson.pretty
|
|
@@ -22286,6 +22700,7 @@ var EXIT_GENERATION = 3;
|
|
|
22286
22700
|
var EXIT_USAGE = 4;
|
|
22287
22701
|
var EXIT_COMPARE_GATE = 5;
|
|
22288
22702
|
var EXIT_REVIEW_GATE = 5;
|
|
22703
|
+
var EXIT_AGENT_GATE = 5;
|
|
22289
22704
|
var EXIT_RELEASE_GATE = 6;
|
|
22290
22705
|
var HELP_TEXT = `
|
|
22291
22706
|
executable-stories \u2014 Generate reports from test results JSON.
|
|
@@ -22297,6 +22712,9 @@ USAGE
|
|
|
22297
22712
|
executable-stories gate-release <dev-run.json> <rc-run.json> [options]
|
|
22298
22713
|
executable-stories review <file> --changed-files <path> [options]
|
|
22299
22714
|
executable-stories list <file> [options]
|
|
22715
|
+
executable-stories check <file> [--baseline <path|auto>] [--check-format text|json] [--no-fail]
|
|
22716
|
+
executable-stories goal <file> [--require-tags <csv>] [--require-tickets <csv>] [--require-scenarios <csv>] [--baseline <path|auto>] [--no-regressions] [--goal-format text|json]
|
|
22717
|
+
executable-stories triage <file> [--baseline <path|auto>] [--triage-format text|json]
|
|
22300
22718
|
executable-stories validate <file>
|
|
22301
22719
|
executable-stories validate --stdin
|
|
22302
22720
|
executable-stories init-astro [directory]
|
|
@@ -22316,6 +22734,9 @@ SUBCOMMANDS
|
|
|
22316
22734
|
gate-release Verify a release candidate against the dev test baseline (RC gate)
|
|
22317
22735
|
review Generate an Evidence Review of AI-authored changes (correlate a run to the diff)
|
|
22318
22736
|
list List scenarios from a test run (text table or JSON)
|
|
22737
|
+
check Backpressure summary: compress passing, expand failing (GWT + error + covers); non-zero exit on failures
|
|
22738
|
+
goal Behavioral definition-of-done for agent loops: required scenarios pass, no regressions, no weakened scenarios (exit 0 = met, 5 = not)
|
|
22739
|
+
triage Discovery worklist for agent loops: failing scenarios, regressions first, each with the code it covers
|
|
22319
22740
|
validate Validate a JSON file against the schema (no output generated)
|
|
22320
22741
|
init-astro Scaffold an Astro docs site for story output (Starlight with themed CSS)
|
|
22321
22742
|
new Scaffold a docs page from a template (adr, runbook, decision-log, incident)
|
|
@@ -22326,7 +22747,7 @@ SUBCOMMANDS
|
|
|
22326
22747
|
deploy Record deployments, show environment status, detect drift
|
|
22327
22748
|
|
|
22328
22749
|
OPTIONS
|
|
22329
|
-
--format <formats> Comma-separated formats: html, markdown, release-manifest, junit, cucumber-json, cucumber-messages, cucumber-html, astro, confluence, story-report-json, scenario-index-json, behavior-manifest-json, or custom names from config (default: html)
|
|
22750
|
+
--format <formats> Comma-separated formats: html, markdown, release-manifest, traceability-matrix, junit, cucumber-json, cucumber-messages, cucumber-html, astro, confluence, story-report-json, scenario-index-json, behavior-manifest-json, or custom names from config (default: html)
|
|
22330
22751
|
astro Themed Markdown (for Astro docs sites with matching CSS)
|
|
22331
22752
|
confluence Atlassian Document Format (ADF) JSON for Confluence / Jira
|
|
22332
22753
|
behavior-manifest-json Agent-readable behavior manifest and debugger warnings
|
|
@@ -22338,6 +22759,7 @@ OPTIONS
|
|
|
22338
22759
|
cucumber-messages Raw NDJSON (Cucumber Messages)
|
|
22339
22760
|
story-report-json StoryReport v1 JSON (consumed by executable-stories-react and other UI renderers)
|
|
22340
22761
|
scenario-index-json Storybook-like scenario index for agents and explorers
|
|
22762
|
+
traceability-matrix Requirement-first matrix (ticket -> scenarios -> covered code -> status)
|
|
22341
22763
|
--config <path> Path to executable-stories.config.js (default: ./executable-stories.config.js)
|
|
22342
22764
|
--input-type <type> Input type: raw, canonical, or ndjson (default: raw)
|
|
22343
22765
|
--output-dir <dir> Output directory (default: reports)
|
|
@@ -22364,6 +22786,15 @@ OPTIONS
|
|
|
22364
22786
|
--stdin Read JSON from stdin instead of file
|
|
22365
22787
|
--list-format <format> list output format: text (default), json, csv, markdown-table
|
|
22366
22788
|
--json-summary Deprecated alias for --list-format json
|
|
22789
|
+
--check-format <format> check output format: text (default) or json
|
|
22790
|
+
--no-fail (check) Report only \u2014 always exit 0 even when scenarios failed
|
|
22791
|
+
--require-tags <csv> (goal) Every scenario carrying any of these tags must pass
|
|
22792
|
+
--require-tickets <csv> (goal) Every scenario carrying any of these tickets must pass
|
|
22793
|
+
--require-scenarios <csv> (goal) These scenarios (by id or exact title) must pass
|
|
22794
|
+
--no-regressions (goal) Not met if any scenario regressed vs --baseline
|
|
22795
|
+
--no-ratchet (goal) Disable the removed/weakened-scenario guard (on by default with --baseline)
|
|
22796
|
+
--goal-format <format> goal output format: text (default) or json
|
|
22797
|
+
--triage-format <format> triage output format: text (default) or json
|
|
22367
22798
|
--baseline <path|auto> Compare baseline file, or auto-pick a prior run for compare
|
|
22368
22799
|
--baseline-dir <dir> Directory to scan when --baseline auto is used
|
|
22369
22800
|
--pr-summary Print a PR-friendly markdown summary after compare
|
|
@@ -22388,6 +22819,31 @@ LIST
|
|
|
22388
22819
|
list supports --include-tags, --exclude-tags for filtering
|
|
22389
22820
|
list supports --input-type and --stdin
|
|
22390
22821
|
|
|
22822
|
+
CHECK
|
|
22823
|
+
check is the inner-loop "backpressure" view for coding agents: run it after tests.
|
|
22824
|
+
Passing scenarios collapse to a single count line; each failing scenario expands
|
|
22825
|
+
to its Given/When/Then steps, the step that broke, the error, and the product
|
|
22826
|
+
code it covers \u2014 so the agent gets an actionable signal, not a wall of green.
|
|
22827
|
+
check exits 5 when any scenario failed (so the agent loop pushes back); pass
|
|
22828
|
+
--no-fail to report only. --baseline <path|auto> adds "N regressed / N fixed"
|
|
22829
|
+
since the prior run. --check-format json emits the structured report.
|
|
22830
|
+
|
|
22831
|
+
GOAL
|
|
22832
|
+
goal is the behavioral stopping condition for an agent loop (the /goal pattern).
|
|
22833
|
+
It is "met" when the required scenarios pass, nothing regressed (with
|
|
22834
|
+
--no-regressions), and no scenario was removed, disabled, or had steps deleted
|
|
22835
|
+
versus --baseline (the ratchet, on by default when a baseline is given). Declare
|
|
22836
|
+
the target with --require-tags / --require-tickets / --require-scenarios; with
|
|
22837
|
+
none given, the goal is "every scenario passes". Exit 0 means met, 5 means not
|
|
22838
|
+
yet, so a loop can run until the verdict flips. --goal-format json for machines.
|
|
22839
|
+
|
|
22840
|
+
TRIAGE
|
|
22841
|
+
triage is the discovery-phase worklist for a loop. It lists failing scenarios,
|
|
22842
|
+
regressions first (with --baseline), each with the product code it covers, the
|
|
22843
|
+
error, and its tickets, so the loop can route each fix to a sub-agent. Failures
|
|
22844
|
+
with no covers are flagged. --triage-format json emits the work queue. triage
|
|
22845
|
+
always exits 0 \u2014 it reports work, it does not gate.
|
|
22846
|
+
|
|
22391
22847
|
COMPARE
|
|
22392
22848
|
compare supports --format html,markdown
|
|
22393
22849
|
compare uses the same --input-type for both baseline and current files
|
|
@@ -22460,9 +22916,16 @@ EXIT CODES
|
|
|
22460
22916
|
2 Canonical validation failure
|
|
22461
22917
|
3 Formatter/generation failure
|
|
22462
22918
|
4 Bad arguments / usage error
|
|
22463
|
-
5 Compare gate failed
|
|
22919
|
+
5 Compare / review / check gate failed
|
|
22464
22920
|
6 Release gate failed
|
|
22465
22921
|
`.trim();
|
|
22922
|
+
function parseTextJsonFormat(flag, value) {
|
|
22923
|
+
if (value !== "text" && value !== "json") {
|
|
22924
|
+
console.error(`Error: ${flag} must be "text" or "json", got "${value}".`);
|
|
22925
|
+
process.exit(EXIT_USAGE);
|
|
22926
|
+
}
|
|
22927
|
+
return value;
|
|
22928
|
+
}
|
|
22466
22929
|
async function parseCliArgs(argv) {
|
|
22467
22930
|
const args = argv.slice(2);
|
|
22468
22931
|
if (args.length === 0 || args.includes("--help") || args.includes("-h")) {
|
|
@@ -22470,9 +22933,9 @@ async function parseCliArgs(argv) {
|
|
|
22470
22933
|
process.exit(EXIT_SUCCESS);
|
|
22471
22934
|
}
|
|
22472
22935
|
const subcommand = args[0];
|
|
22473
|
-
if (subcommand !== "format" && subcommand !== "watch" && subcommand !== "compare" && subcommand !== "gate-release" && subcommand !== "deploy" && subcommand !== "review" && subcommand !== "list" && subcommand !== "validate" && subcommand !== "init-astro" && subcommand !== "build-docs" && subcommand !== "new" && subcommand !== "check-links" && subcommand !== "import-openapi" && subcommand !== "publish-confluence" && subcommand !== "publish-jira") {
|
|
22936
|
+
if (subcommand !== "format" && subcommand !== "watch" && subcommand !== "compare" && subcommand !== "gate-release" && subcommand !== "deploy" && subcommand !== "review" && subcommand !== "list" && subcommand !== "check" && subcommand !== "goal" && subcommand !== "triage" && subcommand !== "validate" && subcommand !== "init-astro" && subcommand !== "build-docs" && subcommand !== "new" && subcommand !== "check-links" && subcommand !== "import-openapi" && subcommand !== "publish-confluence" && subcommand !== "publish-jira") {
|
|
22474
22937
|
console.error(
|
|
22475
|
-
`Unknown subcommand: "${subcommand}". Use "format", "watch", "compare", "gate-release", "deploy", "review", "list", "validate", "init-astro", "build-docs", "new", "check-links", "import-openapi", "publish-confluence", or "publish-jira".`
|
|
22938
|
+
`Unknown subcommand: "${subcommand}". Use "format", "watch", "compare", "gate-release", "deploy", "review", "list", "check", "goal", "triage", "validate", "init-astro", "build-docs", "new", "check-links", "import-openapi", "publish-confluence", or "publish-jira".`
|
|
22476
22939
|
);
|
|
22477
22940
|
process.exit(EXIT_USAGE);
|
|
22478
22941
|
}
|
|
@@ -22560,6 +23023,15 @@ async function parseCliArgs(argv) {
|
|
|
22560
23023
|
stdin: { type: "boolean", default: false },
|
|
22561
23024
|
"json-summary": { type: "boolean", default: false },
|
|
22562
23025
|
"list-format": { type: "string", default: "text" },
|
|
23026
|
+
"check-format": { type: "string", default: "text" },
|
|
23027
|
+
"no-fail": { type: "boolean", default: false },
|
|
23028
|
+
"require-tags": { type: "string" },
|
|
23029
|
+
"require-tickets": { type: "string" },
|
|
23030
|
+
"require-scenarios": { type: "string" },
|
|
23031
|
+
"no-regressions": { type: "boolean", default: false },
|
|
23032
|
+
"no-ratchet": { type: "boolean", default: false },
|
|
23033
|
+
"goal-format": { type: "string", default: "text" },
|
|
23034
|
+
"triage-format": { type: "string", default: "text" },
|
|
22563
23035
|
"emit-canonical": { type: "string" },
|
|
22564
23036
|
"slack-webhook": { type: "string" },
|
|
22565
23037
|
"teams-webhook": { type: "string" },
|
|
@@ -22630,7 +23102,7 @@ async function parseCliArgs(argv) {
|
|
|
22630
23102
|
}
|
|
22631
23103
|
const pluginConfig = await loadConfig(values["config"]);
|
|
22632
23104
|
const customFormatterNames = new Set(Object.keys(pluginConfig.formatters ?? {}));
|
|
22633
|
-
const builtInFormats = /* @__PURE__ */ new Set(["astro", "behavior-manifest-json", "confluence", "html", "markdown", "release-manifest", "junit", "cucumber-json", "cucumber-messages", "cucumber-html", "scenario-index-json", "story-report-json"]);
|
|
23105
|
+
const builtInFormats = /* @__PURE__ */ new Set(["astro", "behavior-manifest-json", "confluence", "html", "markdown", "release-manifest", "traceability-matrix", "junit", "cucumber-json", "cucumber-messages", "cucumber-html", "scenario-index-json", "story-report-json"]);
|
|
22634
23106
|
const formatStr = values.format;
|
|
22635
23107
|
const allRequestedFormats = formatStr.split(",").map((f) => f.trim());
|
|
22636
23108
|
const builtInRequested = allRequestedFormats.filter((f) => builtInFormats.has(f));
|
|
@@ -22638,7 +23110,7 @@ async function parseCliArgs(argv) {
|
|
|
22638
23110
|
const unknownFormats = allRequestedFormats.filter((f) => !builtInFormats.has(f) && !customFormatterNames.has(f));
|
|
22639
23111
|
if (unknownFormats.length > 0) {
|
|
22640
23112
|
const knownCustom = customFormatterNames.size > 0 ? `, ${[...customFormatterNames].join(", ")}` : "";
|
|
22641
|
-
console.error(`Error: Unknown format(s): ${unknownFormats.join(", ")}. Valid built-in: astro, behavior-manifest-json, confluence, html, markdown, release-manifest, junit, cucumber-json, cucumber-messages, cucumber-html, scenario-index-json, story-report-json${knownCustom}.`);
|
|
23113
|
+
console.error(`Error: Unknown format(s): ${unknownFormats.join(", ")}. Valid built-in: astro, behavior-manifest-json, confluence, html, markdown, release-manifest, traceability-matrix, junit, cucumber-json, cucumber-messages, cucumber-html, scenario-index-json, story-report-json${knownCustom}.`);
|
|
22642
23114
|
process.exit(EXIT_USAGE);
|
|
22643
23115
|
}
|
|
22644
23116
|
const formats = builtInRequested;
|
|
@@ -22726,10 +23198,14 @@ async function parseCliArgs(argv) {
|
|
|
22726
23198
|
console.error(`Error: --min-evidence must be "weak", "moderate", or "strong", got "${minEvidenceRaw}".`);
|
|
22727
23199
|
process.exit(EXIT_USAGE);
|
|
22728
23200
|
}
|
|
23201
|
+
const checkFormat = parseTextJsonFormat("--check-format", values["check-format"]);
|
|
23202
|
+
const goalFormat = parseTextJsonFormat("--goal-format", values["goal-format"]);
|
|
23203
|
+
const triageFormat = parseTextJsonFormat("--triage-format", values["triage-format"]);
|
|
22729
23204
|
const cliArgs = {
|
|
22730
23205
|
subcommand,
|
|
22731
23206
|
inputFile,
|
|
22732
23207
|
baselineFile,
|
|
23208
|
+
baselineArg: baselineValue,
|
|
22733
23209
|
currentFile,
|
|
22734
23210
|
baselineMode,
|
|
22735
23211
|
baselineDir: values["baseline-dir"],
|
|
@@ -22756,6 +23232,15 @@ async function parseCliArgs(argv) {
|
|
|
22756
23232
|
htmlThemePicker: values["html-theme-picker"],
|
|
22757
23233
|
jsonSummary: values["json-summary"],
|
|
22758
23234
|
listFormat: values["list-format"],
|
|
23235
|
+
checkFormat,
|
|
23236
|
+
noFail: values["no-fail"],
|
|
23237
|
+
requireTags: parseGlobs(values["require-tags"]),
|
|
23238
|
+
requireTickets: parseGlobs(values["require-tickets"]),
|
|
23239
|
+
requireScenarios: parseGlobs(values["require-scenarios"]),
|
|
23240
|
+
noRegressions: values["no-regressions"],
|
|
23241
|
+
noRatchet: values["no-ratchet"],
|
|
23242
|
+
goalFormat,
|
|
23243
|
+
triageFormat,
|
|
22759
23244
|
emitCanonical: values["emit-canonical"],
|
|
22760
23245
|
slackWebhook,
|
|
22761
23246
|
teamsWebhook,
|
|
@@ -22972,6 +23457,24 @@ function resolveBaselineAuto(currentFile, currentRun, args) {
|
|
|
22972
23457
|
}
|
|
22973
23458
|
return picked.file;
|
|
22974
23459
|
}
|
|
23460
|
+
function resolveBaselineRun(args, currentRun) {
|
|
23461
|
+
if (!args.baselineArg) return void 0;
|
|
23462
|
+
let baselineFile;
|
|
23463
|
+
if (args.baselineArg === "auto") {
|
|
23464
|
+
if (!args.inputFile) {
|
|
23465
|
+
console.error("Error: --baseline auto requires a current input file (not --stdin).");
|
|
23466
|
+
process.exit(EXIT_USAGE);
|
|
23467
|
+
}
|
|
23468
|
+
baselineFile = resolveBaselineAuto(args.inputFile, currentRun, args);
|
|
23469
|
+
} else {
|
|
23470
|
+
baselineFile = args.baselineArg;
|
|
23471
|
+
}
|
|
23472
|
+
return applySelection(normalizeRunFromText(readFileInput(baselineFile), args).run, args);
|
|
23473
|
+
}
|
|
23474
|
+
function resolveBaselineStatusMap(args, currentRun) {
|
|
23475
|
+
const baselineRun = resolveBaselineRun(args, currentRun);
|
|
23476
|
+
return baselineRun ? new Map(baselineRun.testCases.map((tc) => [tc.id, tc.status])) : void 0;
|
|
23477
|
+
}
|
|
22975
23478
|
async function main() {
|
|
22976
23479
|
const { args, pluginConfig, customRequested } = await parseCliArgs(process.argv);
|
|
22977
23480
|
const startMs = Date.now();
|
|
@@ -23082,6 +23585,51 @@ async function main() {
|
|
|
23082
23585
|
console.log(output);
|
|
23083
23586
|
process.exit(EXIT_SUCCESS);
|
|
23084
23587
|
}
|
|
23588
|
+
if (args.subcommand === "check") {
|
|
23589
|
+
const text3 = await readInput(args);
|
|
23590
|
+
const run = applySelection(normalizeRunFromText(text3, args).run, args);
|
|
23591
|
+
const baseline = resolveBaselineStatusMap(args, run);
|
|
23592
|
+
const report = buildCheck(
|
|
23593
|
+
{ testCases: run.testCases, baseline, format: args.checkFormat },
|
|
23594
|
+
{}
|
|
23595
|
+
);
|
|
23596
|
+
console.log(renderCheck(report, args.checkFormat));
|
|
23597
|
+
if (report.summary.failed > 0 && !args.noFail) {
|
|
23598
|
+
process.exit(EXIT_AGENT_GATE);
|
|
23599
|
+
}
|
|
23600
|
+
process.exit(EXIT_SUCCESS);
|
|
23601
|
+
}
|
|
23602
|
+
if (args.subcommand === "goal") {
|
|
23603
|
+
const text3 = await readInput(args);
|
|
23604
|
+
const run = applySelection(normalizeRunFromText(text3, args).run, args);
|
|
23605
|
+
const baseline = resolveBaselineRun(args, run);
|
|
23606
|
+
const report = buildGoal(
|
|
23607
|
+
{
|
|
23608
|
+
run,
|
|
23609
|
+
baseline,
|
|
23610
|
+
requireTags: args.requireTags,
|
|
23611
|
+
requireTickets: args.requireTickets,
|
|
23612
|
+
requireScenarios: args.requireScenarios,
|
|
23613
|
+
enforceNoRegressions: args.noRegressions,
|
|
23614
|
+
enforceRatchet: !args.noRatchet,
|
|
23615
|
+
format: args.goalFormat
|
|
23616
|
+
},
|
|
23617
|
+
{}
|
|
23618
|
+
);
|
|
23619
|
+
console.log(renderGoal(report, args.goalFormat));
|
|
23620
|
+
process.exit(report.met ? EXIT_SUCCESS : EXIT_AGENT_GATE);
|
|
23621
|
+
}
|
|
23622
|
+
if (args.subcommand === "triage") {
|
|
23623
|
+
const text3 = await readInput(args);
|
|
23624
|
+
const run = applySelection(normalizeRunFromText(text3, args).run, args);
|
|
23625
|
+
const baseline = resolveBaselineStatusMap(args, run);
|
|
23626
|
+
const report = buildTriage(
|
|
23627
|
+
{ testCases: run.testCases, baseline, format: args.triageFormat },
|
|
23628
|
+
{}
|
|
23629
|
+
);
|
|
23630
|
+
console.log(renderTriage(report, args.triageFormat));
|
|
23631
|
+
process.exit(EXIT_SUCCESS);
|
|
23632
|
+
}
|
|
23085
23633
|
if (args.subcommand === "watch") {
|
|
23086
23634
|
if (!args.inputFile) {
|
|
23087
23635
|
console.error("Error: watch requires an input file (the raw-run JSON the framework writes).");
|
|
@@ -24200,6 +24748,15 @@ function createDefaultCliArgs() {
|
|
|
24200
24748
|
htmlThemePicker: false,
|
|
24201
24749
|
jsonSummary: false,
|
|
24202
24750
|
listFormat: "text",
|
|
24751
|
+
checkFormat: "text",
|
|
24752
|
+
noFail: false,
|
|
24753
|
+
requireTags: [],
|
|
24754
|
+
requireTickets: [],
|
|
24755
|
+
requireScenarios: [],
|
|
24756
|
+
noRegressions: false,
|
|
24757
|
+
noRatchet: false,
|
|
24758
|
+
goalFormat: "text",
|
|
24759
|
+
triageFormat: "text",
|
|
24203
24760
|
notify: "never",
|
|
24204
24761
|
maxFailedTests: 5,
|
|
24205
24762
|
maxHistoryRuns: 10,
|