@plune-ai/cli 0.17.1 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +25 -7
- package/dist/cli.cjs +209 -130
- package/dist/index.cjs +110 -76
- package/dist/index.d.cts +17 -0
- package/dist/index.d.ts +17 -0
- package/dist/index.js +110 -76
- package/dist/reporter-core.cjs +9 -4
- package/dist/reporter-core.d.cts +17 -3
- package/dist/reporter-core.d.ts +17 -3
- package/dist/reporter-core.js +9 -4
- package/package.json +1 -1
package/dist/cli.cjs
CHANGED
|
@@ -1621,6 +1621,76 @@ var init_concurrency = __esm({
|
|
|
1621
1621
|
}
|
|
1622
1622
|
});
|
|
1623
1623
|
|
|
1624
|
+
// src/types/test-case-schema.ts
|
|
1625
|
+
var import_zod2, TEST_CASE_SCHEMA_VERSION, externalKeySchema, manualStepSchema, assertionResultRecordSchema, manualCaseSchema, automatedCaseSchema, evalCaseSchema, testCaseBaseSchema, testCaseSchema;
|
|
1626
|
+
var init_test_case_schema = __esm({
|
|
1627
|
+
"src/types/test-case-schema.ts"() {
|
|
1628
|
+
"use strict";
|
|
1629
|
+
init_cjs_shims();
|
|
1630
|
+
import_zod2 = require("zod");
|
|
1631
|
+
TEST_CASE_SCHEMA_VERSION = 2;
|
|
1632
|
+
externalKeySchema = import_zod2.z.object({
|
|
1633
|
+
kind: import_zod2.z.enum(["playwright-id", "path-title", "allure-history", "cairn-stable", "eval-id", "qase", "testrail"]),
|
|
1634
|
+
value: import_zod2.z.string().min(1)
|
|
1635
|
+
});
|
|
1636
|
+
manualStepSchema = import_zod2.z.object({
|
|
1637
|
+
action: import_zod2.z.string().min(1),
|
|
1638
|
+
expected: import_zod2.z.string().min(1).optional()
|
|
1639
|
+
});
|
|
1640
|
+
assertionResultRecordSchema = import_zod2.z.object({
|
|
1641
|
+
type: import_zod2.z.string().min(1),
|
|
1642
|
+
passed: import_zod2.z.boolean(),
|
|
1643
|
+
score: import_zod2.z.number().optional(),
|
|
1644
|
+
reason: import_zod2.z.string().optional()
|
|
1645
|
+
});
|
|
1646
|
+
manualCaseSchema = import_zod2.z.object({
|
|
1647
|
+
type: import_zod2.z.literal("manual"),
|
|
1648
|
+
steps: import_zod2.z.array(manualStepSchema).min(1),
|
|
1649
|
+
expected: import_zod2.z.string().min(1).optional()
|
|
1650
|
+
});
|
|
1651
|
+
automatedCaseSchema = import_zod2.z.object({
|
|
1652
|
+
type: import_zod2.z.literal("automated"),
|
|
1653
|
+
specRef: import_zod2.z.string().min(1)
|
|
1654
|
+
});
|
|
1655
|
+
evalCaseSchema = import_zod2.z.object({
|
|
1656
|
+
type: import_zod2.z.literal("eval"),
|
|
1657
|
+
prompt: import_zod2.z.string().min(1),
|
|
1658
|
+
goldenSetId: import_zod2.z.string().min(1).optional(),
|
|
1659
|
+
assertions: import_zod2.z.array(assertionResultRecordSchema).min(1)
|
|
1660
|
+
});
|
|
1661
|
+
testCaseBaseSchema = import_zod2.z.object({
|
|
1662
|
+
schemaVersion: import_zod2.z.literal(TEST_CASE_SCHEMA_VERSION),
|
|
1663
|
+
id: import_zod2.z.string().min(1),
|
|
1664
|
+
externalKeys: import_zod2.z.array(externalKeySchema).optional(),
|
|
1665
|
+
/** ADR 0045 / plune#1246 — optional, purely additive; schemaVersion stays 2. */
|
|
1666
|
+
is_flaky: import_zod2.z.boolean().optional(),
|
|
1667
|
+
/** ADR 0045 / plune#1246 — optional, purely additive; schemaVersion stays 2. */
|
|
1668
|
+
is_muted: import_zod2.z.boolean().optional()
|
|
1669
|
+
});
|
|
1670
|
+
testCaseSchema = import_zod2.z.discriminatedUnion("type", [
|
|
1671
|
+
testCaseBaseSchema.merge(manualCaseSchema),
|
|
1672
|
+
testCaseBaseSchema.merge(automatedCaseSchema),
|
|
1673
|
+
testCaseBaseSchema.merge(evalCaseSchema)
|
|
1674
|
+
]);
|
|
1675
|
+
}
|
|
1676
|
+
});
|
|
1677
|
+
|
|
1678
|
+
// src/types/results.ts
|
|
1679
|
+
function rowStatus(row) {
|
|
1680
|
+
if (row.inconclusive !== void 0) return row.inconclusive;
|
|
1681
|
+
if (row.error !== void 0) return "errored";
|
|
1682
|
+
return row.assertions.every((a) => a.passed) ? "passed" : "failed";
|
|
1683
|
+
}
|
|
1684
|
+
var ROW_INCONCLUSIVE;
|
|
1685
|
+
var init_results = __esm({
|
|
1686
|
+
"src/types/results.ts"() {
|
|
1687
|
+
"use strict";
|
|
1688
|
+
init_cjs_shims();
|
|
1689
|
+
init_test_case_schema();
|
|
1690
|
+
ROW_INCONCLUSIVE = ["blocked", "skipped"];
|
|
1691
|
+
}
|
|
1692
|
+
});
|
|
1693
|
+
|
|
1624
1694
|
// src/orchestrator/run.ts
|
|
1625
1695
|
function resolvePrompt(template, vars) {
|
|
1626
1696
|
return template.replace(/\{\{\s*(\w+)\s*\}\}/g, (_match, key) => {
|
|
@@ -1754,10 +1824,25 @@ function resolveTemplate(ev, baseDir) {
|
|
|
1754
1824
|
}
|
|
1755
1825
|
throw new RunConfigError(`Eval "${ev.id}" has neither prompt nor prompt_file`);
|
|
1756
1826
|
}
|
|
1757
|
-
function
|
|
1758
|
-
|
|
1759
|
-
|
|
1760
|
-
|
|
1827
|
+
function summarize(evals, durationMs) {
|
|
1828
|
+
const summary = {
|
|
1829
|
+
total: 0,
|
|
1830
|
+
passed: 0,
|
|
1831
|
+
failed: 0,
|
|
1832
|
+
errored: 0,
|
|
1833
|
+
blocked: 0,
|
|
1834
|
+
skipped: 0,
|
|
1835
|
+
cost_usd: 0,
|
|
1836
|
+
duration_ms: durationMs
|
|
1837
|
+
};
|
|
1838
|
+
for (const ev of evals) {
|
|
1839
|
+
for (const row of ev.rows) {
|
|
1840
|
+
summary.total += 1;
|
|
1841
|
+
summary[rowStatus(row)] += 1;
|
|
1842
|
+
summary.cost_usd += row.usage?.cost_usd ?? 0;
|
|
1843
|
+
}
|
|
1844
|
+
}
|
|
1845
|
+
return summary;
|
|
1761
1846
|
}
|
|
1762
1847
|
async function runOrchestration(config, options, deps) {
|
|
1763
1848
|
const started = deps.now();
|
|
@@ -1792,26 +1877,12 @@ async function runOrchestration(config, options, deps) {
|
|
|
1792
1877
|
noCache
|
|
1793
1878
|
})
|
|
1794
1879
|
);
|
|
1795
|
-
const passed = rowResults.every((r) =>
|
|
1880
|
+
const passed = rowResults.every((r) => rowStatus(r) === "passed");
|
|
1796
1881
|
evalResults.push({ id: ev.id, tags: ev.tags ?? [], rows: rowResults, passed });
|
|
1797
1882
|
if (options.bail === true && !passed) break;
|
|
1798
1883
|
}
|
|
1799
1884
|
const finished = deps.now();
|
|
1800
|
-
const summary =
|
|
1801
|
-
total: 0,
|
|
1802
|
-
passed: 0,
|
|
1803
|
-
failed: 0,
|
|
1804
|
-
errored: 0,
|
|
1805
|
-
cost_usd: 0,
|
|
1806
|
-
duration_ms: finished - started
|
|
1807
|
-
};
|
|
1808
|
-
for (const ev of evalResults) {
|
|
1809
|
-
for (const row of ev.rows) {
|
|
1810
|
-
summary.total += 1;
|
|
1811
|
-
summary[classify(row)] += 1;
|
|
1812
|
-
summary.cost_usd += row.usage?.cost_usd ?? 0;
|
|
1813
|
-
}
|
|
1814
|
-
}
|
|
1885
|
+
const summary = summarize(evalResults, finished - started);
|
|
1815
1886
|
return {
|
|
1816
1887
|
schemaVersion: 1,
|
|
1817
1888
|
plune_version: PLUNE_VERSION,
|
|
@@ -1834,6 +1905,7 @@ var init_run = __esm({
|
|
|
1834
1905
|
init_judge();
|
|
1835
1906
|
init_concurrency();
|
|
1836
1907
|
init_errors2();
|
|
1908
|
+
init_results();
|
|
1837
1909
|
DEFAULT_MAX_TOKENS2 = 1024;
|
|
1838
1910
|
PLUNE_VERSION = "0.1.0";
|
|
1839
1911
|
RunConfigError = class extends Error {
|
|
@@ -2027,8 +2099,10 @@ function renderConsole(result, opts = {}) {
|
|
|
2027
2099
|
const passedTxt = s.green(`${summary.passed} passed`);
|
|
2028
2100
|
const failedTxt = summary.failed > 0 ? s.red(`${summary.failed} failed`) : `${summary.failed} failed`;
|
|
2029
2101
|
const erroredTxt = summary.errored > 0 ? s.yellow(`${summary.errored} errored`) : `${summary.errored} errored`;
|
|
2102
|
+
const blockedTxt = (summary.blocked ?? 0) > 0 ? ` \xB7 ${s.yellow(`${summary.blocked} blocked`)}` : "";
|
|
2103
|
+
const skippedTxt = (summary.skipped ?? 0) > 0 ? ` \xB7 ${s.dim(`${summary.skipped} skipped`)}` : "";
|
|
2030
2104
|
lines.push(
|
|
2031
|
-
`${passedTxt} \xB7 ${failedTxt} \xB7 ${erroredTxt} \xB7 ${summary.total} total \xB7 $${summary.cost_usd.toFixed(4)} \xB7 ${summary.duration_ms}ms`
|
|
2105
|
+
`${passedTxt} \xB7 ${failedTxt} \xB7 ${erroredTxt}${blockedTxt}${skippedTxt} \xB7 ${summary.total} total \xB7 $${summary.cost_usd.toFixed(4)} \xB7 ${summary.duration_ms}ms`
|
|
2032
2106
|
);
|
|
2033
2107
|
lines.push("");
|
|
2034
2108
|
for (const ev of result.evals) {
|
|
@@ -2037,17 +2111,21 @@ function renderConsole(result, opts = {}) {
|
|
|
2037
2111
|
lines.push(`${mark} ${s.bold(ev.id)}${tags}`);
|
|
2038
2112
|
let passedCount = 0;
|
|
2039
2113
|
for (const row of ev.rows) {
|
|
2114
|
+
const status = rowStatus(row);
|
|
2115
|
+
if (status === "passed") {
|
|
2116
|
+
passedCount += 1;
|
|
2117
|
+
continue;
|
|
2118
|
+
}
|
|
2119
|
+
lines.push(` ${s.dim("vars:")} ${JSON.stringify(row.vars)}`);
|
|
2120
|
+
if (status === "blocked" || status === "skipped") {
|
|
2121
|
+
lines.push(` ${s.yellow(status)} (not judged)`);
|
|
2122
|
+
continue;
|
|
2123
|
+
}
|
|
2040
2124
|
if (row.error !== void 0) {
|
|
2041
|
-
lines.push(` ${s.dim("vars:")} ${JSON.stringify(row.vars)}`);
|
|
2042
2125
|
lines.push(` ${s.red("error:")} ${row.error.message}`);
|
|
2043
2126
|
continue;
|
|
2044
2127
|
}
|
|
2045
2128
|
const failed = row.assertions.filter((a) => !a.passed);
|
|
2046
|
-
if (failed.length === 0) {
|
|
2047
|
-
passedCount += 1;
|
|
2048
|
-
continue;
|
|
2049
|
-
}
|
|
2050
|
-
lines.push(` ${s.dim("vars:")} ${JSON.stringify(row.vars)}`);
|
|
2051
2129
|
if (row.output !== null) {
|
|
2052
2130
|
lines.push(` ${s.dim("output:")} ${truncate(row.output, maxOut)}`);
|
|
2053
2131
|
}
|
|
@@ -2068,6 +2146,7 @@ var init_console = __esm({
|
|
|
2068
2146
|
"src/reporters/console.ts"() {
|
|
2069
2147
|
"use strict";
|
|
2070
2148
|
init_cjs_shims();
|
|
2149
|
+
init_results();
|
|
2071
2150
|
init_style();
|
|
2072
2151
|
init_helpers();
|
|
2073
2152
|
DEFAULT_MAX_OUTPUT = 500;
|
|
@@ -2086,11 +2165,6 @@ var init_json = __esm({
|
|
|
2086
2165
|
});
|
|
2087
2166
|
|
|
2088
2167
|
// src/reporters/markdown.ts
|
|
2089
|
-
function classify2(row) {
|
|
2090
|
-
if (row.error !== void 0) return "errored";
|
|
2091
|
-
if (row.assertions.some((a) => !a.passed)) return "failed";
|
|
2092
|
-
return "passed";
|
|
2093
|
-
}
|
|
2094
2168
|
function fence(content) {
|
|
2095
2169
|
const runs = content.match(new RegExp(TICK + "+", "g"));
|
|
2096
2170
|
const longest = runs ? runs.reduce((m, r) => Math.max(m, r.length), 0) : 0;
|
|
@@ -2111,18 +2185,22 @@ function renderMarkdown(result, opts = {}) {
|
|
|
2111
2185
|
out.push(`| Passed | ${summary.passed} |`);
|
|
2112
2186
|
out.push(`| Failed | ${summary.failed} |`);
|
|
2113
2187
|
out.push(`| Errored | ${summary.errored} |`);
|
|
2188
|
+
if ((summary.blocked ?? 0) > 0) out.push(`| Blocked | ${summary.blocked} |`);
|
|
2189
|
+
if ((summary.skipped ?? 0) > 0) out.push(`| Skipped | ${summary.skipped} |`);
|
|
2114
2190
|
out.push(`| Cost (USD) | ${summary.cost_usd.toFixed(4)} |`);
|
|
2115
2191
|
out.push(`| Duration (ms) | ${summary.duration_ms} |`);
|
|
2116
2192
|
out.push("");
|
|
2117
2193
|
const failures = [];
|
|
2194
|
+
let allPassed = true;
|
|
2118
2195
|
for (const ev of result.evals) {
|
|
2119
2196
|
for (const row of ev.rows) {
|
|
2120
|
-
const state =
|
|
2121
|
-
if (state !== "passed")
|
|
2197
|
+
const state = rowStatus(row);
|
|
2198
|
+
if (state !== "passed") allPassed = false;
|
|
2199
|
+
if (state === "failed" || state === "errored") failures.push({ id: ev.id, row, state });
|
|
2122
2200
|
}
|
|
2123
2201
|
}
|
|
2124
2202
|
if (failures.length === 0) {
|
|
2125
|
-
out.push("All evals passed.");
|
|
2203
|
+
if (allPassed) out.push("All evals passed.");
|
|
2126
2204
|
return out.join("\n");
|
|
2127
2205
|
}
|
|
2128
2206
|
out.push("## Failures");
|
|
@@ -2153,6 +2231,7 @@ var init_markdown = __esm({
|
|
|
2153
2231
|
"src/reporters/markdown.ts"() {
|
|
2154
2232
|
"use strict";
|
|
2155
2233
|
init_cjs_shims();
|
|
2234
|
+
init_results();
|
|
2156
2235
|
init_helpers();
|
|
2157
2236
|
DEFAULT_MAX_OUTPUT2 = 500;
|
|
2158
2237
|
TICK = String.fromCharCode(96);
|
|
@@ -2298,7 +2377,7 @@ function retryAfterMs2(res) {
|
|
|
2298
2377
|
const seconds = Number(raw.trim());
|
|
2299
2378
|
return Number.isFinite(seconds) && seconds >= 0 ? seconds * 1e3 : null;
|
|
2300
2379
|
}
|
|
2301
|
-
function
|
|
2380
|
+
function classify(status) {
|
|
2302
2381
|
if (status === 401 || status === 403) return "auth";
|
|
2303
2382
|
if (status === 409) return "conflict";
|
|
2304
2383
|
if (status === 400) return "invalid";
|
|
@@ -2341,13 +2420,13 @@ function createClient(opts) {
|
|
|
2341
2420
|
}
|
|
2342
2421
|
}
|
|
2343
2422
|
const said = await detailOf(res);
|
|
2344
|
-
const kind =
|
|
2423
|
+
const kind = classify(res.status);
|
|
2345
2424
|
last = { kind, status: res.status, detail: scrub(said.detail), ...said.code !== void 0 ? { code: said.code } : {} };
|
|
2346
2425
|
if (kind !== "unavailable") break;
|
|
2347
2426
|
if (res.status === 429 && retryAfterMs2(res) === null) {
|
|
2348
2427
|
last.kind = "invalid";
|
|
2349
2428
|
last.detail = "quota exceeded (no Retry-After)";
|
|
2350
|
-
if (said.detail) last.detail += ` \u2014 ${said.detail}`;
|
|
2429
|
+
if (said.detail) last.detail += ` \u2014 ${scrub(said.detail)}`;
|
|
2351
2430
|
break;
|
|
2352
2431
|
}
|
|
2353
2432
|
if (attempt < RETRY_DELAYS_MS.length) {
|
|
@@ -3227,21 +3306,26 @@ async function startRun(passed, declared2) {
|
|
|
3227
3306
|
},
|
|
3228
3307
|
flush,
|
|
3229
3308
|
async finish(reason) {
|
|
3230
|
-
if (done) return;
|
|
3309
|
+
if (done) return void 0;
|
|
3231
3310
|
done = true;
|
|
3232
3311
|
await flush();
|
|
3233
3312
|
await offer();
|
|
3313
|
+
let frozen;
|
|
3234
3314
|
if (runId !== null && !offline) {
|
|
3235
3315
|
const out = await client.post(`/v1/runs/${runId}/events`, {
|
|
3236
3316
|
event: "finish",
|
|
3237
3317
|
...reason !== void 0 ? { reason } : {}
|
|
3238
3318
|
});
|
|
3239
3319
|
if (!out.ok) log(`plune: could not close run ${runId} (${out.detail || out.kind}).`);
|
|
3240
|
-
else
|
|
3241
|
-
|
|
3320
|
+
else {
|
|
3321
|
+
frozen = out.body?.summary ?? void 0;
|
|
3322
|
+
if (typeof out.body?.detached === "number" && out.body.detached > 0) {
|
|
3323
|
+
log(`plune: ${out.body.detached} test case(s) marked detached \u2014 this full run no longer reports them.`);
|
|
3324
|
+
}
|
|
3242
3325
|
}
|
|
3243
3326
|
}
|
|
3244
3327
|
summarise();
|
|
3328
|
+
return frozen;
|
|
3245
3329
|
},
|
|
3246
3330
|
async leaveOpen() {
|
|
3247
3331
|
if (done) return;
|
|
@@ -3319,6 +3403,28 @@ var init_session = __esm({
|
|
|
3319
3403
|
}
|
|
3320
3404
|
});
|
|
3321
3405
|
|
|
3406
|
+
// src/cli/gate.ts
|
|
3407
|
+
function gateExit(verdict, runnerCode) {
|
|
3408
|
+
if (verdict?.statusEvent === "terminated") {
|
|
3409
|
+
return { code: 1, line: "plune: gate failed \u2014 the run was terminated, so it has no verdict to pass on." };
|
|
3410
|
+
}
|
|
3411
|
+
const counts = verdict?.counts;
|
|
3412
|
+
if (counts == null) return { code: 1, line: "plune: gate failed \u2014 no verdict from the platform, and no verdict is no pass." };
|
|
3413
|
+
const n = (status) => counts[status] ?? 0;
|
|
3414
|
+
const notRun = verdict?.notRun?.length ?? 0;
|
|
3415
|
+
if (runnerCode !== void 0 && notRun === 0 && Object.values(counts).every((k) => !k)) {
|
|
3416
|
+
return { code: runnerCode, line: `plune: gate \u2014 no result reached the run, so the exit code is the command's own (${runnerCode}).` };
|
|
3417
|
+
}
|
|
3418
|
+
const said = `${n("failed")} failed \xB7 ${n("broken")} broken \xB7 ${notRun} not run \xB7 ${n("muted")} muted`;
|
|
3419
|
+
return n("failed") + n("broken") > 0 || notRun > 0 ? { code: 1, line: `plune: gate failed \u2014 ${said}` } : { code: 0, line: `plune: gate passed \u2014 ${said}` };
|
|
3420
|
+
}
|
|
3421
|
+
var init_gate = __esm({
|
|
3422
|
+
"src/cli/gate.ts"() {
|
|
3423
|
+
"use strict";
|
|
3424
|
+
init_cjs_shims();
|
|
3425
|
+
}
|
|
3426
|
+
});
|
|
3427
|
+
|
|
3322
3428
|
// src/cli/commands/run-lifecycle.ts
|
|
3323
3429
|
var run_lifecycle_exports = {};
|
|
3324
3430
|
__export(run_lifecycle_exports, {
|
|
@@ -3376,7 +3482,7 @@ async function handleRunStart(options = {}) {
|
|
|
3376
3482
|
}
|
|
3377
3483
|
return result;
|
|
3378
3484
|
}
|
|
3379
|
-
async function handleRunFinish(id, options = {}) {
|
|
3485
|
+
async function handleRunFinish(id, options = {}, runnerCode) {
|
|
3380
3486
|
const { client, write } = connect(options);
|
|
3381
3487
|
const event = options.terminate === true ? "terminate" : "finish";
|
|
3382
3488
|
const out = await client.post(`/v1/runs/${id}/events`, {
|
|
@@ -3389,6 +3495,11 @@ async function handleRunFinish(id, options = {}) {
|
|
|
3389
3495
|
);
|
|
3390
3496
|
}
|
|
3391
3497
|
write(`Run ${id} ${event === "finish" ? "finished" : "terminated"}.`);
|
|
3498
|
+
if (options.failOnFailures !== true) return runnerCode ?? 0;
|
|
3499
|
+
const summary = out.body?.summary;
|
|
3500
|
+
const gate = gateExit({ statusEvent: out.body?.run?.statusEvent, counts: summary?.counts, notRun: summary?.notRun }, runnerCode);
|
|
3501
|
+
write(gate.line);
|
|
3502
|
+
return gate.code;
|
|
3392
3503
|
}
|
|
3393
3504
|
async function handleRunDelete(id, options = {}) {
|
|
3394
3505
|
const { client, write } = connect(options);
|
|
@@ -3404,7 +3515,7 @@ async function handleRunDelete(id, options = {}) {
|
|
|
3404
3515
|
);
|
|
3405
3516
|
}
|
|
3406
3517
|
write(`Run ${id} deleted, with its results and the review-queue entries it produced.`);
|
|
3407
|
-
write("Recoverable for 30 days \u2014
|
|
3518
|
+
write("Recoverable for 30 days \u2014 an owner of the project can put it back from Settings \u2192 Bin.");
|
|
3408
3519
|
}
|
|
3409
3520
|
async function handleRunExec(options) {
|
|
3410
3521
|
const [command, ...args] = options.argv;
|
|
@@ -3420,8 +3531,7 @@ async function handleRunExec(options) {
|
|
|
3420
3531
|
child.on("close", (exitCode) => resolve9(exitCode ?? 1));
|
|
3421
3532
|
child.on("error", () => resolve9(1));
|
|
3422
3533
|
});
|
|
3423
|
-
|
|
3424
|
-
return code;
|
|
3534
|
+
return handleRunFinish(run.id, options, code);
|
|
3425
3535
|
}
|
|
3426
3536
|
async function handleRunReport(options = {}) {
|
|
3427
3537
|
const file = options.file ?? DEFAULT_FALLBACK_PATH;
|
|
@@ -3517,6 +3627,7 @@ var init_run_lifecycle = __esm({
|
|
|
3517
3627
|
init_env2();
|
|
3518
3628
|
init_fallback();
|
|
3519
3629
|
init_session();
|
|
3630
|
+
init_gate();
|
|
3520
3631
|
NoTokenError = class extends Error {
|
|
3521
3632
|
constructor() {
|
|
3522
3633
|
super('Not logged in. Run "plune login" first, or set PLUNE_TOKEN.');
|
|
@@ -4390,8 +4501,9 @@ async function handleRunImport(options) {
|
|
|
4390
4501
|
results.map((r) => r.keys)
|
|
4391
4502
|
);
|
|
4392
4503
|
for (const result2 of results) await session.add(result2);
|
|
4504
|
+
let frozen;
|
|
4393
4505
|
if (env.keepOpen) await session.leaveOpen();
|
|
4394
|
-
else await session.finish();
|
|
4506
|
+
else frozen = await session.finish();
|
|
4395
4507
|
const stats = session.stats;
|
|
4396
4508
|
const result = {
|
|
4397
4509
|
runId: session.runId,
|
|
@@ -4420,6 +4532,14 @@ async function handleRunImport(options) {
|
|
|
4420
4532
|
offering: options.create === true
|
|
4421
4533
|
};
|
|
4422
4534
|
for (const line of describe(result, apiUrl)) write(line);
|
|
4535
|
+
if (options.failOnFailures === true) {
|
|
4536
|
+
if (env.keepOpen) {
|
|
4537
|
+
write("plune: the run is left open for the jobs still reporting, so there is no verdict yet \u2014 gate on the step that finishes it.");
|
|
4538
|
+
}
|
|
4539
|
+
const gate = gateExit(frozen);
|
|
4540
|
+
write(gate.line);
|
|
4541
|
+
result.gate = gate.code;
|
|
4542
|
+
}
|
|
4423
4543
|
return result;
|
|
4424
4544
|
}
|
|
4425
4545
|
function runUrl2(apiUrl, id) {
|
|
@@ -4486,6 +4606,7 @@ var init_run_import = __esm({
|
|
|
4486
4606
|
init_run_lifecycle();
|
|
4487
4607
|
init_api_url();
|
|
4488
4608
|
init_credentials();
|
|
4609
|
+
init_gate();
|
|
4489
4610
|
init_env2();
|
|
4490
4611
|
NEXT = {
|
|
4491
4612
|
queue: "the review queue is full. Approve or reject what is waiting, then import this report again to offer the rest.",
|
|
@@ -4544,18 +4665,13 @@ var init_report = __esm({
|
|
|
4544
4665
|
});
|
|
4545
4666
|
|
|
4546
4667
|
// src/diff/diff.ts
|
|
4547
|
-
function rowStatus(r) {
|
|
4548
|
-
if (r.error !== void 0) return "errored";
|
|
4549
|
-
if (r.assertions.some((a) => !a.passed)) return "failed";
|
|
4550
|
-
return "passed";
|
|
4551
|
-
}
|
|
4552
4668
|
function evalStatus(e) {
|
|
4553
4669
|
const statuses = e.rows.map(rowStatus);
|
|
4554
4670
|
if (statuses.includes("failed")) return "failed";
|
|
4555
4671
|
if (statuses.includes("errored")) return "errored";
|
|
4556
4672
|
return "passed";
|
|
4557
4673
|
}
|
|
4558
|
-
function
|
|
4674
|
+
function classify2(baseline, current) {
|
|
4559
4675
|
if (current === "errored") return "errored";
|
|
4560
4676
|
if (baseline === "absent") return current === "passed" ? "new-pass" : "new-fail";
|
|
4561
4677
|
if (current === "absent") return "removed";
|
|
@@ -4569,7 +4685,7 @@ function diffRuns(baseline, current) {
|
|
|
4569
4685
|
const evals = ids.map((id) => {
|
|
4570
4686
|
const baselineP = baseMap.get(id) ?? "absent";
|
|
4571
4687
|
const currentP = curMap.get(id) ?? "absent";
|
|
4572
|
-
return { id, status:
|
|
4688
|
+
return { id, status: classify2(baselineP, currentP), baseline: baselineP, current: currentP };
|
|
4573
4689
|
});
|
|
4574
4690
|
const count2 = (s) => evals.filter((e) => e.status === s).length;
|
|
4575
4691
|
const summary = {
|
|
@@ -4589,6 +4705,7 @@ var init_diff = __esm({
|
|
|
4589
4705
|
"src/diff/diff.ts"() {
|
|
4590
4706
|
"use strict";
|
|
4591
4707
|
init_cjs_shims();
|
|
4708
|
+
init_results();
|
|
4592
4709
|
}
|
|
4593
4710
|
});
|
|
4594
4711
|
|
|
@@ -5007,71 +5124,6 @@ var init_logout = __esm({
|
|
|
5007
5124
|
}
|
|
5008
5125
|
});
|
|
5009
5126
|
|
|
5010
|
-
// src/types/test-case-schema.ts
|
|
5011
|
-
var import_zod2, TEST_CASE_SCHEMA_VERSION, externalKeySchema, manualStepSchema, assertionResultRecordSchema, manualCaseSchema, automatedCaseSchema, evalCaseSchema, testCaseBaseSchema, testCaseSchema;
|
|
5012
|
-
var init_test_case_schema = __esm({
|
|
5013
|
-
"src/types/test-case-schema.ts"() {
|
|
5014
|
-
"use strict";
|
|
5015
|
-
init_cjs_shims();
|
|
5016
|
-
import_zod2 = require("zod");
|
|
5017
|
-
TEST_CASE_SCHEMA_VERSION = 2;
|
|
5018
|
-
externalKeySchema = import_zod2.z.object({
|
|
5019
|
-
kind: import_zod2.z.enum(["playwright-id", "path-title", "allure-history", "cairn-stable", "eval-id", "qase", "testrail"]),
|
|
5020
|
-
value: import_zod2.z.string().min(1)
|
|
5021
|
-
});
|
|
5022
|
-
manualStepSchema = import_zod2.z.object({
|
|
5023
|
-
action: import_zod2.z.string().min(1),
|
|
5024
|
-
expected: import_zod2.z.string().min(1).optional()
|
|
5025
|
-
});
|
|
5026
|
-
assertionResultRecordSchema = import_zod2.z.object({
|
|
5027
|
-
type: import_zod2.z.string().min(1),
|
|
5028
|
-
passed: import_zod2.z.boolean(),
|
|
5029
|
-
score: import_zod2.z.number().optional(),
|
|
5030
|
-
reason: import_zod2.z.string().optional()
|
|
5031
|
-
});
|
|
5032
|
-
manualCaseSchema = import_zod2.z.object({
|
|
5033
|
-
type: import_zod2.z.literal("manual"),
|
|
5034
|
-
steps: import_zod2.z.array(manualStepSchema).min(1),
|
|
5035
|
-
expected: import_zod2.z.string().min(1).optional()
|
|
5036
|
-
});
|
|
5037
|
-
automatedCaseSchema = import_zod2.z.object({
|
|
5038
|
-
type: import_zod2.z.literal("automated"),
|
|
5039
|
-
specRef: import_zod2.z.string().min(1)
|
|
5040
|
-
});
|
|
5041
|
-
evalCaseSchema = import_zod2.z.object({
|
|
5042
|
-
type: import_zod2.z.literal("eval"),
|
|
5043
|
-
prompt: import_zod2.z.string().min(1),
|
|
5044
|
-
goldenSetId: import_zod2.z.string().min(1).optional(),
|
|
5045
|
-
assertions: import_zod2.z.array(assertionResultRecordSchema).min(1)
|
|
5046
|
-
});
|
|
5047
|
-
testCaseBaseSchema = import_zod2.z.object({
|
|
5048
|
-
schemaVersion: import_zod2.z.literal(TEST_CASE_SCHEMA_VERSION),
|
|
5049
|
-
id: import_zod2.z.string().min(1),
|
|
5050
|
-
externalKeys: import_zod2.z.array(externalKeySchema).optional(),
|
|
5051
|
-
/** ADR 0045 / plune#1246 — optional, purely additive; schemaVersion stays 2. */
|
|
5052
|
-
is_flaky: import_zod2.z.boolean().optional(),
|
|
5053
|
-
/** ADR 0045 / plune#1246 — optional, purely additive; schemaVersion stays 2. */
|
|
5054
|
-
is_muted: import_zod2.z.boolean().optional()
|
|
5055
|
-
});
|
|
5056
|
-
testCaseSchema = import_zod2.z.discriminatedUnion("type", [
|
|
5057
|
-
testCaseBaseSchema.merge(manualCaseSchema),
|
|
5058
|
-
testCaseBaseSchema.merge(automatedCaseSchema),
|
|
5059
|
-
testCaseBaseSchema.merge(evalCaseSchema)
|
|
5060
|
-
]);
|
|
5061
|
-
}
|
|
5062
|
-
});
|
|
5063
|
-
|
|
5064
|
-
// src/types/results.ts
|
|
5065
|
-
var ROW_INCONCLUSIVE;
|
|
5066
|
-
var init_results = __esm({
|
|
5067
|
-
"src/types/results.ts"() {
|
|
5068
|
-
"use strict";
|
|
5069
|
-
init_cjs_shims();
|
|
5070
|
-
init_test_case_schema();
|
|
5071
|
-
ROW_INCONCLUSIVE = ["blocked", "skipped"];
|
|
5072
|
-
}
|
|
5073
|
-
});
|
|
5074
|
-
|
|
5075
5127
|
// src/types/run-result-schema.ts
|
|
5076
5128
|
function parseRunResult(value2) {
|
|
5077
5129
|
return runResultSchema.parse(value2);
|
|
@@ -5120,6 +5172,9 @@ var init_run_result_schema = __esm({
|
|
|
5120
5172
|
passed: import_zod3.z.number(),
|
|
5121
5173
|
failed: import_zod3.z.number(),
|
|
5122
5174
|
errored: import_zod3.z.number(),
|
|
5175
|
+
/** plune#1245. Optional and additive — a RunResult written before them validates unchanged. */
|
|
5176
|
+
blocked: import_zod3.z.number().optional(),
|
|
5177
|
+
skipped: import_zod3.z.number().optional(),
|
|
5123
5178
|
cost_usd: import_zod3.z.number(),
|
|
5124
5179
|
duration_ms: import_zod3.z.number()
|
|
5125
5180
|
});
|
|
@@ -5207,7 +5262,18 @@ async function handleSync(deps = {}) {
|
|
|
5207
5262
|
}
|
|
5208
5263
|
const body = await res.json().catch(() => ({}));
|
|
5209
5264
|
const id = typeof body.id === "string" ? body.id : "";
|
|
5210
|
-
|
|
5265
|
+
const result = { id, url: `${endpoint}/${id}` };
|
|
5266
|
+
if (deps.failOnFailures === true) result.gate = gateExit(await readVerdict(doFetch, `${endpoint}/${id}/summary`, token));
|
|
5267
|
+
return result;
|
|
5268
|
+
}
|
|
5269
|
+
async function readVerdict(doFetch, url, token) {
|
|
5270
|
+
try {
|
|
5271
|
+
const res = await doFetch(url, { method: "GET", headers: { authorization: `Bearer ${token}` } });
|
|
5272
|
+
const body = await res.json().catch(() => void 0);
|
|
5273
|
+
return res.ok ? { statusEvent: body?.run?.statusEvent, counts: body?.counts, notRun: body?.notRun } : void 0;
|
|
5274
|
+
} catch {
|
|
5275
|
+
return void 0;
|
|
5276
|
+
}
|
|
5211
5277
|
}
|
|
5212
5278
|
function reportSyncFailure(err, write) {
|
|
5213
5279
|
if (err instanceof NotLoggedInError || err instanceof TokenRejectedError2 || err instanceof SyncFileError) {
|
|
@@ -5230,6 +5296,7 @@ var init_sync = __esm({
|
|
|
5230
5296
|
init_run_result_schema();
|
|
5231
5297
|
init_api_url();
|
|
5232
5298
|
init_credentials();
|
|
5299
|
+
init_gate();
|
|
5233
5300
|
NotLoggedInError = class extends Error {
|
|
5234
5301
|
constructor() {
|
|
5235
5302
|
super('Not logged in. Run "plune login" first, or set PLUNE_TOKEN.');
|
|
@@ -5894,6 +5961,7 @@ function createProgram() {
|
|
|
5894
5961
|
process.exit(1);
|
|
5895
5962
|
};
|
|
5896
5963
|
const verboseOf = (command) => command.optsWithGlobals().verbose === true;
|
|
5964
|
+
const FAIL_ON_FAILURES = "Exit with the platform's verdict on the run: 1 on a failed or broken result (muted ones do not count), on an expected case that never ran, or when there is no verdict at all";
|
|
5897
5965
|
runCommand.command("start").description("Open a platform run (or join one already carrying the key) and print its id").option("--key <externalKey>", "The key several processes share; generated when absent").option("--json", "Print one machine-readable line instead of prose", false).action(async (options, command) => {
|
|
5898
5966
|
try {
|
|
5899
5967
|
const { handleRunStart: handleRunStart2 } = await Promise.resolve().then(() => (init_run_lifecycle(), run_lifecycle_exports));
|
|
@@ -5905,14 +5973,16 @@ function createProgram() {
|
|
|
5905
5973
|
await failRunCommand(err, verboseOf(command));
|
|
5906
5974
|
}
|
|
5907
5975
|
});
|
|
5908
|
-
runCommand.command("finish").argument("<id>", "The run to close").description("Close a platform run \u2014 what the reporter tells you to do for a run left open").option("--terminate", "Record the run as cut short rather than finished", false).option("--reason <text>", "Why, for a terminated run").action(
|
|
5976
|
+
runCommand.command("finish").argument("<id>", "The run to close").description("Close a platform run \u2014 what the reporter tells you to do for a run left open").option("--terminate", "Record the run as cut short rather than finished", false).option("--reason <text>", "Why, for a terminated run").option("--fail-on-failures", FAIL_ON_FAILURES, false).action(
|
|
5909
5977
|
async (id, options, command) => {
|
|
5910
5978
|
try {
|
|
5911
5979
|
const { handleRunFinish: handleRunFinish2 } = await Promise.resolve().then(() => (init_run_lifecycle(), run_lifecycle_exports));
|
|
5912
|
-
await handleRunFinish2(id, {
|
|
5980
|
+
const code = await handleRunFinish2(id, {
|
|
5913
5981
|
terminate: options.terminate,
|
|
5914
|
-
...options.reason !== void 0 ? { reason: options.reason } : {}
|
|
5982
|
+
...options.reason !== void 0 ? { reason: options.reason } : {},
|
|
5983
|
+
...options.failOnFailures ? { failOnFailures: true } : {}
|
|
5915
5984
|
});
|
|
5985
|
+
if (options.failOnFailures) process.exitCode = code;
|
|
5916
5986
|
} catch (err) {
|
|
5917
5987
|
await failRunCommand(err, verboseOf(command));
|
|
5918
5988
|
}
|
|
@@ -5920,12 +5990,13 @@ function createProgram() {
|
|
|
5920
5990
|
);
|
|
5921
5991
|
runCommand.command("exec").argument("<command...>", "The command to run inside the run, after `--`").description(
|
|
5922
5992
|
"Open a run, run a command inside it, close the run \u2014 whatever the command returns"
|
|
5923
|
-
).option("--key <externalKey>", "The key several processes share; generated when absent").action(async (argv, options, command) => {
|
|
5993
|
+
).option("--key <externalKey>", "The key several processes share; generated when absent").option("--fail-on-failures", `${FAIL_ON_FAILURES}; the command's own exit code when no result reached the run`, false).action(async (argv, options, command) => {
|
|
5924
5994
|
try {
|
|
5925
5995
|
const { handleRunExec: handleRunExec2 } = await Promise.resolve().then(() => (init_run_lifecycle(), run_lifecycle_exports));
|
|
5926
5996
|
process.exitCode = await handleRunExec2({
|
|
5927
5997
|
argv,
|
|
5928
|
-
...options.key !== void 0 ? { key: options.key } : {}
|
|
5998
|
+
...options.key !== void 0 ? { key: options.key } : {},
|
|
5999
|
+
...options.failOnFailures ? { failOnFailures: true } : {}
|
|
5929
6000
|
});
|
|
5930
6001
|
} catch (err) {
|
|
5931
6002
|
await failRunCommand(err, verboseOf(command));
|
|
@@ -5934,7 +6005,7 @@ function createProgram() {
|
|
|
5934
6005
|
runCommand.command("delete").argument(
|
|
5935
6006
|
"<id>",
|
|
5936
6007
|
"The run id \u2014 the one `plune run import` printed, or the one in the dashboard URL"
|
|
5937
|
-
).description("Delete a run and everything it produced \u2014 recoverable for
|
|
6008
|
+
).description("Delete a run and everything it produced \u2014 recoverable for 30 days").action(async (id, _options, command) => {
|
|
5938
6009
|
try {
|
|
5939
6010
|
const { handleRunDelete: handleRunDelete2 } = await Promise.resolve().then(() => (init_run_lifecycle(), run_lifecycle_exports));
|
|
5940
6011
|
await handleRunDelete2(id);
|
|
@@ -5954,7 +6025,7 @@ function createProgram() {
|
|
|
5954
6025
|
"--fail-on-missing-attachments",
|
|
5955
6026
|
"Exit 1 when a file the report names ([[ATTACHMENT|path]] in JUnit, a source in allure-results) is not there; the results are sent either way",
|
|
5956
6027
|
false
|
|
5957
|
-
).action(
|
|
6028
|
+
).option("--fail-on-failures", FAIL_ON_FAILURES, false).action(
|
|
5958
6029
|
async (file, options, command) => {
|
|
5959
6030
|
const [
|
|
5960
6031
|
{ handleRunImport: handleRunImport2 },
|
|
@@ -5978,9 +6049,11 @@ function createProgram() {
|
|
|
5978
6049
|
file,
|
|
5979
6050
|
...format !== void 0 ? { format } : {},
|
|
5980
6051
|
...options.key !== void 0 ? { key: options.key } : {},
|
|
5981
|
-
...options.create ? { create: true } : {}
|
|
6052
|
+
...options.create ? { create: true } : {},
|
|
6053
|
+
...options.failOnFailures ? { failOnFailures: true } : {}
|
|
5982
6054
|
});
|
|
5983
6055
|
if (options.failOnMissingAttachments && imported.missingAttachments.length > 0) process.exitCode = 1;
|
|
6056
|
+
if (imported.gate === 1) process.exitCode = 1;
|
|
5984
6057
|
} catch (err) {
|
|
5985
6058
|
if (err instanceof UnknownFormatError2 || err instanceof XmlParseError2 || err instanceof JsonReportError2 || err instanceof AllureReportError2) {
|
|
5986
6059
|
process.stderr.write(err.message + "\n");
|
|
@@ -6090,18 +6163,24 @@ function createProgram() {
|
|
|
6090
6163
|
failUnexpected(err, verbose);
|
|
6091
6164
|
}
|
|
6092
6165
|
});
|
|
6093
|
-
program.command("sync").description("Upload your latest local run to the Plune platform (POST /v1/runs)").option("--file <path>", "Sync a specific run JSON instead of .plune/last-run.json").action(async (options, command) => {
|
|
6166
|
+
program.command("sync").description("Upload your latest local run to the Plune platform (POST /v1/runs)").option("--file <path>", "Sync a specific run JSON instead of .plune/last-run.json").option("--fail-on-failures", `${FAIL_ON_FAILURES}; read with one GET of the run's summary after the upload`, false).action(async (options, command) => {
|
|
6094
6167
|
const globals = command.optsWithGlobals();
|
|
6095
6168
|
const verbose = globals.verbose === true;
|
|
6096
6169
|
const { handleSync: handleSync2, reportSyncFailure: reportSyncFailure2 } = await Promise.resolve().then(() => (init_sync(), sync_exports));
|
|
6097
6170
|
const cwd = globals.config !== void 0 ? (0, import_node_path4.dirname)((0, import_node_path4.resolve)(globals.config)) : process.cwd();
|
|
6098
6171
|
try {
|
|
6099
|
-
const { id, url } = await handleSync2({
|
|
6172
|
+
const { id, url, gate } = await handleSync2({
|
|
6100
6173
|
cwd,
|
|
6101
|
-
...options.file !== void 0 ? { file: options.file } : {}
|
|
6174
|
+
...options.file !== void 0 ? { file: options.file } : {},
|
|
6175
|
+
...options.failOnFailures ? { failOnFailures: true } : {}
|
|
6102
6176
|
});
|
|
6103
6177
|
process.stdout.write(`Synced run ${id} \u2192 ${url}
|
|
6104
6178
|
`);
|
|
6179
|
+
if (gate !== void 0) {
|
|
6180
|
+
process.stdout.write(`${gate.line}
|
|
6181
|
+
`);
|
|
6182
|
+
process.exitCode = gate.code;
|
|
6183
|
+
}
|
|
6105
6184
|
} catch (err) {
|
|
6106
6185
|
const code = reportSyncFailure2(err, (s) => process.stderr.write(s));
|
|
6107
6186
|
if (code !== null) {
|