@plune-ai/cli 0.5.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -1
- package/dist/cli.cjs +1093 -141
- package/dist/index.cjs +264 -12
- package/dist/index.d.cts +104 -423
- package/dist/index.d.ts +104 -423
- package/dist/index.js +260 -12
- package/dist/reporter-core.cjs +407 -0
- package/dist/reporter-core.d.cts +315 -0
- package/dist/reporter-core.d.ts +315 -0
- package/dist/reporter-core.js +363 -0
- package/package.json +22 -5
package/dist/index.js
CHANGED
|
@@ -97,9 +97,9 @@ var modelPriceSchema = z.object({
|
|
|
97
97
|
input_per_1k_usd: z.number().nonnegative(),
|
|
98
98
|
output_per_1k_usd: z.number().nonnegative()
|
|
99
99
|
});
|
|
100
|
-
var pricingSchema = z.record(modelPriceSchema);
|
|
100
|
+
var pricingSchema = z.record(z.string(), modelPriceSchema);
|
|
101
101
|
var datasetRowSchema = z.object({
|
|
102
|
-
vars: z.record(z.union([z.string(), z.number(), z.boolean()])),
|
|
102
|
+
vars: z.record(z.string(), z.union([z.string(), z.number(), z.boolean()])),
|
|
103
103
|
expected: z.string().optional()
|
|
104
104
|
});
|
|
105
105
|
var datasetRefSchema = z.union([
|
|
@@ -129,7 +129,7 @@ var containsAllAssertionSchema = z.object({
|
|
|
129
129
|
});
|
|
130
130
|
var jsonSchemaAssertionSchema = z.object({
|
|
131
131
|
type: z.literal("json-schema"),
|
|
132
|
-
schema: z.record(z.unknown()),
|
|
132
|
+
schema: z.record(z.string(), z.unknown()),
|
|
133
133
|
extract: z.enum(["auto", "strict"]).optional()
|
|
134
134
|
});
|
|
135
135
|
var llmJudgeAssertionSchema = z.object({
|
|
@@ -595,9 +595,11 @@ var XenovaEmbedder = class {
|
|
|
595
595
|
return rows.map((row) => Float32Array.from(row));
|
|
596
596
|
}
|
|
597
597
|
load() {
|
|
598
|
-
this.extractor ??= import("@huggingface/transformers").
|
|
599
|
-
|
|
600
|
-
|
|
598
|
+
this.extractor ??= import("@huggingface/transformers").catch(() => {
|
|
599
|
+
throw new Error(
|
|
600
|
+
"The `semantic-similarity` and RAG assertions need local embeddings, which ship separately: install @huggingface/transformers to enable them. Every other assertion works without it."
|
|
601
|
+
);
|
|
602
|
+
}).then(({ pipeline }) => pipeline("feature-extraction", MODEL_ID));
|
|
601
603
|
return this.extractor;
|
|
602
604
|
}
|
|
603
605
|
};
|
|
@@ -1489,7 +1491,7 @@ var runErrorSchema = z2.object({
|
|
|
1489
1491
|
message: z2.string()
|
|
1490
1492
|
});
|
|
1491
1493
|
var rowResultSchema = z2.object({
|
|
1492
|
-
vars: z2.record(z2.unknown()),
|
|
1494
|
+
vars: z2.record(z2.string(), z2.unknown()),
|
|
1493
1495
|
output: z2.string().nullable(),
|
|
1494
1496
|
cached: z2.boolean(),
|
|
1495
1497
|
usage: usageSchema.optional(),
|
|
@@ -1524,6 +1526,13 @@ function parseRunResult(value) {
|
|
|
1524
1526
|
return runResultSchema.parse(value);
|
|
1525
1527
|
}
|
|
1526
1528
|
|
|
1529
|
+
// src/cli/api-url.ts
|
|
1530
|
+
var DEFAULT_API_URL = "https://beta-api.plune.ai";
|
|
1531
|
+
function resolveApiUrl(explicit) {
|
|
1532
|
+
const raw = explicit ?? process.env["PLUNE_API_URL"] ?? DEFAULT_API_URL;
|
|
1533
|
+
return raw.replace(/\/+$/, "");
|
|
1534
|
+
}
|
|
1535
|
+
|
|
1527
1536
|
// src/cli/credentials.ts
|
|
1528
1537
|
import { chmodSync, existsSync as existsSync2, mkdirSync as mkdirSync2, readFileSync as readFileSync3, rmSync, writeFileSync as writeFileSync2 } from "fs";
|
|
1529
1538
|
import { homedir } from "os";
|
|
@@ -1542,7 +1551,6 @@ function loadToken() {
|
|
|
1542
1551
|
}
|
|
1543
1552
|
|
|
1544
1553
|
// src/cli/commands/sync.ts
|
|
1545
|
-
var DEFAULT_API_URL = "https://beta-api.plune.ai";
|
|
1546
1554
|
var NotLoggedInError = class extends Error {
|
|
1547
1555
|
constructor() {
|
|
1548
1556
|
super('Not logged in. Run "plune login" first to save your API token.');
|
|
@@ -1579,10 +1587,6 @@ var SyncHttpError = class extends Error {
|
|
|
1579
1587
|
}
|
|
1580
1588
|
status;
|
|
1581
1589
|
};
|
|
1582
|
-
function resolveApiUrl(explicit) {
|
|
1583
|
-
const raw = explicit ?? process.env["PLUNE_API_URL"] ?? DEFAULT_API_URL;
|
|
1584
|
-
return raw.replace(/\/+$/, "");
|
|
1585
|
-
}
|
|
1586
1590
|
function readRun(cwd, file) {
|
|
1587
1591
|
const target = file !== void 0 ? path5.resolve(cwd, file) : path5.join(cwd, ".plune", "last-run.json");
|
|
1588
1592
|
let raw;
|
|
@@ -1651,6 +1655,246 @@ function reportSyncFailure(err, write) {
|
|
|
1651
1655
|
}
|
|
1652
1656
|
return null;
|
|
1653
1657
|
}
|
|
1658
|
+
|
|
1659
|
+
// src/cli/commands/cairn-artifact.ts
|
|
1660
|
+
import * as fs7 from "fs/promises";
|
|
1661
|
+
import * as path6 from "path";
|
|
1662
|
+
var SUPPORTED_ARTIFACT_VERSION = 1;
|
|
1663
|
+
var INGEST_SCHEMA_VERSION = 1;
|
|
1664
|
+
var NoCairnRunsError = class extends Error {
|
|
1665
|
+
constructor(runsDir) {
|
|
1666
|
+
super(`No Cairn run found under ${runsDir} \u2014 name the directory instead: plune ingest <dir>.`);
|
|
1667
|
+
this.name = "NoCairnRunsError";
|
|
1668
|
+
}
|
|
1669
|
+
};
|
|
1670
|
+
async function findLatestCairnRun(runsDir = "runs") {
|
|
1671
|
+
const entries = await fs7.readdir(runsDir, { withFileTypes: true }).catch(() => null);
|
|
1672
|
+
if (entries === null) throw new NoCairnRunsError(runsDir);
|
|
1673
|
+
let newest;
|
|
1674
|
+
for (const entry of entries) {
|
|
1675
|
+
if (!entry.isDirectory()) continue;
|
|
1676
|
+
const dir = path6.join(runsDir, entry.name);
|
|
1677
|
+
try {
|
|
1678
|
+
const report = await fs7.lstat(path6.join(dir, "report.json"));
|
|
1679
|
+
if (!report.isFile()) continue;
|
|
1680
|
+
if (newest === void 0 || report.mtimeMs > newest.at) {
|
|
1681
|
+
newest = { dir, at: report.mtimeMs };
|
|
1682
|
+
}
|
|
1683
|
+
} catch {
|
|
1684
|
+
}
|
|
1685
|
+
}
|
|
1686
|
+
if (newest === void 0) throw new NoCairnRunsError(runsDir);
|
|
1687
|
+
return newest.dir;
|
|
1688
|
+
}
|
|
1689
|
+
var CairnRunNotFoundError = class extends Error {
|
|
1690
|
+
constructor(dir) {
|
|
1691
|
+
super(
|
|
1692
|
+
`No Cairn run found in ${dir} \u2014 expected a report.json written by "cairn design|explore|api".`
|
|
1693
|
+
);
|
|
1694
|
+
this.name = "CairnRunNotFoundError";
|
|
1695
|
+
}
|
|
1696
|
+
};
|
|
1697
|
+
var CairnRunVersionError = class extends Error {
|
|
1698
|
+
constructor(found) {
|
|
1699
|
+
super(
|
|
1700
|
+
found === void 0 ? "This run predates versioned Cairn artifacts. Re-run it with Cairn 0.7.0 or newer, which stamps the format so it can be read safely." : `Unsupported Cairn artifact version ${String(found)} (this CLI reads version ${SUPPORTED_ARTIFACT_VERSION}). Upgrade @plune-ai/cli.`
|
|
1701
|
+
);
|
|
1702
|
+
this.name = "CairnRunVersionError";
|
|
1703
|
+
}
|
|
1704
|
+
};
|
|
1705
|
+
var CairnRunPartialError = class extends Error {
|
|
1706
|
+
constructor(reason) {
|
|
1707
|
+
super(`That Cairn run did not finish, so its cases are incomplete: ${reason}`);
|
|
1708
|
+
this.name = "CairnRunPartialError";
|
|
1709
|
+
}
|
|
1710
|
+
};
|
|
1711
|
+
var CairnEvidenceAmbiguousError = class extends Error {
|
|
1712
|
+
constructor(name) {
|
|
1713
|
+
super(
|
|
1714
|
+
`Cannot attach results safely: the operation "${name}" appears more than once in this run's evidence, and the evidence file carries no case identity to disambiguate it. Nothing was uploaded.`
|
|
1715
|
+
);
|
|
1716
|
+
this.name = "CairnEvidenceAmbiguousError";
|
|
1717
|
+
}
|
|
1718
|
+
};
|
|
1719
|
+
var asRecord = (v) => typeof v === "object" && v !== null ? v : {};
|
|
1720
|
+
var str = (v, fallback = "") => typeof v === "string" && v !== "" ? v : fallback;
|
|
1721
|
+
async function readJson(file) {
|
|
1722
|
+
try {
|
|
1723
|
+
return JSON.parse(await fs7.readFile(file, "utf8"));
|
|
1724
|
+
} catch {
|
|
1725
|
+
return void 0;
|
|
1726
|
+
}
|
|
1727
|
+
}
|
|
1728
|
+
async function defaultReadEvidence(dir) {
|
|
1729
|
+
const raw = await readJson(path6.join(dir, "api-evidence.json"));
|
|
1730
|
+
if (!Array.isArray(raw)) return void 0;
|
|
1731
|
+
return raw.map((r) => {
|
|
1732
|
+
const rec = asRecord(r);
|
|
1733
|
+
return { name: str(rec["name"]), passed: rec["passed"] === true };
|
|
1734
|
+
});
|
|
1735
|
+
}
|
|
1736
|
+
function casesFromDesign(report) {
|
|
1737
|
+
const list = Array.isArray(report["testCases"]) ? report["testCases"] : [];
|
|
1738
|
+
return list.map((raw) => {
|
|
1739
|
+
const c = asRecord(raw);
|
|
1740
|
+
const steps = (Array.isArray(c["steps"]) ? c["steps"] : []).map((s) => str(s)).filter((s) => s !== "");
|
|
1741
|
+
return {
|
|
1742
|
+
stableId: str(c["stableId"]),
|
|
1743
|
+
title: str(c["title"], "untitled case"),
|
|
1744
|
+
execution: c["execution"] === "manual" ? "manual" : "auto",
|
|
1745
|
+
steps: steps.length > 0 ? steps : ["(no steps recorded)"],
|
|
1746
|
+
expected: str(c["expected"], "(no expected result recorded)"),
|
|
1747
|
+
...typeof c["technique"] === "string" ? { technique: c["technique"] } : {},
|
|
1748
|
+
...typeof c["priority"] === "string" ? { priority: c["priority"] } : {}
|
|
1749
|
+
};
|
|
1750
|
+
});
|
|
1751
|
+
}
|
|
1752
|
+
function casesFromApi(report, verdicts) {
|
|
1753
|
+
const list = Array.isArray(report["cases"]) ? report["cases"] : [];
|
|
1754
|
+
return list.map((raw) => {
|
|
1755
|
+
const c = asRecord(raw);
|
|
1756
|
+
const name = str(c["name"], "unnamed operation");
|
|
1757
|
+
const request = `${str(c["method"], "GET")} ${str(c["path"], "/")}`;
|
|
1758
|
+
const verdict = verdicts.get(name);
|
|
1759
|
+
return {
|
|
1760
|
+
stableId: str(c["stableId"]),
|
|
1761
|
+
title: name,
|
|
1762
|
+
execution: "auto",
|
|
1763
|
+
steps: [request],
|
|
1764
|
+
expected: `HTTP ${str(c["expectedStatus"], "default")}`,
|
|
1765
|
+
...verdict !== void 0 ? { verdict } : {},
|
|
1766
|
+
...typeof c["technique"] === "string" ? { technique: c["technique"] } : {}
|
|
1767
|
+
};
|
|
1768
|
+
});
|
|
1769
|
+
}
|
|
1770
|
+
async function readCairnRun(dir, deps = {}) {
|
|
1771
|
+
const raw = await readJson(path6.join(dir, "report.json"));
|
|
1772
|
+
if (raw === void 0) throw new CairnRunNotFoundError(dir);
|
|
1773
|
+
const report = asRecord(raw);
|
|
1774
|
+
if (report["schemaVersion"] !== SUPPORTED_ARTIFACT_VERSION) {
|
|
1775
|
+
throw new CairnRunVersionError(report["schemaVersion"]);
|
|
1776
|
+
}
|
|
1777
|
+
if (report["partial"] === true) {
|
|
1778
|
+
throw new CairnRunPartialError(str(report["error"], "no reason recorded"));
|
|
1779
|
+
}
|
|
1780
|
+
const mode = report["mode"];
|
|
1781
|
+
if (mode !== "design" && mode !== "explore" && mode !== "api") {
|
|
1782
|
+
throw new CairnRunVersionError(
|
|
1783
|
+
`${SUPPORTED_ARTIFACT_VERSION} (unknown run mode ${String(mode)})`
|
|
1784
|
+
);
|
|
1785
|
+
}
|
|
1786
|
+
let cases;
|
|
1787
|
+
if (mode === "api") {
|
|
1788
|
+
const evidence = await (deps.readEvidence ?? defaultReadEvidence)(dir);
|
|
1789
|
+
const verdicts = /* @__PURE__ */ new Map();
|
|
1790
|
+
for (const e of evidence ?? []) {
|
|
1791
|
+
if (verdicts.has(e.name)) throw new CairnEvidenceAmbiguousError(e.name);
|
|
1792
|
+
verdicts.set(e.name, e.passed ? "passed" : "failed");
|
|
1793
|
+
}
|
|
1794
|
+
cases = casesFromApi(report, verdicts);
|
|
1795
|
+
} else {
|
|
1796
|
+
cases = casesFromDesign(report);
|
|
1797
|
+
}
|
|
1798
|
+
return {
|
|
1799
|
+
schemaVersion: INGEST_SCHEMA_VERSION,
|
|
1800
|
+
run: {
|
|
1801
|
+
runId: str(report["runId"], path6.basename(dir)),
|
|
1802
|
+
mode,
|
|
1803
|
+
target: str(report["url"], "unknown")
|
|
1804
|
+
},
|
|
1805
|
+
cases
|
|
1806
|
+
};
|
|
1807
|
+
}
|
|
1808
|
+
|
|
1809
|
+
// src/cli/commands/ingest.ts
|
|
1810
|
+
var IngestNotLoggedInError = class extends Error {
|
|
1811
|
+
constructor() {
|
|
1812
|
+
super('Not logged in. Run "plune login" first to save your API token.');
|
|
1813
|
+
this.name = "IngestNotLoggedInError";
|
|
1814
|
+
}
|
|
1815
|
+
};
|
|
1816
|
+
var IngestTokenRejectedError = class extends Error {
|
|
1817
|
+
constructor() {
|
|
1818
|
+
super('Your API token was rejected (401). Run "plune login" again with a fresh token.');
|
|
1819
|
+
this.name = "IngestTokenRejectedError";
|
|
1820
|
+
}
|
|
1821
|
+
};
|
|
1822
|
+
var IngestNetworkError = class extends Error {
|
|
1823
|
+
constructor(apiUrl) {
|
|
1824
|
+
super(`Could not reach the Plune API at ${apiUrl}. Check your connection, then try again.`);
|
|
1825
|
+
this.name = "IngestNetworkError";
|
|
1826
|
+
}
|
|
1827
|
+
};
|
|
1828
|
+
var IngestHttpError = class extends Error {
|
|
1829
|
+
constructor(status, detail) {
|
|
1830
|
+
super(`Plune API returned ${status}${detail !== "" ? ` \u2014 ${detail}` : ""}.`);
|
|
1831
|
+
this.status = status;
|
|
1832
|
+
this.name = "IngestHttpError";
|
|
1833
|
+
}
|
|
1834
|
+
status;
|
|
1835
|
+
};
|
|
1836
|
+
async function safeErrorDetail2(res) {
|
|
1837
|
+
try {
|
|
1838
|
+
const body = await res.json();
|
|
1839
|
+
return typeof body.error === "string" ? body.error : "";
|
|
1840
|
+
} catch {
|
|
1841
|
+
return "";
|
|
1842
|
+
}
|
|
1843
|
+
}
|
|
1844
|
+
async function handleIngest(deps) {
|
|
1845
|
+
const apiUrl = resolveApiUrl(deps.apiUrl);
|
|
1846
|
+
const token = (deps.loadToken ?? loadToken)();
|
|
1847
|
+
if (token === null) throw new IngestNotLoggedInError();
|
|
1848
|
+
const dir = deps.dir ?? await findLatestCairnRun(deps.runsDir);
|
|
1849
|
+
const payload = await readCairnRun(dir);
|
|
1850
|
+
const doFetch = deps.fetchImpl ?? fetch;
|
|
1851
|
+
const endpoint = `${apiUrl}/v1/ingest/cairn`;
|
|
1852
|
+
let res;
|
|
1853
|
+
try {
|
|
1854
|
+
res = await doFetch(endpoint, {
|
|
1855
|
+
method: "POST",
|
|
1856
|
+
headers: { "content-type": "application/json", authorization: `Bearer ${token}` },
|
|
1857
|
+
body: JSON.stringify(payload)
|
|
1858
|
+
});
|
|
1859
|
+
} catch {
|
|
1860
|
+
throw new IngestNetworkError(apiUrl);
|
|
1861
|
+
}
|
|
1862
|
+
if (!res.ok) {
|
|
1863
|
+
const detail = await safeErrorDetail2(res);
|
|
1864
|
+
if (res.status === 401) throw new IngestTokenRejectedError();
|
|
1865
|
+
throw new IngestHttpError(res.status, detail);
|
|
1866
|
+
}
|
|
1867
|
+
const body = await res.json().catch(() => ({}));
|
|
1868
|
+
return {
|
|
1869
|
+
runId: typeof body.runId === "string" ? body.runId : "",
|
|
1870
|
+
linked: typeof body.linked === "number" ? body.linked : 0,
|
|
1871
|
+
proposed: typeof body.proposed === "number" ? body.proposed : 0,
|
|
1872
|
+
skipped: typeof body.skipped === "number" ? body.skipped : 0
|
|
1873
|
+
};
|
|
1874
|
+
}
|
|
1875
|
+
function reportIngestFailure(err, write) {
|
|
1876
|
+
if (err instanceof IngestNotLoggedInError || err instanceof IngestTokenRejectedError || err instanceof CairnRunNotFoundError || err instanceof NoCairnRunsError || err instanceof CairnRunVersionError || err instanceof CairnRunPartialError || err instanceof CairnEvidenceAmbiguousError) {
|
|
1877
|
+
write(`${err.message}
|
|
1878
|
+
`);
|
|
1879
|
+
return 2;
|
|
1880
|
+
}
|
|
1881
|
+
if (err instanceof IngestNetworkError || err instanceof IngestHttpError) {
|
|
1882
|
+
write(`${err.message}
|
|
1883
|
+
`);
|
|
1884
|
+
return 1;
|
|
1885
|
+
}
|
|
1886
|
+
return null;
|
|
1887
|
+
}
|
|
1888
|
+
function formatIngestResult(r) {
|
|
1889
|
+
const lines = [
|
|
1890
|
+
`Ingested Cairn run \u2192 ${r.runId}`,
|
|
1891
|
+
` ${r.linked} result(s) attached to existing cases`,
|
|
1892
|
+
` ${r.proposed} case(s) proposed \u2014 review them in Plune to accept`,
|
|
1893
|
+
` ${r.skipped} skipped (already queued, already rejected, repeated, or never executed)`
|
|
1894
|
+
];
|
|
1895
|
+
return `${lines.join("\n")}
|
|
1896
|
+
`;
|
|
1897
|
+
}
|
|
1654
1898
|
export {
|
|
1655
1899
|
NotLoggedInError,
|
|
1656
1900
|
RUN_RESULT_SCHEMA_VERSION,
|
|
@@ -1661,7 +1905,11 @@ export {
|
|
|
1661
1905
|
assertionConfigSchema,
|
|
1662
1906
|
assertionResultRecordSchema,
|
|
1663
1907
|
binaryVerdictSchema,
|
|
1908
|
+
formatIngestResult,
|
|
1909
|
+
handleIngest as ingest,
|
|
1664
1910
|
parseRunResult,
|
|
1911
|
+
readCairnRun,
|
|
1912
|
+
reportIngestFailure,
|
|
1665
1913
|
reportSyncFailure,
|
|
1666
1914
|
handleRun as run,
|
|
1667
1915
|
runResultSchema,
|