@plune-ai/cli 0.5.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -1
- package/dist/cli.cjs +1093 -141
- package/dist/index.cjs +264 -12
- package/dist/index.d.cts +104 -423
- package/dist/index.d.ts +104 -423
- package/dist/index.js +260 -12
- package/dist/reporter-core.cjs +407 -0
- package/dist/reporter-core.d.cts +315 -0
- package/dist/reporter-core.d.ts +315 -0
- package/dist/reporter-core.js +363 -0
- package/package.json +22 -5
package/dist/index.cjs
CHANGED
|
@@ -39,7 +39,11 @@ __export(src_exports, {
|
|
|
39
39
|
assertionConfigSchema: () => assertionConfigSchema,
|
|
40
40
|
assertionResultRecordSchema: () => assertionResultRecordSchema,
|
|
41
41
|
binaryVerdictSchema: () => binaryVerdictSchema,
|
|
42
|
+
formatIngestResult: () => formatIngestResult,
|
|
43
|
+
ingest: () => handleIngest,
|
|
42
44
|
parseRunResult: () => parseRunResult,
|
|
45
|
+
readCairnRun: () => readCairnRun,
|
|
46
|
+
reportIngestFailure: () => reportIngestFailure,
|
|
43
47
|
reportSyncFailure: () => reportSyncFailure,
|
|
44
48
|
run: () => handleRun,
|
|
45
49
|
runResultSchema: () => runResultSchema,
|
|
@@ -146,9 +150,9 @@ var modelPriceSchema = import_zod.z.object({
|
|
|
146
150
|
input_per_1k_usd: import_zod.z.number().nonnegative(),
|
|
147
151
|
output_per_1k_usd: import_zod.z.number().nonnegative()
|
|
148
152
|
});
|
|
149
|
-
var pricingSchema = import_zod.z.record(modelPriceSchema);
|
|
153
|
+
var pricingSchema = import_zod.z.record(import_zod.z.string(), modelPriceSchema);
|
|
150
154
|
var datasetRowSchema = import_zod.z.object({
|
|
151
|
-
vars: import_zod.z.record(import_zod.z.union([import_zod.z.string(), import_zod.z.number(), import_zod.z.boolean()])),
|
|
155
|
+
vars: import_zod.z.record(import_zod.z.string(), import_zod.z.union([import_zod.z.string(), import_zod.z.number(), import_zod.z.boolean()])),
|
|
152
156
|
expected: import_zod.z.string().optional()
|
|
153
157
|
});
|
|
154
158
|
var datasetRefSchema = import_zod.z.union([
|
|
@@ -178,7 +182,7 @@ var containsAllAssertionSchema = import_zod.z.object({
|
|
|
178
182
|
});
|
|
179
183
|
var jsonSchemaAssertionSchema = import_zod.z.object({
|
|
180
184
|
type: import_zod.z.literal("json-schema"),
|
|
181
|
-
schema: import_zod.z.record(import_zod.z.unknown()),
|
|
185
|
+
schema: import_zod.z.record(import_zod.z.string(), import_zod.z.unknown()),
|
|
182
186
|
extract: import_zod.z.enum(["auto", "strict"]).optional()
|
|
183
187
|
});
|
|
184
188
|
var llmJudgeAssertionSchema = import_zod.z.object({
|
|
@@ -644,9 +648,11 @@ var XenovaEmbedder = class {
|
|
|
644
648
|
return rows.map((row) => Float32Array.from(row));
|
|
645
649
|
}
|
|
646
650
|
load() {
|
|
647
|
-
this.extractor ??= import("@huggingface/transformers").
|
|
648
|
-
|
|
649
|
-
|
|
651
|
+
this.extractor ??= import("@huggingface/transformers").catch(() => {
|
|
652
|
+
throw new Error(
|
|
653
|
+
"The `semantic-similarity` and RAG assertions need local embeddings, which ship separately: install @huggingface/transformers to enable them. Every other assertion works without it."
|
|
654
|
+
);
|
|
655
|
+
}).then(({ pipeline }) => pipeline("feature-extraction", MODEL_ID));
|
|
650
656
|
return this.extractor;
|
|
651
657
|
}
|
|
652
658
|
};
|
|
@@ -1538,7 +1544,7 @@ var runErrorSchema = import_zod2.z.object({
|
|
|
1538
1544
|
message: import_zod2.z.string()
|
|
1539
1545
|
});
|
|
1540
1546
|
var rowResultSchema = import_zod2.z.object({
|
|
1541
|
-
vars: import_zod2.z.record(import_zod2.z.unknown()),
|
|
1547
|
+
vars: import_zod2.z.record(import_zod2.z.string(), import_zod2.z.unknown()),
|
|
1542
1548
|
output: import_zod2.z.string().nullable(),
|
|
1543
1549
|
cached: import_zod2.z.boolean(),
|
|
1544
1550
|
usage: usageSchema.optional(),
|
|
@@ -1573,6 +1579,13 @@ function parseRunResult(value) {
|
|
|
1573
1579
|
return runResultSchema.parse(value);
|
|
1574
1580
|
}
|
|
1575
1581
|
|
|
1582
|
+
// src/cli/api-url.ts
|
|
1583
|
+
var DEFAULT_API_URL = "https://beta-api.plune.ai";
|
|
1584
|
+
function resolveApiUrl(explicit) {
|
|
1585
|
+
const raw = explicit ?? process.env["PLUNE_API_URL"] ?? DEFAULT_API_URL;
|
|
1586
|
+
return raw.replace(/\/+$/, "");
|
|
1587
|
+
}
|
|
1588
|
+
|
|
1576
1589
|
// src/cli/credentials.ts
|
|
1577
1590
|
var import_node_fs = require("fs");
|
|
1578
1591
|
var import_node_os = require("os");
|
|
@@ -1591,7 +1604,6 @@ function loadToken() {
|
|
|
1591
1604
|
}
|
|
1592
1605
|
|
|
1593
1606
|
// src/cli/commands/sync.ts
|
|
1594
|
-
var DEFAULT_API_URL = "https://beta-api.plune.ai";
|
|
1595
1607
|
var NotLoggedInError = class extends Error {
|
|
1596
1608
|
constructor() {
|
|
1597
1609
|
super('Not logged in. Run "plune login" first to save your API token.');
|
|
@@ -1628,10 +1640,6 @@ var SyncHttpError = class extends Error {
|
|
|
1628
1640
|
}
|
|
1629
1641
|
status;
|
|
1630
1642
|
};
|
|
1631
|
-
function resolveApiUrl(explicit) {
|
|
1632
|
-
const raw = explicit ?? process.env["PLUNE_API_URL"] ?? DEFAULT_API_URL;
|
|
1633
|
-
return raw.replace(/\/+$/, "");
|
|
1634
|
-
}
|
|
1635
1643
|
function readRun(cwd, file) {
|
|
1636
1644
|
const target = file !== void 0 ? path5.resolve(cwd, file) : path5.join(cwd, ".plune", "last-run.json");
|
|
1637
1645
|
let raw;
|
|
@@ -1700,6 +1708,246 @@ function reportSyncFailure(err, write) {
|
|
|
1700
1708
|
}
|
|
1701
1709
|
return null;
|
|
1702
1710
|
}
|
|
1711
|
+
|
|
1712
|
+
// src/cli/commands/cairn-artifact.ts
|
|
1713
|
+
var fs7 = __toESM(require("fs/promises"), 1);
|
|
1714
|
+
var path6 = __toESM(require("path"), 1);
|
|
1715
|
+
var SUPPORTED_ARTIFACT_VERSION = 1;
|
|
1716
|
+
var INGEST_SCHEMA_VERSION = 1;
|
|
1717
|
+
var NoCairnRunsError = class extends Error {
|
|
1718
|
+
constructor(runsDir) {
|
|
1719
|
+
super(`No Cairn run found under ${runsDir} \u2014 name the directory instead: plune ingest <dir>.`);
|
|
1720
|
+
this.name = "NoCairnRunsError";
|
|
1721
|
+
}
|
|
1722
|
+
};
|
|
1723
|
+
async function findLatestCairnRun(runsDir = "runs") {
|
|
1724
|
+
const entries = await fs7.readdir(runsDir, { withFileTypes: true }).catch(() => null);
|
|
1725
|
+
if (entries === null) throw new NoCairnRunsError(runsDir);
|
|
1726
|
+
let newest;
|
|
1727
|
+
for (const entry of entries) {
|
|
1728
|
+
if (!entry.isDirectory()) continue;
|
|
1729
|
+
const dir = path6.join(runsDir, entry.name);
|
|
1730
|
+
try {
|
|
1731
|
+
const report = await fs7.lstat(path6.join(dir, "report.json"));
|
|
1732
|
+
if (!report.isFile()) continue;
|
|
1733
|
+
if (newest === void 0 || report.mtimeMs > newest.at) {
|
|
1734
|
+
newest = { dir, at: report.mtimeMs };
|
|
1735
|
+
}
|
|
1736
|
+
} catch {
|
|
1737
|
+
}
|
|
1738
|
+
}
|
|
1739
|
+
if (newest === void 0) throw new NoCairnRunsError(runsDir);
|
|
1740
|
+
return newest.dir;
|
|
1741
|
+
}
|
|
1742
|
+
var CairnRunNotFoundError = class extends Error {
|
|
1743
|
+
constructor(dir) {
|
|
1744
|
+
super(
|
|
1745
|
+
`No Cairn run found in ${dir} \u2014 expected a report.json written by "cairn design|explore|api".`
|
|
1746
|
+
);
|
|
1747
|
+
this.name = "CairnRunNotFoundError";
|
|
1748
|
+
}
|
|
1749
|
+
};
|
|
1750
|
+
var CairnRunVersionError = class extends Error {
|
|
1751
|
+
constructor(found) {
|
|
1752
|
+
super(
|
|
1753
|
+
found === void 0 ? "This run predates versioned Cairn artifacts. Re-run it with Cairn 0.7.0 or newer, which stamps the format so it can be read safely." : `Unsupported Cairn artifact version ${String(found)} (this CLI reads version ${SUPPORTED_ARTIFACT_VERSION}). Upgrade @plune-ai/cli.`
|
|
1754
|
+
);
|
|
1755
|
+
this.name = "CairnRunVersionError";
|
|
1756
|
+
}
|
|
1757
|
+
};
|
|
1758
|
+
var CairnRunPartialError = class extends Error {
|
|
1759
|
+
constructor(reason) {
|
|
1760
|
+
super(`That Cairn run did not finish, so its cases are incomplete: ${reason}`);
|
|
1761
|
+
this.name = "CairnRunPartialError";
|
|
1762
|
+
}
|
|
1763
|
+
};
|
|
1764
|
+
var CairnEvidenceAmbiguousError = class extends Error {
|
|
1765
|
+
constructor(name) {
|
|
1766
|
+
super(
|
|
1767
|
+
`Cannot attach results safely: the operation "${name}" appears more than once in this run's evidence, and the evidence file carries no case identity to disambiguate it. Nothing was uploaded.`
|
|
1768
|
+
);
|
|
1769
|
+
this.name = "CairnEvidenceAmbiguousError";
|
|
1770
|
+
}
|
|
1771
|
+
};
|
|
1772
|
+
var asRecord = (v) => typeof v === "object" && v !== null ? v : {};
|
|
1773
|
+
var str = (v, fallback = "") => typeof v === "string" && v !== "" ? v : fallback;
|
|
1774
|
+
async function readJson(file) {
|
|
1775
|
+
try {
|
|
1776
|
+
return JSON.parse(await fs7.readFile(file, "utf8"));
|
|
1777
|
+
} catch {
|
|
1778
|
+
return void 0;
|
|
1779
|
+
}
|
|
1780
|
+
}
|
|
1781
|
+
async function defaultReadEvidence(dir) {
|
|
1782
|
+
const raw = await readJson(path6.join(dir, "api-evidence.json"));
|
|
1783
|
+
if (!Array.isArray(raw)) return void 0;
|
|
1784
|
+
return raw.map((r) => {
|
|
1785
|
+
const rec = asRecord(r);
|
|
1786
|
+
return { name: str(rec["name"]), passed: rec["passed"] === true };
|
|
1787
|
+
});
|
|
1788
|
+
}
|
|
1789
|
+
function casesFromDesign(report) {
|
|
1790
|
+
const list = Array.isArray(report["testCases"]) ? report["testCases"] : [];
|
|
1791
|
+
return list.map((raw) => {
|
|
1792
|
+
const c = asRecord(raw);
|
|
1793
|
+
const steps = (Array.isArray(c["steps"]) ? c["steps"] : []).map((s) => str(s)).filter((s) => s !== "");
|
|
1794
|
+
return {
|
|
1795
|
+
stableId: str(c["stableId"]),
|
|
1796
|
+
title: str(c["title"], "untitled case"),
|
|
1797
|
+
execution: c["execution"] === "manual" ? "manual" : "auto",
|
|
1798
|
+
steps: steps.length > 0 ? steps : ["(no steps recorded)"],
|
|
1799
|
+
expected: str(c["expected"], "(no expected result recorded)"),
|
|
1800
|
+
...typeof c["technique"] === "string" ? { technique: c["technique"] } : {},
|
|
1801
|
+
...typeof c["priority"] === "string" ? { priority: c["priority"] } : {}
|
|
1802
|
+
};
|
|
1803
|
+
});
|
|
1804
|
+
}
|
|
1805
|
+
function casesFromApi(report, verdicts) {
|
|
1806
|
+
const list = Array.isArray(report["cases"]) ? report["cases"] : [];
|
|
1807
|
+
return list.map((raw) => {
|
|
1808
|
+
const c = asRecord(raw);
|
|
1809
|
+
const name = str(c["name"], "unnamed operation");
|
|
1810
|
+
const request = `${str(c["method"], "GET")} ${str(c["path"], "/")}`;
|
|
1811
|
+
const verdict = verdicts.get(name);
|
|
1812
|
+
return {
|
|
1813
|
+
stableId: str(c["stableId"]),
|
|
1814
|
+
title: name,
|
|
1815
|
+
execution: "auto",
|
|
1816
|
+
steps: [request],
|
|
1817
|
+
expected: `HTTP ${str(c["expectedStatus"], "default")}`,
|
|
1818
|
+
...verdict !== void 0 ? { verdict } : {},
|
|
1819
|
+
...typeof c["technique"] === "string" ? { technique: c["technique"] } : {}
|
|
1820
|
+
};
|
|
1821
|
+
});
|
|
1822
|
+
}
|
|
1823
|
+
async function readCairnRun(dir, deps = {}) {
|
|
1824
|
+
const raw = await readJson(path6.join(dir, "report.json"));
|
|
1825
|
+
if (raw === void 0) throw new CairnRunNotFoundError(dir);
|
|
1826
|
+
const report = asRecord(raw);
|
|
1827
|
+
if (report["schemaVersion"] !== SUPPORTED_ARTIFACT_VERSION) {
|
|
1828
|
+
throw new CairnRunVersionError(report["schemaVersion"]);
|
|
1829
|
+
}
|
|
1830
|
+
if (report["partial"] === true) {
|
|
1831
|
+
throw new CairnRunPartialError(str(report["error"], "no reason recorded"));
|
|
1832
|
+
}
|
|
1833
|
+
const mode = report["mode"];
|
|
1834
|
+
if (mode !== "design" && mode !== "explore" && mode !== "api") {
|
|
1835
|
+
throw new CairnRunVersionError(
|
|
1836
|
+
`${SUPPORTED_ARTIFACT_VERSION} (unknown run mode ${String(mode)})`
|
|
1837
|
+
);
|
|
1838
|
+
}
|
|
1839
|
+
let cases;
|
|
1840
|
+
if (mode === "api") {
|
|
1841
|
+
const evidence = await (deps.readEvidence ?? defaultReadEvidence)(dir);
|
|
1842
|
+
const verdicts = /* @__PURE__ */ new Map();
|
|
1843
|
+
for (const e of evidence ?? []) {
|
|
1844
|
+
if (verdicts.has(e.name)) throw new CairnEvidenceAmbiguousError(e.name);
|
|
1845
|
+
verdicts.set(e.name, e.passed ? "passed" : "failed");
|
|
1846
|
+
}
|
|
1847
|
+
cases = casesFromApi(report, verdicts);
|
|
1848
|
+
} else {
|
|
1849
|
+
cases = casesFromDesign(report);
|
|
1850
|
+
}
|
|
1851
|
+
return {
|
|
1852
|
+
schemaVersion: INGEST_SCHEMA_VERSION,
|
|
1853
|
+
run: {
|
|
1854
|
+
runId: str(report["runId"], path6.basename(dir)),
|
|
1855
|
+
mode,
|
|
1856
|
+
target: str(report["url"], "unknown")
|
|
1857
|
+
},
|
|
1858
|
+
cases
|
|
1859
|
+
};
|
|
1860
|
+
}
|
|
1861
|
+
|
|
1862
|
+
// src/cli/commands/ingest.ts
|
|
1863
|
+
var IngestNotLoggedInError = class extends Error {
|
|
1864
|
+
constructor() {
|
|
1865
|
+
super('Not logged in. Run "plune login" first to save your API token.');
|
|
1866
|
+
this.name = "IngestNotLoggedInError";
|
|
1867
|
+
}
|
|
1868
|
+
};
|
|
1869
|
+
var IngestTokenRejectedError = class extends Error {
|
|
1870
|
+
constructor() {
|
|
1871
|
+
super('Your API token was rejected (401). Run "plune login" again with a fresh token.');
|
|
1872
|
+
this.name = "IngestTokenRejectedError";
|
|
1873
|
+
}
|
|
1874
|
+
};
|
|
1875
|
+
var IngestNetworkError = class extends Error {
|
|
1876
|
+
constructor(apiUrl) {
|
|
1877
|
+
super(`Could not reach the Plune API at ${apiUrl}. Check your connection, then try again.`);
|
|
1878
|
+
this.name = "IngestNetworkError";
|
|
1879
|
+
}
|
|
1880
|
+
};
|
|
1881
|
+
var IngestHttpError = class extends Error {
|
|
1882
|
+
constructor(status, detail) {
|
|
1883
|
+
super(`Plune API returned ${status}${detail !== "" ? ` \u2014 ${detail}` : ""}.`);
|
|
1884
|
+
this.status = status;
|
|
1885
|
+
this.name = "IngestHttpError";
|
|
1886
|
+
}
|
|
1887
|
+
status;
|
|
1888
|
+
};
|
|
1889
|
+
async function safeErrorDetail2(res) {
|
|
1890
|
+
try {
|
|
1891
|
+
const body = await res.json();
|
|
1892
|
+
return typeof body.error === "string" ? body.error : "";
|
|
1893
|
+
} catch {
|
|
1894
|
+
return "";
|
|
1895
|
+
}
|
|
1896
|
+
}
|
|
1897
|
+
async function handleIngest(deps) {
|
|
1898
|
+
const apiUrl = resolveApiUrl(deps.apiUrl);
|
|
1899
|
+
const token = (deps.loadToken ?? loadToken)();
|
|
1900
|
+
if (token === null) throw new IngestNotLoggedInError();
|
|
1901
|
+
const dir = deps.dir ?? await findLatestCairnRun(deps.runsDir);
|
|
1902
|
+
const payload = await readCairnRun(dir);
|
|
1903
|
+
const doFetch = deps.fetchImpl ?? fetch;
|
|
1904
|
+
const endpoint = `${apiUrl}/v1/ingest/cairn`;
|
|
1905
|
+
let res;
|
|
1906
|
+
try {
|
|
1907
|
+
res = await doFetch(endpoint, {
|
|
1908
|
+
method: "POST",
|
|
1909
|
+
headers: { "content-type": "application/json", authorization: `Bearer ${token}` },
|
|
1910
|
+
body: JSON.stringify(payload)
|
|
1911
|
+
});
|
|
1912
|
+
} catch {
|
|
1913
|
+
throw new IngestNetworkError(apiUrl);
|
|
1914
|
+
}
|
|
1915
|
+
if (!res.ok) {
|
|
1916
|
+
const detail = await safeErrorDetail2(res);
|
|
1917
|
+
if (res.status === 401) throw new IngestTokenRejectedError();
|
|
1918
|
+
throw new IngestHttpError(res.status, detail);
|
|
1919
|
+
}
|
|
1920
|
+
const body = await res.json().catch(() => ({}));
|
|
1921
|
+
return {
|
|
1922
|
+
runId: typeof body.runId === "string" ? body.runId : "",
|
|
1923
|
+
linked: typeof body.linked === "number" ? body.linked : 0,
|
|
1924
|
+
proposed: typeof body.proposed === "number" ? body.proposed : 0,
|
|
1925
|
+
skipped: typeof body.skipped === "number" ? body.skipped : 0
|
|
1926
|
+
};
|
|
1927
|
+
}
|
|
1928
|
+
function reportIngestFailure(err, write) {
|
|
1929
|
+
if (err instanceof IngestNotLoggedInError || err instanceof IngestTokenRejectedError || err instanceof CairnRunNotFoundError || err instanceof NoCairnRunsError || err instanceof CairnRunVersionError || err instanceof CairnRunPartialError || err instanceof CairnEvidenceAmbiguousError) {
|
|
1930
|
+
write(`${err.message}
|
|
1931
|
+
`);
|
|
1932
|
+
return 2;
|
|
1933
|
+
}
|
|
1934
|
+
if (err instanceof IngestNetworkError || err instanceof IngestHttpError) {
|
|
1935
|
+
write(`${err.message}
|
|
1936
|
+
`);
|
|
1937
|
+
return 1;
|
|
1938
|
+
}
|
|
1939
|
+
return null;
|
|
1940
|
+
}
|
|
1941
|
+
function formatIngestResult(r) {
|
|
1942
|
+
const lines = [
|
|
1943
|
+
`Ingested Cairn run \u2192 ${r.runId}`,
|
|
1944
|
+
` ${r.linked} result(s) attached to existing cases`,
|
|
1945
|
+
` ${r.proposed} case(s) proposed \u2014 review them in Plune to accept`,
|
|
1946
|
+
` ${r.skipped} skipped (already queued, already rejected, repeated, or never executed)`
|
|
1947
|
+
];
|
|
1948
|
+
return `${lines.join("\n")}
|
|
1949
|
+
`;
|
|
1950
|
+
}
|
|
1703
1951
|
// Annotate the CommonJS export names for ESM import in node:
|
|
1704
1952
|
0 && (module.exports = {
|
|
1705
1953
|
NotLoggedInError,
|
|
@@ -1711,7 +1959,11 @@ function reportSyncFailure(err, write) {
|
|
|
1711
1959
|
assertionConfigSchema,
|
|
1712
1960
|
assertionResultRecordSchema,
|
|
1713
1961
|
binaryVerdictSchema,
|
|
1962
|
+
formatIngestResult,
|
|
1963
|
+
ingest,
|
|
1714
1964
|
parseRunResult,
|
|
1965
|
+
readCairnRun,
|
|
1966
|
+
reportIngestFailure,
|
|
1715
1967
|
reportSyncFailure,
|
|
1716
1968
|
run,
|
|
1717
1969
|
runResultSchema,
|