@plune-ai/cli 0.6.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -1
- package/dist/cli.cjs +915 -162
- package/dist/index.cjs +46 -19
- package/dist/index.d.cts +39 -424
- package/dist/index.d.ts +39 -424
- package/dist/index.js +46 -19
- package/dist/reporter-core.cjs +467 -0
- package/dist/reporter-core.d.cts +339 -0
- package/dist/reporter-core.d.ts +339 -0
- package/dist/reporter-core.js +423 -0
- package/package.json +12 -3
package/dist/index.cjs
CHANGED
|
@@ -150,9 +150,9 @@ var modelPriceSchema = import_zod.z.object({
|
|
|
150
150
|
input_per_1k_usd: import_zod.z.number().nonnegative(),
|
|
151
151
|
output_per_1k_usd: import_zod.z.number().nonnegative()
|
|
152
152
|
});
|
|
153
|
-
var pricingSchema = import_zod.z.record(modelPriceSchema);
|
|
153
|
+
var pricingSchema = import_zod.z.record(import_zod.z.string(), modelPriceSchema);
|
|
154
154
|
var datasetRowSchema = import_zod.z.object({
|
|
155
|
-
vars: import_zod.z.record(import_zod.z.union([import_zod.z.string(), import_zod.z.number(), import_zod.z.boolean()])),
|
|
155
|
+
vars: import_zod.z.record(import_zod.z.string(), import_zod.z.union([import_zod.z.string(), import_zod.z.number(), import_zod.z.boolean()])),
|
|
156
156
|
expected: import_zod.z.string().optional()
|
|
157
157
|
});
|
|
158
158
|
var datasetRefSchema = import_zod.z.union([
|
|
@@ -182,7 +182,7 @@ var containsAllAssertionSchema = import_zod.z.object({
|
|
|
182
182
|
});
|
|
183
183
|
var jsonSchemaAssertionSchema = import_zod.z.object({
|
|
184
184
|
type: import_zod.z.literal("json-schema"),
|
|
185
|
-
schema: import_zod.z.record(import_zod.z.unknown()),
|
|
185
|
+
schema: import_zod.z.record(import_zod.z.string(), import_zod.z.unknown()),
|
|
186
186
|
extract: import_zod.z.enum(["auto", "strict"]).optional()
|
|
187
187
|
});
|
|
188
188
|
var llmJudgeAssertionSchema = import_zod.z.object({
|
|
@@ -1544,7 +1544,7 @@ var runErrorSchema = import_zod2.z.object({
|
|
|
1544
1544
|
message: import_zod2.z.string()
|
|
1545
1545
|
});
|
|
1546
1546
|
var rowResultSchema = import_zod2.z.object({
|
|
1547
|
-
vars: import_zod2.z.record(import_zod2.z.unknown()),
|
|
1547
|
+
vars: import_zod2.z.record(import_zod2.z.string(), import_zod2.z.unknown()),
|
|
1548
1548
|
output: import_zod2.z.string().nullable(),
|
|
1549
1549
|
cached: import_zod2.z.boolean(),
|
|
1550
1550
|
usage: usageSchema.optional(),
|
|
@@ -1579,6 +1579,13 @@ function parseRunResult(value) {
|
|
|
1579
1579
|
return runResultSchema.parse(value);
|
|
1580
1580
|
}
|
|
1581
1581
|
|
|
1582
|
+
// src/cli/api-url.ts
|
|
1583
|
+
var DEFAULT_API_URL = "https://beta-api.plune.ai";
|
|
1584
|
+
function resolveApiUrl(explicit) {
|
|
1585
|
+
const raw = explicit ?? process.env["PLUNE_API_URL"] ?? DEFAULT_API_URL;
|
|
1586
|
+
return raw.replace(/\/+$/, "");
|
|
1587
|
+
}
|
|
1588
|
+
|
|
1582
1589
|
// src/cli/credentials.ts
|
|
1583
1590
|
var import_node_fs = require("fs");
|
|
1584
1591
|
var import_node_os = require("os");
|
|
@@ -1597,7 +1604,6 @@ function loadToken() {
|
|
|
1597
1604
|
}
|
|
1598
1605
|
|
|
1599
1606
|
// src/cli/commands/sync.ts
|
|
1600
|
-
var DEFAULT_API_URL = "https://beta-api.plune.ai";
|
|
1601
1607
|
var NotLoggedInError = class extends Error {
|
|
1602
1608
|
constructor() {
|
|
1603
1609
|
super('Not logged in. Run "plune login" first to save your API token.');
|
|
@@ -1634,10 +1640,6 @@ var SyncHttpError = class extends Error {
|
|
|
1634
1640
|
}
|
|
1635
1641
|
status;
|
|
1636
1642
|
};
|
|
1637
|
-
function resolveApiUrl(explicit) {
|
|
1638
|
-
const raw = explicit ?? process.env["PLUNE_API_URL"] ?? DEFAULT_API_URL;
|
|
1639
|
-
return raw.replace(/\/+$/, "");
|
|
1640
|
-
}
|
|
1641
1643
|
function readRun(cwd, file) {
|
|
1642
1644
|
const target = file !== void 0 ? path5.resolve(cwd, file) : path5.join(cwd, ".plune", "last-run.json");
|
|
1643
1645
|
let raw;
|
|
@@ -1712,9 +1714,36 @@ var fs7 = __toESM(require("fs/promises"), 1);
|
|
|
1712
1714
|
var path6 = __toESM(require("path"), 1);
|
|
1713
1715
|
var SUPPORTED_ARTIFACT_VERSION = 1;
|
|
1714
1716
|
var INGEST_SCHEMA_VERSION = 1;
|
|
1717
|
+
var NoCairnRunsError = class extends Error {
|
|
1718
|
+
constructor(runsDir) {
|
|
1719
|
+
super(`No Cairn run found under ${runsDir} \u2014 name the directory instead: plune ingest <dir>.`);
|
|
1720
|
+
this.name = "NoCairnRunsError";
|
|
1721
|
+
}
|
|
1722
|
+
};
|
|
1723
|
+
async function findLatestCairnRun(runsDir = "runs") {
|
|
1724
|
+
const entries = await fs7.readdir(runsDir, { withFileTypes: true }).catch(() => null);
|
|
1725
|
+
if (entries === null) throw new NoCairnRunsError(runsDir);
|
|
1726
|
+
let newest;
|
|
1727
|
+
for (const entry of entries) {
|
|
1728
|
+
if (!entry.isDirectory()) continue;
|
|
1729
|
+
const dir = path6.join(runsDir, entry.name);
|
|
1730
|
+
try {
|
|
1731
|
+
const report = await fs7.lstat(path6.join(dir, "report.json"));
|
|
1732
|
+
if (!report.isFile()) continue;
|
|
1733
|
+
if (newest === void 0 || report.mtimeMs > newest.at) {
|
|
1734
|
+
newest = { dir, at: report.mtimeMs };
|
|
1735
|
+
}
|
|
1736
|
+
} catch {
|
|
1737
|
+
}
|
|
1738
|
+
}
|
|
1739
|
+
if (newest === void 0) throw new NoCairnRunsError(runsDir);
|
|
1740
|
+
return newest.dir;
|
|
1741
|
+
}
|
|
1715
1742
|
var CairnRunNotFoundError = class extends Error {
|
|
1716
1743
|
constructor(dir) {
|
|
1717
|
-
super(
|
|
1744
|
+
super(
|
|
1745
|
+
`No Cairn run found in ${dir} \u2014 expected a report.json written by "cairn design|explore|api".`
|
|
1746
|
+
);
|
|
1718
1747
|
this.name = "CairnRunNotFoundError";
|
|
1719
1748
|
}
|
|
1720
1749
|
};
|
|
@@ -1803,7 +1832,9 @@ async function readCairnRun(dir, deps = {}) {
|
|
|
1803
1832
|
}
|
|
1804
1833
|
const mode = report["mode"];
|
|
1805
1834
|
if (mode !== "design" && mode !== "explore" && mode !== "api") {
|
|
1806
|
-
throw new CairnRunVersionError(
|
|
1835
|
+
throw new CairnRunVersionError(
|
|
1836
|
+
`${SUPPORTED_ARTIFACT_VERSION} (unknown run mode ${String(mode)})`
|
|
1837
|
+
);
|
|
1807
1838
|
}
|
|
1808
1839
|
let cases;
|
|
1809
1840
|
if (mode === "api") {
|
|
@@ -1829,7 +1860,6 @@ async function readCairnRun(dir, deps = {}) {
|
|
|
1829
1860
|
}
|
|
1830
1861
|
|
|
1831
1862
|
// src/cli/commands/ingest.ts
|
|
1832
|
-
var DEFAULT_API_URL2 = "https://beta-api.plune.ai";
|
|
1833
1863
|
var IngestNotLoggedInError = class extends Error {
|
|
1834
1864
|
constructor() {
|
|
1835
1865
|
super('Not logged in. Run "plune login" first to save your API token.');
|
|
@@ -1856,10 +1886,6 @@ var IngestHttpError = class extends Error {
|
|
|
1856
1886
|
}
|
|
1857
1887
|
status;
|
|
1858
1888
|
};
|
|
1859
|
-
function resolveApiUrl2(explicit) {
|
|
1860
|
-
const raw = explicit ?? process.env["PLUNE_API_URL"] ?? DEFAULT_API_URL2;
|
|
1861
|
-
return raw.replace(/\/+$/, "");
|
|
1862
|
-
}
|
|
1863
1889
|
async function safeErrorDetail2(res) {
|
|
1864
1890
|
try {
|
|
1865
1891
|
const body = await res.json();
|
|
@@ -1869,10 +1895,11 @@ async function safeErrorDetail2(res) {
|
|
|
1869
1895
|
}
|
|
1870
1896
|
}
|
|
1871
1897
|
async function handleIngest(deps) {
|
|
1872
|
-
const apiUrl =
|
|
1898
|
+
const apiUrl = resolveApiUrl(deps.apiUrl);
|
|
1873
1899
|
const token = (deps.loadToken ?? loadToken)();
|
|
1874
1900
|
if (token === null) throw new IngestNotLoggedInError();
|
|
1875
|
-
const
|
|
1901
|
+
const dir = deps.dir ?? await findLatestCairnRun(deps.runsDir);
|
|
1902
|
+
const payload = await readCairnRun(dir);
|
|
1876
1903
|
const doFetch = deps.fetchImpl ?? fetch;
|
|
1877
1904
|
const endpoint = `${apiUrl}/v1/ingest/cairn`;
|
|
1878
1905
|
let res;
|
|
@@ -1899,7 +1926,7 @@ async function handleIngest(deps) {
|
|
|
1899
1926
|
};
|
|
1900
1927
|
}
|
|
1901
1928
|
function reportIngestFailure(err, write) {
|
|
1902
|
-
if (err instanceof IngestNotLoggedInError || err instanceof IngestTokenRejectedError || err instanceof CairnRunNotFoundError || err instanceof CairnRunVersionError || err instanceof CairnRunPartialError || err instanceof CairnEvidenceAmbiguousError) {
|
|
1929
|
+
if (err instanceof IngestNotLoggedInError || err instanceof IngestTokenRejectedError || err instanceof CairnRunNotFoundError || err instanceof NoCairnRunsError || err instanceof CairnRunVersionError || err instanceof CairnRunPartialError || err instanceof CairnEvidenceAmbiguousError) {
|
|
1903
1930
|
write(`${err.message}
|
|
1904
1931
|
`);
|
|
1905
1932
|
return 2;
|
package/dist/index.d.cts
CHANGED
|
@@ -265,176 +265,65 @@ declare function handleSync(deps?: SyncDeps): Promise<SyncResult>;
|
|
|
265
265
|
*/
|
|
266
266
|
declare function reportSyncFailure(err: unknown, write: (s: string) => void): number | null;
|
|
267
267
|
|
|
268
|
-
declare const assertionConfigSchema: z.ZodDiscriminatedUnion<
|
|
268
|
+
declare const assertionConfigSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
269
269
|
type: z.ZodLiteral<"exact-match">;
|
|
270
270
|
value: z.ZodString;
|
|
271
271
|
trim: z.ZodOptional<z.ZodBoolean>;
|
|
272
272
|
ignore_case: z.ZodOptional<z.ZodBoolean>;
|
|
273
|
-
},
|
|
274
|
-
type: "exact-match";
|
|
275
|
-
value: string;
|
|
276
|
-
trim?: boolean | undefined;
|
|
277
|
-
ignore_case?: boolean | undefined;
|
|
278
|
-
}, {
|
|
279
|
-
type: "exact-match";
|
|
280
|
-
value: string;
|
|
281
|
-
trim?: boolean | undefined;
|
|
282
|
-
ignore_case?: boolean | undefined;
|
|
283
|
-
}>, z.ZodObject<{
|
|
273
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
284
274
|
type: z.ZodLiteral<"contains">;
|
|
285
275
|
value: z.ZodString;
|
|
286
276
|
ignore_case: z.ZodOptional<z.ZodBoolean>;
|
|
287
|
-
},
|
|
288
|
-
type: "contains";
|
|
289
|
-
value: string;
|
|
290
|
-
ignore_case?: boolean | undefined;
|
|
291
|
-
}, {
|
|
292
|
-
type: "contains";
|
|
293
|
-
value: string;
|
|
294
|
-
ignore_case?: boolean | undefined;
|
|
295
|
-
}>, z.ZodObject<{
|
|
277
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
296
278
|
type: z.ZodLiteral<"contains-any">;
|
|
297
|
-
values: z.ZodArray<z.ZodString
|
|
279
|
+
values: z.ZodArray<z.ZodString>;
|
|
298
280
|
ignore_case: z.ZodOptional<z.ZodBoolean>;
|
|
299
|
-
},
|
|
300
|
-
type: "contains-any";
|
|
301
|
-
values: string[];
|
|
302
|
-
ignore_case?: boolean | undefined;
|
|
303
|
-
}, {
|
|
304
|
-
type: "contains-any";
|
|
305
|
-
values: string[];
|
|
306
|
-
ignore_case?: boolean | undefined;
|
|
307
|
-
}>, z.ZodObject<{
|
|
281
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
308
282
|
type: z.ZodLiteral<"contains-all">;
|
|
309
|
-
values: z.ZodArray<z.ZodString
|
|
283
|
+
values: z.ZodArray<z.ZodString>;
|
|
310
284
|
ignore_case: z.ZodOptional<z.ZodBoolean>;
|
|
311
|
-
},
|
|
312
|
-
type: "contains-all";
|
|
313
|
-
values: string[];
|
|
314
|
-
ignore_case?: boolean | undefined;
|
|
315
|
-
}, {
|
|
316
|
-
type: "contains-all";
|
|
317
|
-
values: string[];
|
|
318
|
-
ignore_case?: boolean | undefined;
|
|
319
|
-
}>, z.ZodObject<{
|
|
285
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
320
286
|
type: z.ZodLiteral<"json-schema">;
|
|
321
287
|
schema: z.ZodRecord<z.ZodString, z.ZodUnknown>;
|
|
322
|
-
extract: z.ZodOptional<z.ZodEnum<
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
}, {
|
|
328
|
-
type: "json-schema";
|
|
329
|
-
schema: Record<string, unknown>;
|
|
330
|
-
extract?: "auto" | "strict" | undefined;
|
|
331
|
-
}>, z.ZodObject<{
|
|
288
|
+
extract: z.ZodOptional<z.ZodEnum<{
|
|
289
|
+
auto: "auto";
|
|
290
|
+
strict: "strict";
|
|
291
|
+
}>>;
|
|
292
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
332
293
|
type: z.ZodLiteral<"llm-judge">;
|
|
333
294
|
criteria: z.ZodString;
|
|
334
295
|
provider: z.ZodOptional<z.ZodObject<{
|
|
335
|
-
type: z.ZodOptional<z.ZodEnum<
|
|
296
|
+
type: z.ZodOptional<z.ZodEnum<{
|
|
297
|
+
anthropic: "anthropic";
|
|
298
|
+
openai: "openai";
|
|
299
|
+
openrouter: "openrouter";
|
|
300
|
+
}>>;
|
|
336
301
|
model: z.ZodOptional<z.ZodString>;
|
|
337
302
|
temperature: z.ZodOptional<z.ZodOptional<z.ZodNumber>>;
|
|
338
303
|
max_tokens: z.ZodOptional<z.ZodOptional<z.ZodNumber>>;
|
|
339
304
|
concurrency: z.ZodOptional<z.ZodOptional<z.ZodNumber>>;
|
|
340
305
|
timeout: z.ZodOptional<z.ZodOptional<z.ZodNumber>>;
|
|
341
306
|
max_retries: z.ZodOptional<z.ZodOptional<z.ZodNumber>>;
|
|
342
|
-
},
|
|
343
|
-
type?: "anthropic" | "openai" | "openrouter" | undefined;
|
|
344
|
-
model?: string | undefined;
|
|
345
|
-
temperature?: number | undefined;
|
|
346
|
-
max_tokens?: number | undefined;
|
|
347
|
-
concurrency?: number | undefined;
|
|
348
|
-
timeout?: number | undefined;
|
|
349
|
-
max_retries?: number | undefined;
|
|
350
|
-
}, {
|
|
351
|
-
type?: "anthropic" | "openai" | "openrouter" | undefined;
|
|
352
|
-
model?: string | undefined;
|
|
353
|
-
temperature?: number | undefined;
|
|
354
|
-
max_tokens?: number | undefined;
|
|
355
|
-
concurrency?: number | undefined;
|
|
356
|
-
timeout?: number | undefined;
|
|
357
|
-
max_retries?: number | undefined;
|
|
358
|
-
}>>;
|
|
307
|
+
}, z.core.$strip>>;
|
|
359
308
|
pass_threshold: z.ZodOptional<z.ZodNumber>;
|
|
360
|
-
},
|
|
361
|
-
type: "llm-judge";
|
|
362
|
-
criteria: string;
|
|
363
|
-
provider?: {
|
|
364
|
-
type?: "anthropic" | "openai" | "openrouter" | undefined;
|
|
365
|
-
model?: string | undefined;
|
|
366
|
-
temperature?: number | undefined;
|
|
367
|
-
max_tokens?: number | undefined;
|
|
368
|
-
concurrency?: number | undefined;
|
|
369
|
-
timeout?: number | undefined;
|
|
370
|
-
max_retries?: number | undefined;
|
|
371
|
-
} | undefined;
|
|
372
|
-
pass_threshold?: number | undefined;
|
|
373
|
-
}, {
|
|
374
|
-
type: "llm-judge";
|
|
375
|
-
criteria: string;
|
|
376
|
-
provider?: {
|
|
377
|
-
type?: "anthropic" | "openai" | "openrouter" | undefined;
|
|
378
|
-
model?: string | undefined;
|
|
379
|
-
temperature?: number | undefined;
|
|
380
|
-
max_tokens?: number | undefined;
|
|
381
|
-
concurrency?: number | undefined;
|
|
382
|
-
timeout?: number | undefined;
|
|
383
|
-
max_retries?: number | undefined;
|
|
384
|
-
} | undefined;
|
|
385
|
-
pass_threshold?: number | undefined;
|
|
386
|
-
}>, z.ZodObject<{
|
|
309
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
387
310
|
type: z.ZodLiteral<"semantic-similarity">;
|
|
388
311
|
reference: z.ZodString;
|
|
389
312
|
threshold: z.ZodOptional<z.ZodNumber>;
|
|
390
|
-
},
|
|
391
|
-
type: "semantic-similarity";
|
|
392
|
-
reference: string;
|
|
393
|
-
threshold?: number | undefined;
|
|
394
|
-
}, {
|
|
395
|
-
type: "semantic-similarity";
|
|
396
|
-
reference: string;
|
|
397
|
-
threshold?: number | undefined;
|
|
398
|
-
}>, z.ZodObject<{
|
|
313
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
399
314
|
type: z.ZodLiteral<"faithfulness">;
|
|
400
315
|
context: z.ZodString;
|
|
401
316
|
threshold: z.ZodOptional<z.ZodNumber>;
|
|
402
|
-
},
|
|
403
|
-
type: "faithfulness";
|
|
404
|
-
context: string;
|
|
405
|
-
threshold?: number | undefined;
|
|
406
|
-
}, {
|
|
407
|
-
type: "faithfulness";
|
|
408
|
-
context: string;
|
|
409
|
-
threshold?: number | undefined;
|
|
410
|
-
}>, z.ZodObject<{
|
|
317
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
411
318
|
type: z.ZodLiteral<"answer-relevance">;
|
|
412
319
|
question: z.ZodString;
|
|
413
320
|
threshold: z.ZodOptional<z.ZodNumber>;
|
|
414
|
-
},
|
|
415
|
-
type: "answer-relevance";
|
|
416
|
-
question: string;
|
|
417
|
-
threshold?: number | undefined;
|
|
418
|
-
}, {
|
|
419
|
-
type: "answer-relevance";
|
|
420
|
-
question: string;
|
|
421
|
-
threshold?: number | undefined;
|
|
422
|
-
}>, z.ZodObject<{
|
|
321
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
423
322
|
type: z.ZodLiteral<"context-precision">;
|
|
424
323
|
context: z.ZodString;
|
|
425
324
|
question: z.ZodString;
|
|
426
325
|
threshold: z.ZodOptional<z.ZodNumber>;
|
|
427
|
-
},
|
|
428
|
-
type: "context-precision";
|
|
429
|
-
context: string;
|
|
430
|
-
question: string;
|
|
431
|
-
threshold?: number | undefined;
|
|
432
|
-
}, {
|
|
433
|
-
type: "context-precision";
|
|
434
|
-
context: string;
|
|
435
|
-
question: string;
|
|
436
|
-
threshold?: number | undefined;
|
|
437
|
-
}>]>;
|
|
326
|
+
}, z.core.$strip>], "type">;
|
|
438
327
|
|
|
439
328
|
/**
|
|
440
329
|
* Runtime validation for the RunResult contract — the JSON that `plune run --format json` emits.
|
|
@@ -458,13 +347,7 @@ declare const RUN_RESULT_SCHEMA_VERSION: RunResult['schemaVersion'];
|
|
|
458
347
|
declare const binaryVerdictSchema: z.ZodObject<{
|
|
459
348
|
question: z.ZodString;
|
|
460
349
|
passed: z.ZodBoolean;
|
|
461
|
-
},
|
|
462
|
-
question: string;
|
|
463
|
-
passed: boolean;
|
|
464
|
-
}, {
|
|
465
|
-
question: string;
|
|
466
|
-
passed: boolean;
|
|
467
|
-
}>;
|
|
350
|
+
}, z.core.$strip>;
|
|
468
351
|
declare const assertionResultRecordSchema: z.ZodObject<{
|
|
469
352
|
type: z.ZodString;
|
|
470
353
|
passed: z.ZodBoolean;
|
|
@@ -473,32 +356,8 @@ declare const assertionResultRecordSchema: z.ZodObject<{
|
|
|
473
356
|
binaryVerdicts: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
474
357
|
question: z.ZodString;
|
|
475
358
|
passed: z.ZodBoolean;
|
|
476
|
-
},
|
|
477
|
-
|
|
478
|
-
passed: boolean;
|
|
479
|
-
}, {
|
|
480
|
-
question: string;
|
|
481
|
-
passed: boolean;
|
|
482
|
-
}>, "many">>;
|
|
483
|
-
}, "strip", z.ZodTypeAny, {
|
|
484
|
-
type: string;
|
|
485
|
-
passed: boolean;
|
|
486
|
-
score?: number | undefined;
|
|
487
|
-
reason?: string | undefined;
|
|
488
|
-
binaryVerdicts?: {
|
|
489
|
-
question: string;
|
|
490
|
-
passed: boolean;
|
|
491
|
-
}[] | undefined;
|
|
492
|
-
}, {
|
|
493
|
-
type: string;
|
|
494
|
-
passed: boolean;
|
|
495
|
-
score?: number | undefined;
|
|
496
|
-
reason?: string | undefined;
|
|
497
|
-
binaryVerdicts?: {
|
|
498
|
-
question: string;
|
|
499
|
-
passed: boolean;
|
|
500
|
-
}[] | undefined;
|
|
501
|
-
}>;
|
|
359
|
+
}, z.core.$strip>>>;
|
|
360
|
+
}, z.core.$strip>;
|
|
502
361
|
declare const runResultSchema: z.ZodObject<{
|
|
503
362
|
schemaVersion: z.ZodLiteral<1>;
|
|
504
363
|
plune_version: z.ZodString;
|
|
@@ -512,24 +371,10 @@ declare const runResultSchema: z.ZodObject<{
|
|
|
512
371
|
errored: z.ZodNumber;
|
|
513
372
|
cost_usd: z.ZodNumber;
|
|
514
373
|
duration_ms: z.ZodNumber;
|
|
515
|
-
},
|
|
516
|
-
cost_usd: number;
|
|
517
|
-
passed: number;
|
|
518
|
-
failed: number;
|
|
519
|
-
errored: number;
|
|
520
|
-
total: number;
|
|
521
|
-
duration_ms: number;
|
|
522
|
-
}, {
|
|
523
|
-
cost_usd: number;
|
|
524
|
-
passed: number;
|
|
525
|
-
failed: number;
|
|
526
|
-
errored: number;
|
|
527
|
-
total: number;
|
|
528
|
-
duration_ms: number;
|
|
529
|
-
}>;
|
|
374
|
+
}, z.core.$strip>;
|
|
530
375
|
evals: z.ZodArray<z.ZodObject<{
|
|
531
376
|
id: z.ZodString;
|
|
532
|
-
tags: z.ZodArray<z.ZodString
|
|
377
|
+
tags: z.ZodArray<z.ZodString>;
|
|
533
378
|
rows: z.ZodArray<z.ZodObject<{
|
|
534
379
|
vars: z.ZodRecord<z.ZodString, z.ZodUnknown>;
|
|
535
380
|
output: z.ZodNullable<z.ZodString>;
|
|
@@ -538,26 +383,12 @@ declare const runResultSchema: z.ZodObject<{
|
|
|
538
383
|
input_tokens: z.ZodNumber;
|
|
539
384
|
output_tokens: z.ZodNumber;
|
|
540
385
|
cost_usd: z.ZodNumber;
|
|
541
|
-
},
|
|
542
|
-
cost_usd: number;
|
|
543
|
-
input_tokens: number;
|
|
544
|
-
output_tokens: number;
|
|
545
|
-
}, {
|
|
546
|
-
cost_usd: number;
|
|
547
|
-
input_tokens: number;
|
|
548
|
-
output_tokens: number;
|
|
549
|
-
}>>;
|
|
386
|
+
}, z.core.$strip>>;
|
|
550
387
|
latency_ms: z.ZodOptional<z.ZodNumber>;
|
|
551
388
|
error: z.ZodOptional<z.ZodObject<{
|
|
552
389
|
code: z.ZodString;
|
|
553
390
|
message: z.ZodString;
|
|
554
|
-
},
|
|
555
|
-
code: string;
|
|
556
|
-
message: string;
|
|
557
|
-
}, {
|
|
558
|
-
code: string;
|
|
559
|
-
message: string;
|
|
560
|
-
}>>;
|
|
391
|
+
}, z.core.$strip>>;
|
|
561
392
|
assertions: z.ZodArray<z.ZodObject<{
|
|
562
393
|
type: z.ZodString;
|
|
563
394
|
passed: z.ZodBoolean;
|
|
@@ -566,230 +397,12 @@ declare const runResultSchema: z.ZodObject<{
|
|
|
566
397
|
binaryVerdicts: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
567
398
|
question: z.ZodString;
|
|
568
399
|
passed: z.ZodBoolean;
|
|
569
|
-
},
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
}, {
|
|
573
|
-
question: string;
|
|
574
|
-
passed: boolean;
|
|
575
|
-
}>, "many">>;
|
|
576
|
-
}, "strip", z.ZodTypeAny, {
|
|
577
|
-
type: string;
|
|
578
|
-
passed: boolean;
|
|
579
|
-
score?: number | undefined;
|
|
580
|
-
reason?: string | undefined;
|
|
581
|
-
binaryVerdicts?: {
|
|
582
|
-
question: string;
|
|
583
|
-
passed: boolean;
|
|
584
|
-
}[] | undefined;
|
|
585
|
-
}, {
|
|
586
|
-
type: string;
|
|
587
|
-
passed: boolean;
|
|
588
|
-
score?: number | undefined;
|
|
589
|
-
reason?: string | undefined;
|
|
590
|
-
binaryVerdicts?: {
|
|
591
|
-
question: string;
|
|
592
|
-
passed: boolean;
|
|
593
|
-
}[] | undefined;
|
|
594
|
-
}>, "many">;
|
|
595
|
-
}, "strip", z.ZodTypeAny, {
|
|
596
|
-
vars: Record<string, unknown>;
|
|
597
|
-
assertions: {
|
|
598
|
-
type: string;
|
|
599
|
-
passed: boolean;
|
|
600
|
-
score?: number | undefined;
|
|
601
|
-
reason?: string | undefined;
|
|
602
|
-
binaryVerdicts?: {
|
|
603
|
-
question: string;
|
|
604
|
-
passed: boolean;
|
|
605
|
-
}[] | undefined;
|
|
606
|
-
}[];
|
|
607
|
-
output: string | null;
|
|
608
|
-
cached: boolean;
|
|
609
|
-
error?: {
|
|
610
|
-
code: string;
|
|
611
|
-
message: string;
|
|
612
|
-
} | undefined;
|
|
613
|
-
usage?: {
|
|
614
|
-
cost_usd: number;
|
|
615
|
-
input_tokens: number;
|
|
616
|
-
output_tokens: number;
|
|
617
|
-
} | undefined;
|
|
618
|
-
latency_ms?: number | undefined;
|
|
619
|
-
}, {
|
|
620
|
-
vars: Record<string, unknown>;
|
|
621
|
-
assertions: {
|
|
622
|
-
type: string;
|
|
623
|
-
passed: boolean;
|
|
624
|
-
score?: number | undefined;
|
|
625
|
-
reason?: string | undefined;
|
|
626
|
-
binaryVerdicts?: {
|
|
627
|
-
question: string;
|
|
628
|
-
passed: boolean;
|
|
629
|
-
}[] | undefined;
|
|
630
|
-
}[];
|
|
631
|
-
output: string | null;
|
|
632
|
-
cached: boolean;
|
|
633
|
-
error?: {
|
|
634
|
-
code: string;
|
|
635
|
-
message: string;
|
|
636
|
-
} | undefined;
|
|
637
|
-
usage?: {
|
|
638
|
-
cost_usd: number;
|
|
639
|
-
input_tokens: number;
|
|
640
|
-
output_tokens: number;
|
|
641
|
-
} | undefined;
|
|
642
|
-
latency_ms?: number | undefined;
|
|
643
|
-
}>, "many">;
|
|
400
|
+
}, z.core.$strip>>>;
|
|
401
|
+
}, z.core.$strip>>;
|
|
402
|
+
}, z.core.$strip>>;
|
|
644
403
|
passed: z.ZodBoolean;
|
|
645
|
-
},
|
|
646
|
-
|
|
647
|
-
tags: string[];
|
|
648
|
-
passed: boolean;
|
|
649
|
-
rows: {
|
|
650
|
-
vars: Record<string, unknown>;
|
|
651
|
-
assertions: {
|
|
652
|
-
type: string;
|
|
653
|
-
passed: boolean;
|
|
654
|
-
score?: number | undefined;
|
|
655
|
-
reason?: string | undefined;
|
|
656
|
-
binaryVerdicts?: {
|
|
657
|
-
question: string;
|
|
658
|
-
passed: boolean;
|
|
659
|
-
}[] | undefined;
|
|
660
|
-
}[];
|
|
661
|
-
output: string | null;
|
|
662
|
-
cached: boolean;
|
|
663
|
-
error?: {
|
|
664
|
-
code: string;
|
|
665
|
-
message: string;
|
|
666
|
-
} | undefined;
|
|
667
|
-
usage?: {
|
|
668
|
-
cost_usd: number;
|
|
669
|
-
input_tokens: number;
|
|
670
|
-
output_tokens: number;
|
|
671
|
-
} | undefined;
|
|
672
|
-
latency_ms?: number | undefined;
|
|
673
|
-
}[];
|
|
674
|
-
}, {
|
|
675
|
-
id: string;
|
|
676
|
-
tags: string[];
|
|
677
|
-
passed: boolean;
|
|
678
|
-
rows: {
|
|
679
|
-
vars: Record<string, unknown>;
|
|
680
|
-
assertions: {
|
|
681
|
-
type: string;
|
|
682
|
-
passed: boolean;
|
|
683
|
-
score?: number | undefined;
|
|
684
|
-
reason?: string | undefined;
|
|
685
|
-
binaryVerdicts?: {
|
|
686
|
-
question: string;
|
|
687
|
-
passed: boolean;
|
|
688
|
-
}[] | undefined;
|
|
689
|
-
}[];
|
|
690
|
-
output: string | null;
|
|
691
|
-
cached: boolean;
|
|
692
|
-
error?: {
|
|
693
|
-
code: string;
|
|
694
|
-
message: string;
|
|
695
|
-
} | undefined;
|
|
696
|
-
usage?: {
|
|
697
|
-
cost_usd: number;
|
|
698
|
-
input_tokens: number;
|
|
699
|
-
output_tokens: number;
|
|
700
|
-
} | undefined;
|
|
701
|
-
latency_ms?: number | undefined;
|
|
702
|
-
}[];
|
|
703
|
-
}>, "many">;
|
|
704
|
-
}, "strip", z.ZodTypeAny, {
|
|
705
|
-
evals: {
|
|
706
|
-
id: string;
|
|
707
|
-
tags: string[];
|
|
708
|
-
passed: boolean;
|
|
709
|
-
rows: {
|
|
710
|
-
vars: Record<string, unknown>;
|
|
711
|
-
assertions: {
|
|
712
|
-
type: string;
|
|
713
|
-
passed: boolean;
|
|
714
|
-
score?: number | undefined;
|
|
715
|
-
reason?: string | undefined;
|
|
716
|
-
binaryVerdicts?: {
|
|
717
|
-
question: string;
|
|
718
|
-
passed: boolean;
|
|
719
|
-
}[] | undefined;
|
|
720
|
-
}[];
|
|
721
|
-
output: string | null;
|
|
722
|
-
cached: boolean;
|
|
723
|
-
error?: {
|
|
724
|
-
code: string;
|
|
725
|
-
message: string;
|
|
726
|
-
} | undefined;
|
|
727
|
-
usage?: {
|
|
728
|
-
cost_usd: number;
|
|
729
|
-
input_tokens: number;
|
|
730
|
-
output_tokens: number;
|
|
731
|
-
} | undefined;
|
|
732
|
-
latency_ms?: number | undefined;
|
|
733
|
-
}[];
|
|
734
|
-
}[];
|
|
735
|
-
schemaVersion: 1;
|
|
736
|
-
plune_version: string;
|
|
737
|
-
summary: {
|
|
738
|
-
cost_usd: number;
|
|
739
|
-
passed: number;
|
|
740
|
-
failed: number;
|
|
741
|
-
errored: number;
|
|
742
|
-
total: number;
|
|
743
|
-
duration_ms: number;
|
|
744
|
-
};
|
|
745
|
-
started_at: string;
|
|
746
|
-
finished_at: string;
|
|
747
|
-
config_hash: string;
|
|
748
|
-
}, {
|
|
749
|
-
evals: {
|
|
750
|
-
id: string;
|
|
751
|
-
tags: string[];
|
|
752
|
-
passed: boolean;
|
|
753
|
-
rows: {
|
|
754
|
-
vars: Record<string, unknown>;
|
|
755
|
-
assertions: {
|
|
756
|
-
type: string;
|
|
757
|
-
passed: boolean;
|
|
758
|
-
score?: number | undefined;
|
|
759
|
-
reason?: string | undefined;
|
|
760
|
-
binaryVerdicts?: {
|
|
761
|
-
question: string;
|
|
762
|
-
passed: boolean;
|
|
763
|
-
}[] | undefined;
|
|
764
|
-
}[];
|
|
765
|
-
output: string | null;
|
|
766
|
-
cached: boolean;
|
|
767
|
-
error?: {
|
|
768
|
-
code: string;
|
|
769
|
-
message: string;
|
|
770
|
-
} | undefined;
|
|
771
|
-
usage?: {
|
|
772
|
-
cost_usd: number;
|
|
773
|
-
input_tokens: number;
|
|
774
|
-
output_tokens: number;
|
|
775
|
-
} | undefined;
|
|
776
|
-
latency_ms?: number | undefined;
|
|
777
|
-
}[];
|
|
778
|
-
}[];
|
|
779
|
-
schemaVersion: 1;
|
|
780
|
-
plune_version: string;
|
|
781
|
-
summary: {
|
|
782
|
-
cost_usd: number;
|
|
783
|
-
passed: number;
|
|
784
|
-
failed: number;
|
|
785
|
-
errored: number;
|
|
786
|
-
total: number;
|
|
787
|
-
duration_ms: number;
|
|
788
|
-
};
|
|
789
|
-
started_at: string;
|
|
790
|
-
finished_at: string;
|
|
791
|
-
config_hash: string;
|
|
792
|
-
}>;
|
|
404
|
+
}, z.core.$strip>>;
|
|
405
|
+
}, z.core.$strip>;
|
|
793
406
|
/**
|
|
794
407
|
* Type inferred from the zod mirror — structurally a superset of the frozen `RunResult`
|
|
795
408
|
* (adds optional `binaryVerdicts`). The canonical `RunResult` remains the source of truth.
|
|
@@ -800,8 +413,10 @@ type BinaryVerdict = z.infer<typeof binaryVerdictSchema>;
|
|
|
800
413
|
declare function parseRunResult(value: unknown): ParsedRunResult;
|
|
801
414
|
|
|
802
415
|
interface IngestDeps {
|
|
803
|
-
/** The Cairn run directory to read. */
|
|
804
|
-
dir
|
|
416
|
+
/** The Cairn run directory to read. Omit it and the newest run under `runsDir` is used. */
|
|
417
|
+
dir?: string;
|
|
418
|
+
/** Where to look when `dir` is omitted; relative paths resolve against the process's cwd. */
|
|
419
|
+
runsDir?: string;
|
|
805
420
|
apiUrl?: string;
|
|
806
421
|
loadToken?: () => string | null;
|
|
807
422
|
fetchImpl?: typeof fetch;
|