@plune-ai/cli 0.6.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -1
- package/dist/cli.cjs +915 -162
- package/dist/index.cjs +46 -19
- package/dist/index.d.cts +39 -424
- package/dist/index.d.ts +39 -424
- package/dist/index.js +46 -19
- package/dist/reporter-core.cjs +467 -0
- package/dist/reporter-core.d.cts +339 -0
- package/dist/reporter-core.d.ts +339 -0
- package/dist/reporter-core.js +423 -0
- package/package.json +12 -3
package/dist/index.d.ts
CHANGED
|
@@ -265,176 +265,65 @@ declare function handleSync(deps?: SyncDeps): Promise<SyncResult>;
|
|
|
265
265
|
*/
|
|
266
266
|
declare function reportSyncFailure(err: unknown, write: (s: string) => void): number | null;
|
|
267
267
|
|
|
268
|
-
declare const assertionConfigSchema: z.ZodDiscriminatedUnion<
|
|
268
|
+
declare const assertionConfigSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
269
269
|
type: z.ZodLiteral<"exact-match">;
|
|
270
270
|
value: z.ZodString;
|
|
271
271
|
trim: z.ZodOptional<z.ZodBoolean>;
|
|
272
272
|
ignore_case: z.ZodOptional<z.ZodBoolean>;
|
|
273
|
-
},
|
|
274
|
-
type: "exact-match";
|
|
275
|
-
value: string;
|
|
276
|
-
trim?: boolean | undefined;
|
|
277
|
-
ignore_case?: boolean | undefined;
|
|
278
|
-
}, {
|
|
279
|
-
type: "exact-match";
|
|
280
|
-
value: string;
|
|
281
|
-
trim?: boolean | undefined;
|
|
282
|
-
ignore_case?: boolean | undefined;
|
|
283
|
-
}>, z.ZodObject<{
|
|
273
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
284
274
|
type: z.ZodLiteral<"contains">;
|
|
285
275
|
value: z.ZodString;
|
|
286
276
|
ignore_case: z.ZodOptional<z.ZodBoolean>;
|
|
287
|
-
},
|
|
288
|
-
type: "contains";
|
|
289
|
-
value: string;
|
|
290
|
-
ignore_case?: boolean | undefined;
|
|
291
|
-
}, {
|
|
292
|
-
type: "contains";
|
|
293
|
-
value: string;
|
|
294
|
-
ignore_case?: boolean | undefined;
|
|
295
|
-
}>, z.ZodObject<{
|
|
277
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
296
278
|
type: z.ZodLiteral<"contains-any">;
|
|
297
|
-
values: z.ZodArray<z.ZodString
|
|
279
|
+
values: z.ZodArray<z.ZodString>;
|
|
298
280
|
ignore_case: z.ZodOptional<z.ZodBoolean>;
|
|
299
|
-
},
|
|
300
|
-
type: "contains-any";
|
|
301
|
-
values: string[];
|
|
302
|
-
ignore_case?: boolean | undefined;
|
|
303
|
-
}, {
|
|
304
|
-
type: "contains-any";
|
|
305
|
-
values: string[];
|
|
306
|
-
ignore_case?: boolean | undefined;
|
|
307
|
-
}>, z.ZodObject<{
|
|
281
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
308
282
|
type: z.ZodLiteral<"contains-all">;
|
|
309
|
-
values: z.ZodArray<z.ZodString
|
|
283
|
+
values: z.ZodArray<z.ZodString>;
|
|
310
284
|
ignore_case: z.ZodOptional<z.ZodBoolean>;
|
|
311
|
-
},
|
|
312
|
-
type: "contains-all";
|
|
313
|
-
values: string[];
|
|
314
|
-
ignore_case?: boolean | undefined;
|
|
315
|
-
}, {
|
|
316
|
-
type: "contains-all";
|
|
317
|
-
values: string[];
|
|
318
|
-
ignore_case?: boolean | undefined;
|
|
319
|
-
}>, z.ZodObject<{
|
|
285
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
320
286
|
type: z.ZodLiteral<"json-schema">;
|
|
321
287
|
schema: z.ZodRecord<z.ZodString, z.ZodUnknown>;
|
|
322
|
-
extract: z.ZodOptional<z.ZodEnum<
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
}, {
|
|
328
|
-
type: "json-schema";
|
|
329
|
-
schema: Record<string, unknown>;
|
|
330
|
-
extract?: "auto" | "strict" | undefined;
|
|
331
|
-
}>, z.ZodObject<{
|
|
288
|
+
extract: z.ZodOptional<z.ZodEnum<{
|
|
289
|
+
auto: "auto";
|
|
290
|
+
strict: "strict";
|
|
291
|
+
}>>;
|
|
292
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
332
293
|
type: z.ZodLiteral<"llm-judge">;
|
|
333
294
|
criteria: z.ZodString;
|
|
334
295
|
provider: z.ZodOptional<z.ZodObject<{
|
|
335
|
-
type: z.ZodOptional<z.ZodEnum<
|
|
296
|
+
type: z.ZodOptional<z.ZodEnum<{
|
|
297
|
+
anthropic: "anthropic";
|
|
298
|
+
openai: "openai";
|
|
299
|
+
openrouter: "openrouter";
|
|
300
|
+
}>>;
|
|
336
301
|
model: z.ZodOptional<z.ZodString>;
|
|
337
302
|
temperature: z.ZodOptional<z.ZodOptional<z.ZodNumber>>;
|
|
338
303
|
max_tokens: z.ZodOptional<z.ZodOptional<z.ZodNumber>>;
|
|
339
304
|
concurrency: z.ZodOptional<z.ZodOptional<z.ZodNumber>>;
|
|
340
305
|
timeout: z.ZodOptional<z.ZodOptional<z.ZodNumber>>;
|
|
341
306
|
max_retries: z.ZodOptional<z.ZodOptional<z.ZodNumber>>;
|
|
342
|
-
},
|
|
343
|
-
type?: "anthropic" | "openai" | "openrouter" | undefined;
|
|
344
|
-
model?: string | undefined;
|
|
345
|
-
temperature?: number | undefined;
|
|
346
|
-
max_tokens?: number | undefined;
|
|
347
|
-
concurrency?: number | undefined;
|
|
348
|
-
timeout?: number | undefined;
|
|
349
|
-
max_retries?: number | undefined;
|
|
350
|
-
}, {
|
|
351
|
-
type?: "anthropic" | "openai" | "openrouter" | undefined;
|
|
352
|
-
model?: string | undefined;
|
|
353
|
-
temperature?: number | undefined;
|
|
354
|
-
max_tokens?: number | undefined;
|
|
355
|
-
concurrency?: number | undefined;
|
|
356
|
-
timeout?: number | undefined;
|
|
357
|
-
max_retries?: number | undefined;
|
|
358
|
-
}>>;
|
|
307
|
+
}, z.core.$strip>>;
|
|
359
308
|
pass_threshold: z.ZodOptional<z.ZodNumber>;
|
|
360
|
-
},
|
|
361
|
-
type: "llm-judge";
|
|
362
|
-
criteria: string;
|
|
363
|
-
provider?: {
|
|
364
|
-
type?: "anthropic" | "openai" | "openrouter" | undefined;
|
|
365
|
-
model?: string | undefined;
|
|
366
|
-
temperature?: number | undefined;
|
|
367
|
-
max_tokens?: number | undefined;
|
|
368
|
-
concurrency?: number | undefined;
|
|
369
|
-
timeout?: number | undefined;
|
|
370
|
-
max_retries?: number | undefined;
|
|
371
|
-
} | undefined;
|
|
372
|
-
pass_threshold?: number | undefined;
|
|
373
|
-
}, {
|
|
374
|
-
type: "llm-judge";
|
|
375
|
-
criteria: string;
|
|
376
|
-
provider?: {
|
|
377
|
-
type?: "anthropic" | "openai" | "openrouter" | undefined;
|
|
378
|
-
model?: string | undefined;
|
|
379
|
-
temperature?: number | undefined;
|
|
380
|
-
max_tokens?: number | undefined;
|
|
381
|
-
concurrency?: number | undefined;
|
|
382
|
-
timeout?: number | undefined;
|
|
383
|
-
max_retries?: number | undefined;
|
|
384
|
-
} | undefined;
|
|
385
|
-
pass_threshold?: number | undefined;
|
|
386
|
-
}>, z.ZodObject<{
|
|
309
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
387
310
|
type: z.ZodLiteral<"semantic-similarity">;
|
|
388
311
|
reference: z.ZodString;
|
|
389
312
|
threshold: z.ZodOptional<z.ZodNumber>;
|
|
390
|
-
},
|
|
391
|
-
type: "semantic-similarity";
|
|
392
|
-
reference: string;
|
|
393
|
-
threshold?: number | undefined;
|
|
394
|
-
}, {
|
|
395
|
-
type: "semantic-similarity";
|
|
396
|
-
reference: string;
|
|
397
|
-
threshold?: number | undefined;
|
|
398
|
-
}>, z.ZodObject<{
|
|
313
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
399
314
|
type: z.ZodLiteral<"faithfulness">;
|
|
400
315
|
context: z.ZodString;
|
|
401
316
|
threshold: z.ZodOptional<z.ZodNumber>;
|
|
402
|
-
},
|
|
403
|
-
type: "faithfulness";
|
|
404
|
-
context: string;
|
|
405
|
-
threshold?: number | undefined;
|
|
406
|
-
}, {
|
|
407
|
-
type: "faithfulness";
|
|
408
|
-
context: string;
|
|
409
|
-
threshold?: number | undefined;
|
|
410
|
-
}>, z.ZodObject<{
|
|
317
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
411
318
|
type: z.ZodLiteral<"answer-relevance">;
|
|
412
319
|
question: z.ZodString;
|
|
413
320
|
threshold: z.ZodOptional<z.ZodNumber>;
|
|
414
|
-
},
|
|
415
|
-
type: "answer-relevance";
|
|
416
|
-
question: string;
|
|
417
|
-
threshold?: number | undefined;
|
|
418
|
-
}, {
|
|
419
|
-
type: "answer-relevance";
|
|
420
|
-
question: string;
|
|
421
|
-
threshold?: number | undefined;
|
|
422
|
-
}>, z.ZodObject<{
|
|
321
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
423
322
|
type: z.ZodLiteral<"context-precision">;
|
|
424
323
|
context: z.ZodString;
|
|
425
324
|
question: z.ZodString;
|
|
426
325
|
threshold: z.ZodOptional<z.ZodNumber>;
|
|
427
|
-
},
|
|
428
|
-
type: "context-precision";
|
|
429
|
-
context: string;
|
|
430
|
-
question: string;
|
|
431
|
-
threshold?: number | undefined;
|
|
432
|
-
}, {
|
|
433
|
-
type: "context-precision";
|
|
434
|
-
context: string;
|
|
435
|
-
question: string;
|
|
436
|
-
threshold?: number | undefined;
|
|
437
|
-
}>]>;
|
|
326
|
+
}, z.core.$strip>], "type">;
|
|
438
327
|
|
|
439
328
|
/**
|
|
440
329
|
* Runtime validation for the RunResult contract — the JSON that `plune run --format json` emits.
|
|
@@ -458,13 +347,7 @@ declare const RUN_RESULT_SCHEMA_VERSION: RunResult['schemaVersion'];
|
|
|
458
347
|
declare const binaryVerdictSchema: z.ZodObject<{
|
|
459
348
|
question: z.ZodString;
|
|
460
349
|
passed: z.ZodBoolean;
|
|
461
|
-
},
|
|
462
|
-
question: string;
|
|
463
|
-
passed: boolean;
|
|
464
|
-
}, {
|
|
465
|
-
question: string;
|
|
466
|
-
passed: boolean;
|
|
467
|
-
}>;
|
|
350
|
+
}, z.core.$strip>;
|
|
468
351
|
declare const assertionResultRecordSchema: z.ZodObject<{
|
|
469
352
|
type: z.ZodString;
|
|
470
353
|
passed: z.ZodBoolean;
|
|
@@ -473,32 +356,8 @@ declare const assertionResultRecordSchema: z.ZodObject<{
|
|
|
473
356
|
binaryVerdicts: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
474
357
|
question: z.ZodString;
|
|
475
358
|
passed: z.ZodBoolean;
|
|
476
|
-
},
|
|
477
|
-
|
|
478
|
-
passed: boolean;
|
|
479
|
-
}, {
|
|
480
|
-
question: string;
|
|
481
|
-
passed: boolean;
|
|
482
|
-
}>, "many">>;
|
|
483
|
-
}, "strip", z.ZodTypeAny, {
|
|
484
|
-
type: string;
|
|
485
|
-
passed: boolean;
|
|
486
|
-
score?: number | undefined;
|
|
487
|
-
reason?: string | undefined;
|
|
488
|
-
binaryVerdicts?: {
|
|
489
|
-
question: string;
|
|
490
|
-
passed: boolean;
|
|
491
|
-
}[] | undefined;
|
|
492
|
-
}, {
|
|
493
|
-
type: string;
|
|
494
|
-
passed: boolean;
|
|
495
|
-
score?: number | undefined;
|
|
496
|
-
reason?: string | undefined;
|
|
497
|
-
binaryVerdicts?: {
|
|
498
|
-
question: string;
|
|
499
|
-
passed: boolean;
|
|
500
|
-
}[] | undefined;
|
|
501
|
-
}>;
|
|
359
|
+
}, z.core.$strip>>>;
|
|
360
|
+
}, z.core.$strip>;
|
|
502
361
|
declare const runResultSchema: z.ZodObject<{
|
|
503
362
|
schemaVersion: z.ZodLiteral<1>;
|
|
504
363
|
plune_version: z.ZodString;
|
|
@@ -512,24 +371,10 @@ declare const runResultSchema: z.ZodObject<{
|
|
|
512
371
|
errored: z.ZodNumber;
|
|
513
372
|
cost_usd: z.ZodNumber;
|
|
514
373
|
duration_ms: z.ZodNumber;
|
|
515
|
-
},
|
|
516
|
-
cost_usd: number;
|
|
517
|
-
passed: number;
|
|
518
|
-
failed: number;
|
|
519
|
-
errored: number;
|
|
520
|
-
total: number;
|
|
521
|
-
duration_ms: number;
|
|
522
|
-
}, {
|
|
523
|
-
cost_usd: number;
|
|
524
|
-
passed: number;
|
|
525
|
-
failed: number;
|
|
526
|
-
errored: number;
|
|
527
|
-
total: number;
|
|
528
|
-
duration_ms: number;
|
|
529
|
-
}>;
|
|
374
|
+
}, z.core.$strip>;
|
|
530
375
|
evals: z.ZodArray<z.ZodObject<{
|
|
531
376
|
id: z.ZodString;
|
|
532
|
-
tags: z.ZodArray<z.ZodString
|
|
377
|
+
tags: z.ZodArray<z.ZodString>;
|
|
533
378
|
rows: z.ZodArray<z.ZodObject<{
|
|
534
379
|
vars: z.ZodRecord<z.ZodString, z.ZodUnknown>;
|
|
535
380
|
output: z.ZodNullable<z.ZodString>;
|
|
@@ -538,26 +383,12 @@ declare const runResultSchema: z.ZodObject<{
|
|
|
538
383
|
input_tokens: z.ZodNumber;
|
|
539
384
|
output_tokens: z.ZodNumber;
|
|
540
385
|
cost_usd: z.ZodNumber;
|
|
541
|
-
},
|
|
542
|
-
cost_usd: number;
|
|
543
|
-
input_tokens: number;
|
|
544
|
-
output_tokens: number;
|
|
545
|
-
}, {
|
|
546
|
-
cost_usd: number;
|
|
547
|
-
input_tokens: number;
|
|
548
|
-
output_tokens: number;
|
|
549
|
-
}>>;
|
|
386
|
+
}, z.core.$strip>>;
|
|
550
387
|
latency_ms: z.ZodOptional<z.ZodNumber>;
|
|
551
388
|
error: z.ZodOptional<z.ZodObject<{
|
|
552
389
|
code: z.ZodString;
|
|
553
390
|
message: z.ZodString;
|
|
554
|
-
},
|
|
555
|
-
code: string;
|
|
556
|
-
message: string;
|
|
557
|
-
}, {
|
|
558
|
-
code: string;
|
|
559
|
-
message: string;
|
|
560
|
-
}>>;
|
|
391
|
+
}, z.core.$strip>>;
|
|
561
392
|
assertions: z.ZodArray<z.ZodObject<{
|
|
562
393
|
type: z.ZodString;
|
|
563
394
|
passed: z.ZodBoolean;
|
|
@@ -566,230 +397,12 @@ declare const runResultSchema: z.ZodObject<{
|
|
|
566
397
|
binaryVerdicts: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
567
398
|
question: z.ZodString;
|
|
568
399
|
passed: z.ZodBoolean;
|
|
569
|
-
},
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
}, {
|
|
573
|
-
question: string;
|
|
574
|
-
passed: boolean;
|
|
575
|
-
}>, "many">>;
|
|
576
|
-
}, "strip", z.ZodTypeAny, {
|
|
577
|
-
type: string;
|
|
578
|
-
passed: boolean;
|
|
579
|
-
score?: number | undefined;
|
|
580
|
-
reason?: string | undefined;
|
|
581
|
-
binaryVerdicts?: {
|
|
582
|
-
question: string;
|
|
583
|
-
passed: boolean;
|
|
584
|
-
}[] | undefined;
|
|
585
|
-
}, {
|
|
586
|
-
type: string;
|
|
587
|
-
passed: boolean;
|
|
588
|
-
score?: number | undefined;
|
|
589
|
-
reason?: string | undefined;
|
|
590
|
-
binaryVerdicts?: {
|
|
591
|
-
question: string;
|
|
592
|
-
passed: boolean;
|
|
593
|
-
}[] | undefined;
|
|
594
|
-
}>, "many">;
|
|
595
|
-
}, "strip", z.ZodTypeAny, {
|
|
596
|
-
vars: Record<string, unknown>;
|
|
597
|
-
assertions: {
|
|
598
|
-
type: string;
|
|
599
|
-
passed: boolean;
|
|
600
|
-
score?: number | undefined;
|
|
601
|
-
reason?: string | undefined;
|
|
602
|
-
binaryVerdicts?: {
|
|
603
|
-
question: string;
|
|
604
|
-
passed: boolean;
|
|
605
|
-
}[] | undefined;
|
|
606
|
-
}[];
|
|
607
|
-
output: string | null;
|
|
608
|
-
cached: boolean;
|
|
609
|
-
error?: {
|
|
610
|
-
code: string;
|
|
611
|
-
message: string;
|
|
612
|
-
} | undefined;
|
|
613
|
-
usage?: {
|
|
614
|
-
cost_usd: number;
|
|
615
|
-
input_tokens: number;
|
|
616
|
-
output_tokens: number;
|
|
617
|
-
} | undefined;
|
|
618
|
-
latency_ms?: number | undefined;
|
|
619
|
-
}, {
|
|
620
|
-
vars: Record<string, unknown>;
|
|
621
|
-
assertions: {
|
|
622
|
-
type: string;
|
|
623
|
-
passed: boolean;
|
|
624
|
-
score?: number | undefined;
|
|
625
|
-
reason?: string | undefined;
|
|
626
|
-
binaryVerdicts?: {
|
|
627
|
-
question: string;
|
|
628
|
-
passed: boolean;
|
|
629
|
-
}[] | undefined;
|
|
630
|
-
}[];
|
|
631
|
-
output: string | null;
|
|
632
|
-
cached: boolean;
|
|
633
|
-
error?: {
|
|
634
|
-
code: string;
|
|
635
|
-
message: string;
|
|
636
|
-
} | undefined;
|
|
637
|
-
usage?: {
|
|
638
|
-
cost_usd: number;
|
|
639
|
-
input_tokens: number;
|
|
640
|
-
output_tokens: number;
|
|
641
|
-
} | undefined;
|
|
642
|
-
latency_ms?: number | undefined;
|
|
643
|
-
}>, "many">;
|
|
400
|
+
}, z.core.$strip>>>;
|
|
401
|
+
}, z.core.$strip>>;
|
|
402
|
+
}, z.core.$strip>>;
|
|
644
403
|
passed: z.ZodBoolean;
|
|
645
|
-
},
|
|
646
|
-
|
|
647
|
-
tags: string[];
|
|
648
|
-
passed: boolean;
|
|
649
|
-
rows: {
|
|
650
|
-
vars: Record<string, unknown>;
|
|
651
|
-
assertions: {
|
|
652
|
-
type: string;
|
|
653
|
-
passed: boolean;
|
|
654
|
-
score?: number | undefined;
|
|
655
|
-
reason?: string | undefined;
|
|
656
|
-
binaryVerdicts?: {
|
|
657
|
-
question: string;
|
|
658
|
-
passed: boolean;
|
|
659
|
-
}[] | undefined;
|
|
660
|
-
}[];
|
|
661
|
-
output: string | null;
|
|
662
|
-
cached: boolean;
|
|
663
|
-
error?: {
|
|
664
|
-
code: string;
|
|
665
|
-
message: string;
|
|
666
|
-
} | undefined;
|
|
667
|
-
usage?: {
|
|
668
|
-
cost_usd: number;
|
|
669
|
-
input_tokens: number;
|
|
670
|
-
output_tokens: number;
|
|
671
|
-
} | undefined;
|
|
672
|
-
latency_ms?: number | undefined;
|
|
673
|
-
}[];
|
|
674
|
-
}, {
|
|
675
|
-
id: string;
|
|
676
|
-
tags: string[];
|
|
677
|
-
passed: boolean;
|
|
678
|
-
rows: {
|
|
679
|
-
vars: Record<string, unknown>;
|
|
680
|
-
assertions: {
|
|
681
|
-
type: string;
|
|
682
|
-
passed: boolean;
|
|
683
|
-
score?: number | undefined;
|
|
684
|
-
reason?: string | undefined;
|
|
685
|
-
binaryVerdicts?: {
|
|
686
|
-
question: string;
|
|
687
|
-
passed: boolean;
|
|
688
|
-
}[] | undefined;
|
|
689
|
-
}[];
|
|
690
|
-
output: string | null;
|
|
691
|
-
cached: boolean;
|
|
692
|
-
error?: {
|
|
693
|
-
code: string;
|
|
694
|
-
message: string;
|
|
695
|
-
} | undefined;
|
|
696
|
-
usage?: {
|
|
697
|
-
cost_usd: number;
|
|
698
|
-
input_tokens: number;
|
|
699
|
-
output_tokens: number;
|
|
700
|
-
} | undefined;
|
|
701
|
-
latency_ms?: number | undefined;
|
|
702
|
-
}[];
|
|
703
|
-
}>, "many">;
|
|
704
|
-
}, "strip", z.ZodTypeAny, {
|
|
705
|
-
evals: {
|
|
706
|
-
id: string;
|
|
707
|
-
tags: string[];
|
|
708
|
-
passed: boolean;
|
|
709
|
-
rows: {
|
|
710
|
-
vars: Record<string, unknown>;
|
|
711
|
-
assertions: {
|
|
712
|
-
type: string;
|
|
713
|
-
passed: boolean;
|
|
714
|
-
score?: number | undefined;
|
|
715
|
-
reason?: string | undefined;
|
|
716
|
-
binaryVerdicts?: {
|
|
717
|
-
question: string;
|
|
718
|
-
passed: boolean;
|
|
719
|
-
}[] | undefined;
|
|
720
|
-
}[];
|
|
721
|
-
output: string | null;
|
|
722
|
-
cached: boolean;
|
|
723
|
-
error?: {
|
|
724
|
-
code: string;
|
|
725
|
-
message: string;
|
|
726
|
-
} | undefined;
|
|
727
|
-
usage?: {
|
|
728
|
-
cost_usd: number;
|
|
729
|
-
input_tokens: number;
|
|
730
|
-
output_tokens: number;
|
|
731
|
-
} | undefined;
|
|
732
|
-
latency_ms?: number | undefined;
|
|
733
|
-
}[];
|
|
734
|
-
}[];
|
|
735
|
-
schemaVersion: 1;
|
|
736
|
-
plune_version: string;
|
|
737
|
-
summary: {
|
|
738
|
-
cost_usd: number;
|
|
739
|
-
passed: number;
|
|
740
|
-
failed: number;
|
|
741
|
-
errored: number;
|
|
742
|
-
total: number;
|
|
743
|
-
duration_ms: number;
|
|
744
|
-
};
|
|
745
|
-
started_at: string;
|
|
746
|
-
finished_at: string;
|
|
747
|
-
config_hash: string;
|
|
748
|
-
}, {
|
|
749
|
-
evals: {
|
|
750
|
-
id: string;
|
|
751
|
-
tags: string[];
|
|
752
|
-
passed: boolean;
|
|
753
|
-
rows: {
|
|
754
|
-
vars: Record<string, unknown>;
|
|
755
|
-
assertions: {
|
|
756
|
-
type: string;
|
|
757
|
-
passed: boolean;
|
|
758
|
-
score?: number | undefined;
|
|
759
|
-
reason?: string | undefined;
|
|
760
|
-
binaryVerdicts?: {
|
|
761
|
-
question: string;
|
|
762
|
-
passed: boolean;
|
|
763
|
-
}[] | undefined;
|
|
764
|
-
}[];
|
|
765
|
-
output: string | null;
|
|
766
|
-
cached: boolean;
|
|
767
|
-
error?: {
|
|
768
|
-
code: string;
|
|
769
|
-
message: string;
|
|
770
|
-
} | undefined;
|
|
771
|
-
usage?: {
|
|
772
|
-
cost_usd: number;
|
|
773
|
-
input_tokens: number;
|
|
774
|
-
output_tokens: number;
|
|
775
|
-
} | undefined;
|
|
776
|
-
latency_ms?: number | undefined;
|
|
777
|
-
}[];
|
|
778
|
-
}[];
|
|
779
|
-
schemaVersion: 1;
|
|
780
|
-
plune_version: string;
|
|
781
|
-
summary: {
|
|
782
|
-
cost_usd: number;
|
|
783
|
-
passed: number;
|
|
784
|
-
failed: number;
|
|
785
|
-
errored: number;
|
|
786
|
-
total: number;
|
|
787
|
-
duration_ms: number;
|
|
788
|
-
};
|
|
789
|
-
started_at: string;
|
|
790
|
-
finished_at: string;
|
|
791
|
-
config_hash: string;
|
|
792
|
-
}>;
|
|
404
|
+
}, z.core.$strip>>;
|
|
405
|
+
}, z.core.$strip>;
|
|
793
406
|
/**
|
|
794
407
|
* Type inferred from the zod mirror — structurally a superset of the frozen `RunResult`
|
|
795
408
|
* (adds optional `binaryVerdicts`). The canonical `RunResult` remains the source of truth.
|
|
@@ -800,8 +413,10 @@ type BinaryVerdict = z.infer<typeof binaryVerdictSchema>;
|
|
|
800
413
|
declare function parseRunResult(value: unknown): ParsedRunResult;
|
|
801
414
|
|
|
802
415
|
interface IngestDeps {
|
|
803
|
-
/** The Cairn run directory to read. */
|
|
804
|
-
dir
|
|
416
|
+
/** The Cairn run directory to read. Omit it and the newest run under `runsDir` is used. */
|
|
417
|
+
dir?: string;
|
|
418
|
+
/** Where to look when `dir` is omitted; relative paths resolve against the process's cwd. */
|
|
419
|
+
runsDir?: string;
|
|
805
420
|
apiUrl?: string;
|
|
806
421
|
loadToken?: () => string | null;
|
|
807
422
|
fetchImpl?: typeof fetch;
|
package/dist/index.js
CHANGED
|
@@ -97,9 +97,9 @@ var modelPriceSchema = z.object({
|
|
|
97
97
|
input_per_1k_usd: z.number().nonnegative(),
|
|
98
98
|
output_per_1k_usd: z.number().nonnegative()
|
|
99
99
|
});
|
|
100
|
-
var pricingSchema = z.record(modelPriceSchema);
|
|
100
|
+
var pricingSchema = z.record(z.string(), modelPriceSchema);
|
|
101
101
|
var datasetRowSchema = z.object({
|
|
102
|
-
vars: z.record(z.union([z.string(), z.number(), z.boolean()])),
|
|
102
|
+
vars: z.record(z.string(), z.union([z.string(), z.number(), z.boolean()])),
|
|
103
103
|
expected: z.string().optional()
|
|
104
104
|
});
|
|
105
105
|
var datasetRefSchema = z.union([
|
|
@@ -129,7 +129,7 @@ var containsAllAssertionSchema = z.object({
|
|
|
129
129
|
});
|
|
130
130
|
var jsonSchemaAssertionSchema = z.object({
|
|
131
131
|
type: z.literal("json-schema"),
|
|
132
|
-
schema: z.record(z.unknown()),
|
|
132
|
+
schema: z.record(z.string(), z.unknown()),
|
|
133
133
|
extract: z.enum(["auto", "strict"]).optional()
|
|
134
134
|
});
|
|
135
135
|
var llmJudgeAssertionSchema = z.object({
|
|
@@ -1491,7 +1491,7 @@ var runErrorSchema = z2.object({
|
|
|
1491
1491
|
message: z2.string()
|
|
1492
1492
|
});
|
|
1493
1493
|
var rowResultSchema = z2.object({
|
|
1494
|
-
vars: z2.record(z2.unknown()),
|
|
1494
|
+
vars: z2.record(z2.string(), z2.unknown()),
|
|
1495
1495
|
output: z2.string().nullable(),
|
|
1496
1496
|
cached: z2.boolean(),
|
|
1497
1497
|
usage: usageSchema.optional(),
|
|
@@ -1526,6 +1526,13 @@ function parseRunResult(value) {
|
|
|
1526
1526
|
return runResultSchema.parse(value);
|
|
1527
1527
|
}
|
|
1528
1528
|
|
|
1529
|
+
// src/cli/api-url.ts
|
|
1530
|
+
var DEFAULT_API_URL = "https://beta-api.plune.ai";
|
|
1531
|
+
function resolveApiUrl(explicit) {
|
|
1532
|
+
const raw = explicit ?? process.env["PLUNE_API_URL"] ?? DEFAULT_API_URL;
|
|
1533
|
+
return raw.replace(/\/+$/, "");
|
|
1534
|
+
}
|
|
1535
|
+
|
|
1529
1536
|
// src/cli/credentials.ts
|
|
1530
1537
|
import { chmodSync, existsSync as existsSync2, mkdirSync as mkdirSync2, readFileSync as readFileSync3, rmSync, writeFileSync as writeFileSync2 } from "fs";
|
|
1531
1538
|
import { homedir } from "os";
|
|
@@ -1544,7 +1551,6 @@ function loadToken() {
|
|
|
1544
1551
|
}
|
|
1545
1552
|
|
|
1546
1553
|
// src/cli/commands/sync.ts
|
|
1547
|
-
var DEFAULT_API_URL = "https://beta-api.plune.ai";
|
|
1548
1554
|
var NotLoggedInError = class extends Error {
|
|
1549
1555
|
constructor() {
|
|
1550
1556
|
super('Not logged in. Run "plune login" first to save your API token.');
|
|
@@ -1581,10 +1587,6 @@ var SyncHttpError = class extends Error {
|
|
|
1581
1587
|
}
|
|
1582
1588
|
status;
|
|
1583
1589
|
};
|
|
1584
|
-
function resolveApiUrl(explicit) {
|
|
1585
|
-
const raw = explicit ?? process.env["PLUNE_API_URL"] ?? DEFAULT_API_URL;
|
|
1586
|
-
return raw.replace(/\/+$/, "");
|
|
1587
|
-
}
|
|
1588
1590
|
function readRun(cwd, file) {
|
|
1589
1591
|
const target = file !== void 0 ? path5.resolve(cwd, file) : path5.join(cwd, ".plune", "last-run.json");
|
|
1590
1592
|
let raw;
|
|
@@ -1659,9 +1661,36 @@ import * as fs7 from "fs/promises";
|
|
|
1659
1661
|
import * as path6 from "path";
|
|
1660
1662
|
var SUPPORTED_ARTIFACT_VERSION = 1;
|
|
1661
1663
|
var INGEST_SCHEMA_VERSION = 1;
|
|
1664
|
+
var NoCairnRunsError = class extends Error {
|
|
1665
|
+
constructor(runsDir) {
|
|
1666
|
+
super(`No Cairn run found under ${runsDir} \u2014 name the directory instead: plune ingest <dir>.`);
|
|
1667
|
+
this.name = "NoCairnRunsError";
|
|
1668
|
+
}
|
|
1669
|
+
};
|
|
1670
|
+
async function findLatestCairnRun(runsDir = "runs") {
|
|
1671
|
+
const entries = await fs7.readdir(runsDir, { withFileTypes: true }).catch(() => null);
|
|
1672
|
+
if (entries === null) throw new NoCairnRunsError(runsDir);
|
|
1673
|
+
let newest;
|
|
1674
|
+
for (const entry of entries) {
|
|
1675
|
+
if (!entry.isDirectory()) continue;
|
|
1676
|
+
const dir = path6.join(runsDir, entry.name);
|
|
1677
|
+
try {
|
|
1678
|
+
const report = await fs7.lstat(path6.join(dir, "report.json"));
|
|
1679
|
+
if (!report.isFile()) continue;
|
|
1680
|
+
if (newest === void 0 || report.mtimeMs > newest.at) {
|
|
1681
|
+
newest = { dir, at: report.mtimeMs };
|
|
1682
|
+
}
|
|
1683
|
+
} catch {
|
|
1684
|
+
}
|
|
1685
|
+
}
|
|
1686
|
+
if (newest === void 0) throw new NoCairnRunsError(runsDir);
|
|
1687
|
+
return newest.dir;
|
|
1688
|
+
}
|
|
1662
1689
|
var CairnRunNotFoundError = class extends Error {
|
|
1663
1690
|
constructor(dir) {
|
|
1664
|
-
super(
|
|
1691
|
+
super(
|
|
1692
|
+
`No Cairn run found in ${dir} \u2014 expected a report.json written by "cairn design|explore|api".`
|
|
1693
|
+
);
|
|
1665
1694
|
this.name = "CairnRunNotFoundError";
|
|
1666
1695
|
}
|
|
1667
1696
|
};
|
|
@@ -1750,7 +1779,9 @@ async function readCairnRun(dir, deps = {}) {
|
|
|
1750
1779
|
}
|
|
1751
1780
|
const mode = report["mode"];
|
|
1752
1781
|
if (mode !== "design" && mode !== "explore" && mode !== "api") {
|
|
1753
|
-
throw new CairnRunVersionError(
|
|
1782
|
+
throw new CairnRunVersionError(
|
|
1783
|
+
`${SUPPORTED_ARTIFACT_VERSION} (unknown run mode ${String(mode)})`
|
|
1784
|
+
);
|
|
1754
1785
|
}
|
|
1755
1786
|
let cases;
|
|
1756
1787
|
if (mode === "api") {
|
|
@@ -1776,7 +1807,6 @@ async function readCairnRun(dir, deps = {}) {
|
|
|
1776
1807
|
}
|
|
1777
1808
|
|
|
1778
1809
|
// src/cli/commands/ingest.ts
|
|
1779
|
-
var DEFAULT_API_URL2 = "https://beta-api.plune.ai";
|
|
1780
1810
|
var IngestNotLoggedInError = class extends Error {
|
|
1781
1811
|
constructor() {
|
|
1782
1812
|
super('Not logged in. Run "plune login" first to save your API token.');
|
|
@@ -1803,10 +1833,6 @@ var IngestHttpError = class extends Error {
|
|
|
1803
1833
|
}
|
|
1804
1834
|
status;
|
|
1805
1835
|
};
|
|
1806
|
-
function resolveApiUrl2(explicit) {
|
|
1807
|
-
const raw = explicit ?? process.env["PLUNE_API_URL"] ?? DEFAULT_API_URL2;
|
|
1808
|
-
return raw.replace(/\/+$/, "");
|
|
1809
|
-
}
|
|
1810
1836
|
async function safeErrorDetail2(res) {
|
|
1811
1837
|
try {
|
|
1812
1838
|
const body = await res.json();
|
|
@@ -1816,10 +1842,11 @@ async function safeErrorDetail2(res) {
|
|
|
1816
1842
|
}
|
|
1817
1843
|
}
|
|
1818
1844
|
async function handleIngest(deps) {
|
|
1819
|
-
const apiUrl =
|
|
1845
|
+
const apiUrl = resolveApiUrl(deps.apiUrl);
|
|
1820
1846
|
const token = (deps.loadToken ?? loadToken)();
|
|
1821
1847
|
if (token === null) throw new IngestNotLoggedInError();
|
|
1822
|
-
const
|
|
1848
|
+
const dir = deps.dir ?? await findLatestCairnRun(deps.runsDir);
|
|
1849
|
+
const payload = await readCairnRun(dir);
|
|
1823
1850
|
const doFetch = deps.fetchImpl ?? fetch;
|
|
1824
1851
|
const endpoint = `${apiUrl}/v1/ingest/cairn`;
|
|
1825
1852
|
let res;
|
|
@@ -1846,7 +1873,7 @@ async function handleIngest(deps) {
|
|
|
1846
1873
|
};
|
|
1847
1874
|
}
|
|
1848
1875
|
function reportIngestFailure(err, write) {
|
|
1849
|
-
if (err instanceof IngestNotLoggedInError || err instanceof IngestTokenRejectedError || err instanceof CairnRunNotFoundError || err instanceof CairnRunVersionError || err instanceof CairnRunPartialError || err instanceof CairnEvidenceAmbiguousError) {
|
|
1876
|
+
if (err instanceof IngestNotLoggedInError || err instanceof IngestTokenRejectedError || err instanceof CairnRunNotFoundError || err instanceof NoCairnRunsError || err instanceof CairnRunVersionError || err instanceof CairnRunPartialError || err instanceof CairnEvidenceAmbiguousError) {
|
|
1850
1877
|
write(`${err.message}
|
|
1851
1878
|
`);
|
|
1852
1879
|
return 2;
|