@agentv/eval 3.5.0 → 3.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (7) hide show
  1. package/dist/index.cjs +13 -69
  2. package/dist/index.cjs.map +1 -1
  3. package/dist/index.d.cts +30 -70
  4. package/dist/index.d.ts +30 -70
  5. package/dist/index.js +13 -69
  6. package/dist/index.js.map +1 -1
  7. package/package.json +1 -1
package/dist/index.d.cts CHANGED
@@ -153,8 +153,6 @@ declare const MessageSchema: z.ZodObject<{
153
153
  * strings. Structured fields (`input`, `output`, `expectedOutput`) are always `Message[]`.
154
154
  */
155
155
  declare const CodeGraderInputSchema: z.ZodObject<{
156
- /** @deprecated Use `inputText` instead. First user message content as string. */
157
- question: z.ZodString;
158
156
  criteria: z.ZodString;
159
157
  expectedOutput: z.ZodArray<z.ZodObject<{
160
158
  role: z.ZodEnum<["assistant", "user", "system", "tool"]>;
@@ -224,10 +222,8 @@ declare const CodeGraderInputSchema: z.ZodObject<{
224
222
  name?: string | undefined;
225
223
  metadata?: Record<string, unknown> | undefined;
226
224
  }>, "many">;
227
- /** @deprecated Use `expectedOutputText` instead. Expected output content as string. */
228
- referenceAnswer: z.ZodOptional<z.ZodString>;
229
- /** @deprecated Use `outputText` instead. Last assistant message content as string. */
230
- answer: z.ZodString;
225
+ /** Last assistant message content as string. */
226
+ outputText: z.ZodString;
231
227
  output: z.ZodOptional<z.ZodNullable<z.ZodArray<z.ZodObject<{
232
228
  role: z.ZodEnum<["assistant", "user", "system", "tool"]>;
233
229
  content: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodRecord<z.ZodString, z.ZodUnknown>, z.ZodArray<z.ZodRecord<z.ZodString, z.ZodUnknown>, "many">]>>;
@@ -410,11 +406,9 @@ declare const CodeGraderInputSchema: z.ZodObject<{
410
406
  fileChanges: z.ZodOptional<z.ZodNullable<z.ZodString>>;
411
407
  workspacePath: z.ZodOptional<z.ZodNullable<z.ZodString>>;
412
408
  config: z.ZodOptional<z.ZodNullable<z.ZodRecord<z.ZodString, z.ZodUnknown>>>;
413
- /** First user message content as string. Replaces `question`. */
414
- inputText: z.ZodOptional<z.ZodString>;
415
- /** Last assistant message content as string. Replaces `answer`. */
416
- outputText: z.ZodOptional<z.ZodString>;
417
- /** Expected output content as string. Replaces `referenceAnswer`. */
409
+ /** First user message content as string. */
410
+ inputText: z.ZodString;
411
+ /** Expected output content as string. */
418
412
  expectedOutputText: z.ZodOptional<z.ZodString>;
419
413
  }, "strip", z.ZodTypeAny, {
420
414
  input: {
@@ -435,7 +429,6 @@ declare const CodeGraderInputSchema: z.ZodObject<{
435
429
  name?: string | undefined;
436
430
  metadata?: Record<string, unknown> | undefined;
437
431
  }[];
438
- question: string;
439
432
  criteria: string;
440
433
  expectedOutput: {
441
434
  role: "tool" | "assistant" | "user" | "system";
@@ -455,9 +448,10 @@ declare const CodeGraderInputSchema: z.ZodObject<{
455
448
  name?: string | undefined;
456
449
  metadata?: Record<string, unknown> | undefined;
457
450
  }[];
458
- answer: string;
451
+ outputText: string;
459
452
  guidelineFiles: string[];
460
453
  inputFiles: string[];
454
+ inputText: string;
461
455
  output?: {
462
456
  role: "tool" | "assistant" | "user" | "system";
463
457
  startTime?: string | undefined;
@@ -479,7 +473,6 @@ declare const CodeGraderInputSchema: z.ZodObject<{
479
473
  startTime?: string | null | undefined;
480
474
  endTime?: string | null | undefined;
481
475
  durationMs?: number | null | undefined;
482
- referenceAnswer?: string | undefined;
483
476
  outputPath?: string | undefined;
484
477
  trace?: {
485
478
  eventCount: number;
@@ -498,8 +491,6 @@ declare const CodeGraderInputSchema: z.ZodObject<{
498
491
  fileChanges?: string | null | undefined;
499
492
  workspacePath?: string | null | undefined;
500
493
  config?: Record<string, unknown> | null | undefined;
501
- inputText?: string | undefined;
502
- outputText?: string | undefined;
503
494
  expectedOutputText?: string | undefined;
504
495
  }, {
505
496
  input: {
@@ -520,7 +511,6 @@ declare const CodeGraderInputSchema: z.ZodObject<{
520
511
  name?: string | undefined;
521
512
  metadata?: Record<string, unknown> | undefined;
522
513
  }[];
523
- question: string;
524
514
  criteria: string;
525
515
  expectedOutput: {
526
516
  role: "tool" | "assistant" | "user" | "system";
@@ -540,9 +530,10 @@ declare const CodeGraderInputSchema: z.ZodObject<{
540
530
  name?: string | undefined;
541
531
  metadata?: Record<string, unknown> | undefined;
542
532
  }[];
543
- answer: string;
533
+ outputText: string;
544
534
  guidelineFiles: string[];
545
535
  inputFiles: string[];
536
+ inputText: string;
546
537
  output?: {
547
538
  role: "tool" | "assistant" | "user" | "system";
548
539
  startTime?: string | undefined;
@@ -564,7 +555,6 @@ declare const CodeGraderInputSchema: z.ZodObject<{
564
555
  startTime?: string | null | undefined;
565
556
  endTime?: string | null | undefined;
566
557
  durationMs?: number | null | undefined;
567
- referenceAnswer?: string | undefined;
568
558
  outputPath?: string | undefined;
569
559
  trace?: {
570
560
  eventCount: number;
@@ -583,8 +573,6 @@ declare const CodeGraderInputSchema: z.ZodObject<{
583
573
  fileChanges?: string | null | undefined;
584
574
  workspacePath?: string | null | undefined;
585
575
  config?: Record<string, unknown> | null | undefined;
586
- inputText?: string | undefined;
587
- outputText?: string | undefined;
588
576
  expectedOutputText?: string | undefined;
589
577
  }>;
590
578
  /**
@@ -632,19 +620,15 @@ type CodeGraderResult = z.infer<typeof CodeGraderResultSchema>;
632
620
  /**
633
621
  * CodeGraderInput after `enrichInput()` has run.
634
622
  *
635
- * The text convenience accessors (`inputText`, `outputText`, `expectedOutputText`)
623
+ * The text accessors (`inputText`, `outputText`, `expectedOutputText`)
636
624
  * are always populated by the runtime before the handler is called, so they are
637
625
  * guaranteed to be `string` (never `undefined`).
638
626
  *
639
627
  * Handler function signatures (`CodeGraderHandler`, `AssertionHandler`) use this
640
628
  * type so that user code can destructure `{ outputText }` without null-checks.
641
629
  */
642
- type EnrichedCodeGraderInput = Omit<CodeGraderInput, 'inputText' | 'outputText' | 'expectedOutputText'> & {
643
- /** First user message content as string. Replaces `question`. */
644
- readonly inputText: string;
645
- /** Last assistant message content as string. Replaces `answer`. */
646
- readonly outputText: string;
647
- /** Expected output content as string. Replaces `referenceAnswer`. */
630
+ type EnrichedCodeGraderInput = Omit<CodeGraderInput, 'expectedOutputText'> & {
631
+ /** Expected output content as string. */
648
632
  readonly expectedOutputText: string;
649
633
  };
650
634
  type TraceSummary = z.infer<typeof TraceSummarySchema>;
@@ -656,8 +640,6 @@ type TokenUsage = z.infer<typeof TokenUsageSchema>;
656
640
  * Uses the same schema as CodeGraderInput since the orchestrator sends identical payloads.
657
641
  */
658
642
  declare const PromptTemplateInputSchema: z.ZodObject<{
659
- /** @deprecated Use `inputText` instead. First user message content as string. */
660
- question: z.ZodString;
661
643
  criteria: z.ZodString;
662
644
  expectedOutput: z.ZodArray<z.ZodObject<{
663
645
  role: z.ZodEnum<["assistant", "user", "system", "tool"]>;
@@ -727,10 +709,8 @@ declare const PromptTemplateInputSchema: z.ZodObject<{
727
709
  name?: string | undefined;
728
710
  metadata?: Record<string, unknown> | undefined;
729
711
  }>, "many">;
730
- /** @deprecated Use `expectedOutputText` instead. Expected output content as string. */
731
- referenceAnswer: z.ZodOptional<z.ZodString>;
732
- /** @deprecated Use `outputText` instead. Last assistant message content as string. */
733
- answer: z.ZodString;
712
+ /** Last assistant message content as string. */
713
+ outputText: z.ZodString;
734
714
  output: z.ZodOptional<z.ZodNullable<z.ZodArray<z.ZodObject<{
735
715
  role: z.ZodEnum<["assistant", "user", "system", "tool"]>;
736
716
  content: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodRecord<z.ZodString, z.ZodUnknown>, z.ZodArray<z.ZodRecord<z.ZodString, z.ZodUnknown>, "many">]>>;
@@ -913,11 +893,9 @@ declare const PromptTemplateInputSchema: z.ZodObject<{
913
893
  fileChanges: z.ZodOptional<z.ZodNullable<z.ZodString>>;
914
894
  workspacePath: z.ZodOptional<z.ZodNullable<z.ZodString>>;
915
895
  config: z.ZodOptional<z.ZodNullable<z.ZodRecord<z.ZodString, z.ZodUnknown>>>;
916
- /** First user message content as string. Replaces `question`. */
917
- inputText: z.ZodOptional<z.ZodString>;
918
- /** Last assistant message content as string. Replaces `answer`. */
919
- outputText: z.ZodOptional<z.ZodString>;
920
- /** Expected output content as string. Replaces `referenceAnswer`. */
896
+ /** First user message content as string. */
897
+ inputText: z.ZodString;
898
+ /** Expected output content as string. */
921
899
  expectedOutputText: z.ZodOptional<z.ZodString>;
922
900
  }, "strip", z.ZodTypeAny, {
923
901
  input: {
@@ -938,7 +916,6 @@ declare const PromptTemplateInputSchema: z.ZodObject<{
938
916
  name?: string | undefined;
939
917
  metadata?: Record<string, unknown> | undefined;
940
918
  }[];
941
- question: string;
942
919
  criteria: string;
943
920
  expectedOutput: {
944
921
  role: "tool" | "assistant" | "user" | "system";
@@ -958,9 +935,10 @@ declare const PromptTemplateInputSchema: z.ZodObject<{
958
935
  name?: string | undefined;
959
936
  metadata?: Record<string, unknown> | undefined;
960
937
  }[];
961
- answer: string;
938
+ outputText: string;
962
939
  guidelineFiles: string[];
963
940
  inputFiles: string[];
941
+ inputText: string;
964
942
  output?: {
965
943
  role: "tool" | "assistant" | "user" | "system";
966
944
  startTime?: string | undefined;
@@ -982,7 +960,6 @@ declare const PromptTemplateInputSchema: z.ZodObject<{
982
960
  startTime?: string | null | undefined;
983
961
  endTime?: string | null | undefined;
984
962
  durationMs?: number | null | undefined;
985
- referenceAnswer?: string | undefined;
986
963
  outputPath?: string | undefined;
987
964
  trace?: {
988
965
  eventCount: number;
@@ -1001,8 +978,6 @@ declare const PromptTemplateInputSchema: z.ZodObject<{
1001
978
  fileChanges?: string | null | undefined;
1002
979
  workspacePath?: string | null | undefined;
1003
980
  config?: Record<string, unknown> | null | undefined;
1004
- inputText?: string | undefined;
1005
- outputText?: string | undefined;
1006
981
  expectedOutputText?: string | undefined;
1007
982
  }, {
1008
983
  input: {
@@ -1023,7 +998,6 @@ declare const PromptTemplateInputSchema: z.ZodObject<{
1023
998
  name?: string | undefined;
1024
999
  metadata?: Record<string, unknown> | undefined;
1025
1000
  }[];
1026
- question: string;
1027
1001
  criteria: string;
1028
1002
  expectedOutput: {
1029
1003
  role: "tool" | "assistant" | "user" | "system";
@@ -1043,9 +1017,10 @@ declare const PromptTemplateInputSchema: z.ZodObject<{
1043
1017
  name?: string | undefined;
1044
1018
  metadata?: Record<string, unknown> | undefined;
1045
1019
  }[];
1046
- answer: string;
1020
+ outputText: string;
1047
1021
  guidelineFiles: string[];
1048
1022
  inputFiles: string[];
1023
+ inputText: string;
1049
1024
  output?: {
1050
1025
  role: "tool" | "assistant" | "user" | "system";
1051
1026
  startTime?: string | undefined;
@@ -1067,7 +1042,6 @@ declare const PromptTemplateInputSchema: z.ZodObject<{
1067
1042
  startTime?: string | null | undefined;
1068
1043
  endTime?: string | null | undefined;
1069
1044
  durationMs?: number | null | undefined;
1070
- referenceAnswer?: string | undefined;
1071
1045
  outputPath?: string | undefined;
1072
1046
  trace?: {
1073
1047
  eventCount: number;
@@ -1086,15 +1060,11 @@ declare const PromptTemplateInputSchema: z.ZodObject<{
1086
1060
  fileChanges?: string | null | undefined;
1087
1061
  workspacePath?: string | null | undefined;
1088
1062
  config?: Record<string, unknown> | null | undefined;
1089
- inputText?: string | undefined;
1090
- outputText?: string | undefined;
1091
1063
  expectedOutputText?: string | undefined;
1092
1064
  }>;
1093
1065
  type PromptTemplateInput = CodeGraderInput;
1094
1066
  /** @deprecated Use CodeGraderInputSchema */
1095
1067
  declare const CodeJudgeInputSchema: z.ZodObject<{
1096
- /** @deprecated Use `inputText` instead. First user message content as string. */
1097
- question: z.ZodString;
1098
1068
  criteria: z.ZodString;
1099
1069
  expectedOutput: z.ZodArray<z.ZodObject<{
1100
1070
  role: z.ZodEnum<["assistant", "user", "system", "tool"]>;
@@ -1164,10 +1134,8 @@ declare const CodeJudgeInputSchema: z.ZodObject<{
1164
1134
  name?: string | undefined;
1165
1135
  metadata?: Record<string, unknown> | undefined;
1166
1136
  }>, "many">;
1167
- /** @deprecated Use `expectedOutputText` instead. Expected output content as string. */
1168
- referenceAnswer: z.ZodOptional<z.ZodString>;
1169
- /** @deprecated Use `outputText` instead. Last assistant message content as string. */
1170
- answer: z.ZodString;
1137
+ /** Last assistant message content as string. */
1138
+ outputText: z.ZodString;
1171
1139
  output: z.ZodOptional<z.ZodNullable<z.ZodArray<z.ZodObject<{
1172
1140
  role: z.ZodEnum<["assistant", "user", "system", "tool"]>;
1173
1141
  content: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodRecord<z.ZodString, z.ZodUnknown>, z.ZodArray<z.ZodRecord<z.ZodString, z.ZodUnknown>, "many">]>>;
@@ -1350,11 +1318,9 @@ declare const CodeJudgeInputSchema: z.ZodObject<{
1350
1318
  fileChanges: z.ZodOptional<z.ZodNullable<z.ZodString>>;
1351
1319
  workspacePath: z.ZodOptional<z.ZodNullable<z.ZodString>>;
1352
1320
  config: z.ZodOptional<z.ZodNullable<z.ZodRecord<z.ZodString, z.ZodUnknown>>>;
1353
- /** First user message content as string. Replaces `question`. */
1354
- inputText: z.ZodOptional<z.ZodString>;
1355
- /** Last assistant message content as string. Replaces `answer`. */
1356
- outputText: z.ZodOptional<z.ZodString>;
1357
- /** Expected output content as string. Replaces `referenceAnswer`. */
1321
+ /** First user message content as string. */
1322
+ inputText: z.ZodString;
1323
+ /** Expected output content as string. */
1358
1324
  expectedOutputText: z.ZodOptional<z.ZodString>;
1359
1325
  }, "strip", z.ZodTypeAny, {
1360
1326
  input: {
@@ -1375,7 +1341,6 @@ declare const CodeJudgeInputSchema: z.ZodObject<{
1375
1341
  name?: string | undefined;
1376
1342
  metadata?: Record<string, unknown> | undefined;
1377
1343
  }[];
1378
- question: string;
1379
1344
  criteria: string;
1380
1345
  expectedOutput: {
1381
1346
  role: "tool" | "assistant" | "user" | "system";
@@ -1395,9 +1360,10 @@ declare const CodeJudgeInputSchema: z.ZodObject<{
1395
1360
  name?: string | undefined;
1396
1361
  metadata?: Record<string, unknown> | undefined;
1397
1362
  }[];
1398
- answer: string;
1363
+ outputText: string;
1399
1364
  guidelineFiles: string[];
1400
1365
  inputFiles: string[];
1366
+ inputText: string;
1401
1367
  output?: {
1402
1368
  role: "tool" | "assistant" | "user" | "system";
1403
1369
  startTime?: string | undefined;
@@ -1419,7 +1385,6 @@ declare const CodeJudgeInputSchema: z.ZodObject<{
1419
1385
  startTime?: string | null | undefined;
1420
1386
  endTime?: string | null | undefined;
1421
1387
  durationMs?: number | null | undefined;
1422
- referenceAnswer?: string | undefined;
1423
1388
  outputPath?: string | undefined;
1424
1389
  trace?: {
1425
1390
  eventCount: number;
@@ -1438,8 +1403,6 @@ declare const CodeJudgeInputSchema: z.ZodObject<{
1438
1403
  fileChanges?: string | null | undefined;
1439
1404
  workspacePath?: string | null | undefined;
1440
1405
  config?: Record<string, unknown> | null | undefined;
1441
- inputText?: string | undefined;
1442
- outputText?: string | undefined;
1443
1406
  expectedOutputText?: string | undefined;
1444
1407
  }, {
1445
1408
  input: {
@@ -1460,7 +1423,6 @@ declare const CodeJudgeInputSchema: z.ZodObject<{
1460
1423
  name?: string | undefined;
1461
1424
  metadata?: Record<string, unknown> | undefined;
1462
1425
  }[];
1463
- question: string;
1464
1426
  criteria: string;
1465
1427
  expectedOutput: {
1466
1428
  role: "tool" | "assistant" | "user" | "system";
@@ -1480,9 +1442,10 @@ declare const CodeJudgeInputSchema: z.ZodObject<{
1480
1442
  name?: string | undefined;
1481
1443
  metadata?: Record<string, unknown> | undefined;
1482
1444
  }[];
1483
- answer: string;
1445
+ outputText: string;
1484
1446
  guidelineFiles: string[];
1485
1447
  inputFiles: string[];
1448
+ inputText: string;
1486
1449
  output?: {
1487
1450
  role: "tool" | "assistant" | "user" | "system";
1488
1451
  startTime?: string | undefined;
@@ -1504,7 +1467,6 @@ declare const CodeJudgeInputSchema: z.ZodObject<{
1504
1467
  startTime?: string | null | undefined;
1505
1468
  endTime?: string | null | undefined;
1506
1469
  durationMs?: number | null | undefined;
1507
- referenceAnswer?: string | undefined;
1508
1470
  outputPath?: string | undefined;
1509
1471
  trace?: {
1510
1472
  eventCount: number;
@@ -1523,8 +1485,6 @@ declare const CodeJudgeInputSchema: z.ZodObject<{
1523
1485
  fileChanges?: string | null | undefined;
1524
1486
  workspacePath?: string | null | undefined;
1525
1487
  config?: Record<string, unknown> | null | undefined;
1526
- inputText?: string | undefined;
1527
- outputText?: string | undefined;
1528
1488
  expectedOutputText?: string | undefined;
1529
1489
  }>;
1530
1490
  /** @deprecated Use CodeGraderResultSchema */