@agentv/eval 4.35.0 → 4.35.1-next.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (7) hide show
  1. package/dist/index.cjs +2 -1
  2. package/dist/index.cjs.map +1 -1
  3. package/dist/index.d.cts +47 -44
  4. package/dist/index.d.ts +47 -44
  5. package/dist/index.js +2 -1
  6. package/dist/index.js.map +1 -1
  7. package/package.json +1 -1
package/dist/index.d.cts CHANGED
@@ -7,9 +7,11 @@ export { z } from 'zod';
7
7
  *
8
8
  * ## Content model
9
9
  *
10
- * `Message.content` accepts `string | Content[]`:
10
+ * `Message.content` accepts `string | object[] | object`:
11
11
  * - `string` — backward-compatible plain text (most common case)
12
- * - `Content[]` — typed content blocks for multimodal messages
12
+ * - `object[]` — typed content blocks for multimodal messages, plus AgentV
13
+ * eval input blocks such as `{ type: "file", value, path, text }`
14
+ * - `object` — structured YAML/JSON content such as expected outputs
13
15
  *
14
16
  * Content variants:
15
17
  * - `ContentText` — `{ type: 'text', text: string }`
@@ -1708,12 +1710,13 @@ declare const ContentSchema: z.ZodDiscriminatedUnion<"type", [z.ZodObject<{
1708
1710
  /**
1709
1711
  * Unified message schema for input, expected, and output messages.
1710
1712
  *
1711
- * `content` is either a plain string or a `Content[]` array of typed blocks.
1712
- * Use `getTextContent()` from `@agentv/core` to extract plain text from either form.
1713
+ * `content` is a plain string, an array of structured blocks, or a
1714
+ * structured object from YAML/JSON eval files. Use `getTextContent()` from
1715
+ * `@agentv/core` to extract plain text when the content is textual.
1713
1716
  */
1714
1717
  declare const MessageSchema: z.ZodObject<{
1715
1718
  role: z.ZodEnum<["assistant", "user", "system", "tool"]>;
1716
- content: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodArray<z.ZodDiscriminatedUnion<"type", [z.ZodObject<{
1719
+ content: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodArray<z.ZodUnion<[z.ZodDiscriminatedUnion<"type", [z.ZodObject<{
1717
1720
  type: z.ZodLiteral<"text">;
1718
1721
  text: z.ZodString;
1719
1722
  }, "strip", z.ZodTypeAny, {
@@ -1746,7 +1749,7 @@ declare const MessageSchema: z.ZodObject<{
1746
1749
  path: string;
1747
1750
  type: "file";
1748
1751
  media_type: string;
1749
- }>]>, "many">]>>;
1752
+ }>]>, z.ZodRecord<z.ZodString, z.ZodUnknown>]>, "many">, z.ZodRecord<z.ZodString, z.ZodUnknown>]>>;
1750
1753
  toolCalls: z.ZodOptional<z.ZodArray<z.ZodObject<{
1751
1754
  tool: z.ZodString;
1752
1755
  input: z.ZodOptional<z.ZodUnknown>;
@@ -1790,7 +1793,7 @@ declare const MessageSchema: z.ZodObject<{
1790
1793
  }[] | undefined;
1791
1794
  name?: string | undefined;
1792
1795
  metadata?: Record<string, unknown> | undefined;
1793
- content?: string | ({
1796
+ content?: string | Record<string, unknown> | (Record<string, unknown> | {
1794
1797
  type: "text";
1795
1798
  text: string;
1796
1799
  } | {
@@ -1818,7 +1821,7 @@ declare const MessageSchema: z.ZodObject<{
1818
1821
  }[] | undefined;
1819
1822
  name?: string | undefined;
1820
1823
  metadata?: Record<string, unknown> | undefined;
1821
- content?: string | ({
1824
+ content?: string | Record<string, unknown> | (Record<string, unknown> | {
1822
1825
  type: "text";
1823
1826
  text: string;
1824
1827
  } | {
@@ -1844,7 +1847,7 @@ declare const CodeGraderInputSchema: z.ZodObject<{
1844
1847
  criteria: z.ZodString;
1845
1848
  expectedOutput: z.ZodArray<z.ZodObject<{
1846
1849
  role: z.ZodEnum<["assistant", "user", "system", "tool"]>;
1847
- content: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodArray<z.ZodDiscriminatedUnion<"type", [z.ZodObject<{
1850
+ content: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodArray<z.ZodUnion<[z.ZodDiscriminatedUnion<"type", [z.ZodObject<{
1848
1851
  type: z.ZodLiteral<"text">;
1849
1852
  text: z.ZodString;
1850
1853
  }, "strip", z.ZodTypeAny, {
@@ -1877,7 +1880,7 @@ declare const CodeGraderInputSchema: z.ZodObject<{
1877
1880
  path: string;
1878
1881
  type: "file";
1879
1882
  media_type: string;
1880
- }>]>, "many">]>>;
1883
+ }>]>, z.ZodRecord<z.ZodString, z.ZodUnknown>]>, "many">, z.ZodRecord<z.ZodString, z.ZodUnknown>]>>;
1881
1884
  toolCalls: z.ZodOptional<z.ZodArray<z.ZodObject<{
1882
1885
  tool: z.ZodString;
1883
1886
  input: z.ZodOptional<z.ZodUnknown>;
@@ -1921,7 +1924,7 @@ declare const CodeGraderInputSchema: z.ZodObject<{
1921
1924
  }[] | undefined;
1922
1925
  name?: string | undefined;
1923
1926
  metadata?: Record<string, unknown> | undefined;
1924
- content?: string | ({
1927
+ content?: string | Record<string, unknown> | (Record<string, unknown> | {
1925
1928
  type: "text";
1926
1929
  text: string;
1927
1930
  } | {
@@ -1949,7 +1952,7 @@ declare const CodeGraderInputSchema: z.ZodObject<{
1949
1952
  }[] | undefined;
1950
1953
  name?: string | undefined;
1951
1954
  metadata?: Record<string, unknown> | undefined;
1952
- content?: string | ({
1955
+ content?: string | Record<string, unknown> | (Record<string, unknown> | {
1953
1956
  type: "text";
1954
1957
  text: string;
1955
1958
  } | {
@@ -1967,7 +1970,7 @@ declare const CodeGraderInputSchema: z.ZodObject<{
1967
1970
  }>, "many">;
1968
1971
  output: z.ZodOptional<z.ZodNullable<z.ZodArray<z.ZodObject<{
1969
1972
  role: z.ZodEnum<["assistant", "user", "system", "tool"]>;
1970
- content: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodArray<z.ZodDiscriminatedUnion<"type", [z.ZodObject<{
1973
+ content: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodArray<z.ZodUnion<[z.ZodDiscriminatedUnion<"type", [z.ZodObject<{
1971
1974
  type: z.ZodLiteral<"text">;
1972
1975
  text: z.ZodString;
1973
1976
  }, "strip", z.ZodTypeAny, {
@@ -2000,7 +2003,7 @@ declare const CodeGraderInputSchema: z.ZodObject<{
2000
2003
  path: string;
2001
2004
  type: "file";
2002
2005
  media_type: string;
2003
- }>]>, "many">]>>;
2006
+ }>]>, z.ZodRecord<z.ZodString, z.ZodUnknown>]>, "many">, z.ZodRecord<z.ZodString, z.ZodUnknown>]>>;
2004
2007
  toolCalls: z.ZodOptional<z.ZodArray<z.ZodObject<{
2005
2008
  tool: z.ZodString;
2006
2009
  input: z.ZodOptional<z.ZodUnknown>;
@@ -2044,7 +2047,7 @@ declare const CodeGraderInputSchema: z.ZodObject<{
2044
2047
  }[] | undefined;
2045
2048
  name?: string | undefined;
2046
2049
  metadata?: Record<string, unknown> | undefined;
2047
- content?: string | ({
2050
+ content?: string | Record<string, unknown> | (Record<string, unknown> | {
2048
2051
  type: "text";
2049
2052
  text: string;
2050
2053
  } | {
@@ -2072,7 +2075,7 @@ declare const CodeGraderInputSchema: z.ZodObject<{
2072
2075
  }[] | undefined;
2073
2076
  name?: string | undefined;
2074
2077
  metadata?: Record<string, unknown> | undefined;
2075
- content?: string | ({
2078
+ content?: string | Record<string, unknown> | (Record<string, unknown> | {
2076
2079
  type: "text";
2077
2080
  text: string;
2078
2081
  } | {
@@ -2093,7 +2096,7 @@ declare const CodeGraderInputSchema: z.ZodObject<{
2093
2096
  inputFiles: z.ZodArray<z.ZodString, "many">;
2094
2097
  input: z.ZodArray<z.ZodObject<{
2095
2098
  role: z.ZodEnum<["assistant", "user", "system", "tool"]>;
2096
- content: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodArray<z.ZodDiscriminatedUnion<"type", [z.ZodObject<{
2099
+ content: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodArray<z.ZodUnion<[z.ZodDiscriminatedUnion<"type", [z.ZodObject<{
2097
2100
  type: z.ZodLiteral<"text">;
2098
2101
  text: z.ZodString;
2099
2102
  }, "strip", z.ZodTypeAny, {
@@ -2126,7 +2129,7 @@ declare const CodeGraderInputSchema: z.ZodObject<{
2126
2129
  path: string;
2127
2130
  type: "file";
2128
2131
  media_type: string;
2129
- }>]>, "many">]>>;
2132
+ }>]>, z.ZodRecord<z.ZodString, z.ZodUnknown>]>, "many">, z.ZodRecord<z.ZodString, z.ZodUnknown>]>>;
2130
2133
  toolCalls: z.ZodOptional<z.ZodArray<z.ZodObject<{
2131
2134
  tool: z.ZodString;
2132
2135
  input: z.ZodOptional<z.ZodUnknown>;
@@ -2170,7 +2173,7 @@ declare const CodeGraderInputSchema: z.ZodObject<{
2170
2173
  }[] | undefined;
2171
2174
  name?: string | undefined;
2172
2175
  metadata?: Record<string, unknown> | undefined;
2173
- content?: string | ({
2176
+ content?: string | Record<string, unknown> | (Record<string, unknown> | {
2174
2177
  type: "text";
2175
2178
  text: string;
2176
2179
  } | {
@@ -2198,7 +2201,7 @@ declare const CodeGraderInputSchema: z.ZodObject<{
2198
2201
  }[] | undefined;
2199
2202
  name?: string | undefined;
2200
2203
  metadata?: Record<string, unknown> | undefined;
2201
- content?: string | ({
2204
+ content?: string | Record<string, unknown> | (Record<string, unknown> | {
2202
2205
  type: "text";
2203
2206
  text: string;
2204
2207
  } | {
@@ -2271,7 +2274,7 @@ declare const CodeGraderInputSchema: z.ZodObject<{
2271
2274
  }[] | undefined;
2272
2275
  name?: string | undefined;
2273
2276
  metadata?: Record<string, unknown> | undefined;
2274
- content?: string | ({
2277
+ content?: string | Record<string, unknown> | (Record<string, unknown> | {
2275
2278
  type: "text";
2276
2279
  text: string;
2277
2280
  } | {
@@ -2301,7 +2304,7 @@ declare const CodeGraderInputSchema: z.ZodObject<{
2301
2304
  }[] | undefined;
2302
2305
  name?: string | undefined;
2303
2306
  metadata?: Record<string, unknown> | undefined;
2304
- content?: string | ({
2307
+ content?: string | Record<string, unknown> | (Record<string, unknown> | {
2305
2308
  type: "text";
2306
2309
  text: string;
2307
2310
  } | {
@@ -2331,7 +2334,7 @@ declare const CodeGraderInputSchema: z.ZodObject<{
2331
2334
  }[] | undefined;
2332
2335
  name?: string | undefined;
2333
2336
  metadata?: Record<string, unknown> | undefined;
2334
- content?: string | ({
2337
+ content?: string | Record<string, unknown> | (Record<string, unknown> | {
2335
2338
  type: "text";
2336
2339
  text: string;
2337
2340
  } | {
@@ -2383,7 +2386,7 @@ declare const CodeGraderInputSchema: z.ZodObject<{
2383
2386
  }[] | undefined;
2384
2387
  name?: string | undefined;
2385
2388
  metadata?: Record<string, unknown> | undefined;
2386
- content?: string | ({
2389
+ content?: string | Record<string, unknown> | (Record<string, unknown> | {
2387
2390
  type: "text";
2388
2391
  text: string;
2389
2392
  } | {
@@ -2413,7 +2416,7 @@ declare const CodeGraderInputSchema: z.ZodObject<{
2413
2416
  }[] | undefined;
2414
2417
  name?: string | undefined;
2415
2418
  metadata?: Record<string, unknown> | undefined;
2416
- content?: string | ({
2419
+ content?: string | Record<string, unknown> | (Record<string, unknown> | {
2417
2420
  type: "text";
2418
2421
  text: string;
2419
2422
  } | {
@@ -2443,7 +2446,7 @@ declare const CodeGraderInputSchema: z.ZodObject<{
2443
2446
  }[] | undefined;
2444
2447
  name?: string | undefined;
2445
2448
  metadata?: Record<string, unknown> | undefined;
2446
- content?: string | ({
2449
+ content?: string | Record<string, unknown> | (Record<string, unknown> | {
2447
2450
  type: "text";
2448
2451
  text: string;
2449
2452
  } | {
@@ -2552,7 +2555,7 @@ declare const PromptTemplateInputSchema: z.ZodObject<{
2552
2555
  criteria: z.ZodString;
2553
2556
  expectedOutput: z.ZodArray<z.ZodObject<{
2554
2557
  role: z.ZodEnum<["assistant", "user", "system", "tool"]>;
2555
- content: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodArray<z.ZodDiscriminatedUnion<"type", [z.ZodObject<{
2558
+ content: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodArray<z.ZodUnion<[z.ZodDiscriminatedUnion<"type", [z.ZodObject<{
2556
2559
  type: z.ZodLiteral<"text">;
2557
2560
  text: z.ZodString;
2558
2561
  }, "strip", z.ZodTypeAny, {
@@ -2585,7 +2588,7 @@ declare const PromptTemplateInputSchema: z.ZodObject<{
2585
2588
  path: string;
2586
2589
  type: "file";
2587
2590
  media_type: string;
2588
- }>]>, "many">]>>;
2591
+ }>]>, z.ZodRecord<z.ZodString, z.ZodUnknown>]>, "many">, z.ZodRecord<z.ZodString, z.ZodUnknown>]>>;
2589
2592
  toolCalls: z.ZodOptional<z.ZodArray<z.ZodObject<{
2590
2593
  tool: z.ZodString;
2591
2594
  input: z.ZodOptional<z.ZodUnknown>;
@@ -2629,7 +2632,7 @@ declare const PromptTemplateInputSchema: z.ZodObject<{
2629
2632
  }[] | undefined;
2630
2633
  name?: string | undefined;
2631
2634
  metadata?: Record<string, unknown> | undefined;
2632
- content?: string | ({
2635
+ content?: string | Record<string, unknown> | (Record<string, unknown> | {
2633
2636
  type: "text";
2634
2637
  text: string;
2635
2638
  } | {
@@ -2657,7 +2660,7 @@ declare const PromptTemplateInputSchema: z.ZodObject<{
2657
2660
  }[] | undefined;
2658
2661
  name?: string | undefined;
2659
2662
  metadata?: Record<string, unknown> | undefined;
2660
- content?: string | ({
2663
+ content?: string | Record<string, unknown> | (Record<string, unknown> | {
2661
2664
  type: "text";
2662
2665
  text: string;
2663
2666
  } | {
@@ -2675,7 +2678,7 @@ declare const PromptTemplateInputSchema: z.ZodObject<{
2675
2678
  }>, "many">;
2676
2679
  output: z.ZodOptional<z.ZodNullable<z.ZodArray<z.ZodObject<{
2677
2680
  role: z.ZodEnum<["assistant", "user", "system", "tool"]>;
2678
- content: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodArray<z.ZodDiscriminatedUnion<"type", [z.ZodObject<{
2681
+ content: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodArray<z.ZodUnion<[z.ZodDiscriminatedUnion<"type", [z.ZodObject<{
2679
2682
  type: z.ZodLiteral<"text">;
2680
2683
  text: z.ZodString;
2681
2684
  }, "strip", z.ZodTypeAny, {
@@ -2708,7 +2711,7 @@ declare const PromptTemplateInputSchema: z.ZodObject<{
2708
2711
  path: string;
2709
2712
  type: "file";
2710
2713
  media_type: string;
2711
- }>]>, "many">]>>;
2714
+ }>]>, z.ZodRecord<z.ZodString, z.ZodUnknown>]>, "many">, z.ZodRecord<z.ZodString, z.ZodUnknown>]>>;
2712
2715
  toolCalls: z.ZodOptional<z.ZodArray<z.ZodObject<{
2713
2716
  tool: z.ZodString;
2714
2717
  input: z.ZodOptional<z.ZodUnknown>;
@@ -2752,7 +2755,7 @@ declare const PromptTemplateInputSchema: z.ZodObject<{
2752
2755
  }[] | undefined;
2753
2756
  name?: string | undefined;
2754
2757
  metadata?: Record<string, unknown> | undefined;
2755
- content?: string | ({
2758
+ content?: string | Record<string, unknown> | (Record<string, unknown> | {
2756
2759
  type: "text";
2757
2760
  text: string;
2758
2761
  } | {
@@ -2780,7 +2783,7 @@ declare const PromptTemplateInputSchema: z.ZodObject<{
2780
2783
  }[] | undefined;
2781
2784
  name?: string | undefined;
2782
2785
  metadata?: Record<string, unknown> | undefined;
2783
- content?: string | ({
2786
+ content?: string | Record<string, unknown> | (Record<string, unknown> | {
2784
2787
  type: "text";
2785
2788
  text: string;
2786
2789
  } | {
@@ -2801,7 +2804,7 @@ declare const PromptTemplateInputSchema: z.ZodObject<{
2801
2804
  inputFiles: z.ZodArray<z.ZodString, "many">;
2802
2805
  input: z.ZodArray<z.ZodObject<{
2803
2806
  role: z.ZodEnum<["assistant", "user", "system", "tool"]>;
2804
- content: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodArray<z.ZodDiscriminatedUnion<"type", [z.ZodObject<{
2807
+ content: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodArray<z.ZodUnion<[z.ZodDiscriminatedUnion<"type", [z.ZodObject<{
2805
2808
  type: z.ZodLiteral<"text">;
2806
2809
  text: z.ZodString;
2807
2810
  }, "strip", z.ZodTypeAny, {
@@ -2834,7 +2837,7 @@ declare const PromptTemplateInputSchema: z.ZodObject<{
2834
2837
  path: string;
2835
2838
  type: "file";
2836
2839
  media_type: string;
2837
- }>]>, "many">]>>;
2840
+ }>]>, z.ZodRecord<z.ZodString, z.ZodUnknown>]>, "many">, z.ZodRecord<z.ZodString, z.ZodUnknown>]>>;
2838
2841
  toolCalls: z.ZodOptional<z.ZodArray<z.ZodObject<{
2839
2842
  tool: z.ZodString;
2840
2843
  input: z.ZodOptional<z.ZodUnknown>;
@@ -2878,7 +2881,7 @@ declare const PromptTemplateInputSchema: z.ZodObject<{
2878
2881
  }[] | undefined;
2879
2882
  name?: string | undefined;
2880
2883
  metadata?: Record<string, unknown> | undefined;
2881
- content?: string | ({
2884
+ content?: string | Record<string, unknown> | (Record<string, unknown> | {
2882
2885
  type: "text";
2883
2886
  text: string;
2884
2887
  } | {
@@ -2906,7 +2909,7 @@ declare const PromptTemplateInputSchema: z.ZodObject<{
2906
2909
  }[] | undefined;
2907
2910
  name?: string | undefined;
2908
2911
  metadata?: Record<string, unknown> | undefined;
2909
- content?: string | ({
2912
+ content?: string | Record<string, unknown> | (Record<string, unknown> | {
2910
2913
  type: "text";
2911
2914
  text: string;
2912
2915
  } | {
@@ -2979,7 +2982,7 @@ declare const PromptTemplateInputSchema: z.ZodObject<{
2979
2982
  }[] | undefined;
2980
2983
  name?: string | undefined;
2981
2984
  metadata?: Record<string, unknown> | undefined;
2982
- content?: string | ({
2985
+ content?: string | Record<string, unknown> | (Record<string, unknown> | {
2983
2986
  type: "text";
2984
2987
  text: string;
2985
2988
  } | {
@@ -3009,7 +3012,7 @@ declare const PromptTemplateInputSchema: z.ZodObject<{
3009
3012
  }[] | undefined;
3010
3013
  name?: string | undefined;
3011
3014
  metadata?: Record<string, unknown> | undefined;
3012
- content?: string | ({
3015
+ content?: string | Record<string, unknown> | (Record<string, unknown> | {
3013
3016
  type: "text";
3014
3017
  text: string;
3015
3018
  } | {
@@ -3039,7 +3042,7 @@ declare const PromptTemplateInputSchema: z.ZodObject<{
3039
3042
  }[] | undefined;
3040
3043
  name?: string | undefined;
3041
3044
  metadata?: Record<string, unknown> | undefined;
3042
- content?: string | ({
3045
+ content?: string | Record<string, unknown> | (Record<string, unknown> | {
3043
3046
  type: "text";
3044
3047
  text: string;
3045
3048
  } | {
@@ -3091,7 +3094,7 @@ declare const PromptTemplateInputSchema: z.ZodObject<{
3091
3094
  }[] | undefined;
3092
3095
  name?: string | undefined;
3093
3096
  metadata?: Record<string, unknown> | undefined;
3094
- content?: string | ({
3097
+ content?: string | Record<string, unknown> | (Record<string, unknown> | {
3095
3098
  type: "text";
3096
3099
  text: string;
3097
3100
  } | {
@@ -3121,7 +3124,7 @@ declare const PromptTemplateInputSchema: z.ZodObject<{
3121
3124
  }[] | undefined;
3122
3125
  name?: string | undefined;
3123
3126
  metadata?: Record<string, unknown> | undefined;
3124
- content?: string | ({
3127
+ content?: string | Record<string, unknown> | (Record<string, unknown> | {
3125
3128
  type: "text";
3126
3129
  text: string;
3127
3130
  } | {
@@ -3151,7 +3154,7 @@ declare const PromptTemplateInputSchema: z.ZodObject<{
3151
3154
  }[] | undefined;
3152
3155
  name?: string | undefined;
3153
3156
  metadata?: Record<string, unknown> | undefined;
3154
- content?: string | ({
3157
+ content?: string | Record<string, unknown> | (Record<string, unknown> | {
3155
3158
  type: "text";
3156
3159
  text: string;
3157
3160
  } | {