bem-ai-sdk 0.26.0 → 0.27.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (174) hide show
  1. package/CHANGELOG.md +14 -0
  2. package/client.d.mts +127 -2
  3. package/client.d.mts.map +1 -1
  4. package/client.d.ts +127 -2
  5. package/client.d.ts.map +1 -1
  6. package/client.js +136 -3
  7. package/client.js.map +1 -1
  8. package/client.mjs +136 -3
  9. package/client.mjs.map +1 -1
  10. package/package.json +1 -1
  11. package/resources/buckets.d.mts +272 -0
  12. package/resources/buckets.d.mts.map +1 -0
  13. package/resources/buckets.d.ts +272 -0
  14. package/resources/buckets.d.ts.map +1 -0
  15. package/resources/buckets.js +93 -0
  16. package/resources/buckets.js.map +1 -0
  17. package/resources/buckets.mjs +89 -0
  18. package/resources/buckets.mjs.map +1 -0
  19. package/resources/entities/entities.d.mts +516 -0
  20. package/resources/entities/entities.d.mts.map +1 -0
  21. package/resources/entities/entities.d.ts +516 -0
  22. package/resources/entities/entities.d.ts.map +1 -0
  23. package/resources/entities/entities.js +93 -0
  24. package/resources/entities/entities.js.map +1 -0
  25. package/resources/entities/entities.mjs +88 -0
  26. package/resources/entities/entities.mjs.map +1 -0
  27. package/resources/entities/index.d.mts +3 -0
  28. package/resources/entities/index.d.mts.map +1 -0
  29. package/resources/entities/index.d.ts +3 -0
  30. package/resources/entities/index.d.ts.map +1 -0
  31. package/resources/entities/index.js +9 -0
  32. package/resources/entities/index.js.map +1 -0
  33. package/resources/entities/index.mjs +4 -0
  34. package/resources/entities/index.mjs.map +1 -0
  35. package/resources/entities/synonyms.d.mts +109 -0
  36. package/resources/entities/synonyms.d.mts.map +1 -0
  37. package/resources/entities/synonyms.d.ts +109 -0
  38. package/resources/entities/synonyms.d.ts.map +1 -0
  39. package/resources/entities/synonyms.js +61 -0
  40. package/resources/entities/synonyms.js.map +1 -0
  41. package/resources/entities/synonyms.mjs +57 -0
  42. package/resources/entities/synonyms.mjs.map +1 -0
  43. package/resources/entities.d.mts +2 -0
  44. package/resources/entities.d.mts.map +1 -0
  45. package/resources/entities.d.ts +2 -0
  46. package/resources/entities.d.ts.map +1 -0
  47. package/resources/entities.js +6 -0
  48. package/resources/entities.js.map +1 -0
  49. package/resources/entities.mjs +3 -0
  50. package/resources/entities.mjs.map +1 -0
  51. package/resources/entity-types/entity-types.d.mts +302 -0
  52. package/resources/entity-types/entity-types.d.mts.map +1 -0
  53. package/resources/entity-types/entity-types.d.ts +302 -0
  54. package/resources/entity-types/entity-types.d.ts.map +1 -0
  55. package/resources/entity-types/entity-types.js +104 -0
  56. package/resources/entity-types/entity-types.js.map +1 -0
  57. package/resources/entity-types/entity-types.mjs +99 -0
  58. package/resources/entity-types/entity-types.mjs.map +1 -0
  59. package/resources/entity-types/index.d.mts +3 -0
  60. package/resources/entity-types/index.d.mts.map +1 -0
  61. package/resources/entity-types/index.d.ts +3 -0
  62. package/resources/entity-types/index.d.ts.map +1 -0
  63. package/resources/entity-types/index.js +9 -0
  64. package/resources/entity-types/index.js.map +1 -0
  65. package/resources/entity-types/index.mjs +4 -0
  66. package/resources/entity-types/index.mjs.map +1 -0
  67. package/resources/entity-types/reviewers.d.mts +136 -0
  68. package/resources/entity-types/reviewers.d.mts.map +1 -0
  69. package/resources/entity-types/reviewers.d.ts +136 -0
  70. package/resources/entity-types/reviewers.d.ts.map +1 -0
  71. package/resources/entity-types/reviewers.js +74 -0
  72. package/resources/entity-types/reviewers.js.map +1 -0
  73. package/resources/entity-types/reviewers.mjs +70 -0
  74. package/resources/entity-types/reviewers.mjs.map +1 -0
  75. package/resources/entity-types.d.mts +2 -0
  76. package/resources/entity-types.d.mts.map +1 -0
  77. package/resources/entity-types.d.ts +2 -0
  78. package/resources/entity-types.d.ts.map +1 -0
  79. package/resources/entity-types.js +6 -0
  80. package/resources/entity-types.js.map +1 -0
  81. package/resources/entity-types.mjs +3 -0
  82. package/resources/entity-types.mjs.map +1 -0
  83. package/resources/eval/eval.d.mts +4 -0
  84. package/resources/eval/eval.d.mts.map +1 -1
  85. package/resources/eval/eval.d.ts +4 -0
  86. package/resources/eval/eval.d.ts.map +1 -1
  87. package/resources/eval/eval.js +4 -0
  88. package/resources/eval/eval.js.map +1 -1
  89. package/resources/eval/eval.mjs +4 -0
  90. package/resources/eval/eval.mjs.map +1 -1
  91. package/resources/eval/index.d.mts +1 -0
  92. package/resources/eval/index.d.mts.map +1 -1
  93. package/resources/eval/index.d.ts +1 -0
  94. package/resources/eval/index.d.ts.map +1 -1
  95. package/resources/eval/index.js +3 -1
  96. package/resources/eval/index.js.map +1 -1
  97. package/resources/eval/index.mjs +1 -0
  98. package/resources/eval/index.mjs.map +1 -1
  99. package/resources/eval/score.d.mts +486 -0
  100. package/resources/eval/score.d.mts.map +1 -0
  101. package/resources/eval/score.d.ts +486 -0
  102. package/resources/eval/score.d.ts.map +1 -0
  103. package/resources/eval/score.js +126 -0
  104. package/resources/eval/score.js.map +1 -0
  105. package/resources/eval/score.mjs +122 -0
  106. package/resources/eval/score.mjs.map +1 -0
  107. package/resources/fs.d.mts +28 -0
  108. package/resources/fs.d.mts.map +1 -1
  109. package/resources/fs.d.ts +28 -0
  110. package/resources/fs.d.ts.map +1 -1
  111. package/resources/index.d.mts +6 -0
  112. package/resources/index.d.mts.map +1 -1
  113. package/resources/index.d.ts +6 -0
  114. package/resources/index.d.ts.map +1 -1
  115. package/resources/index.js +13 -1
  116. package/resources/index.js.map +1 -1
  117. package/resources/index.mjs +6 -0
  118. package/resources/index.mjs.map +1 -1
  119. package/resources/knowledge-graph.d.mts +125 -0
  120. package/resources/knowledge-graph.d.mts.map +1 -0
  121. package/resources/knowledge-graph.d.ts +125 -0
  122. package/resources/knowledge-graph.d.ts.map +1 -0
  123. package/resources/knowledge-graph.js +34 -0
  124. package/resources/knowledge-graph.js.map +1 -0
  125. package/resources/knowledge-graph.mjs +30 -0
  126. package/resources/knowledge-graph.mjs.map +1 -0
  127. package/resources/review-queue.d.mts +192 -0
  128. package/resources/review-queue.d.mts.map +1 -0
  129. package/resources/review-queue.d.ts +192 -0
  130. package/resources/review-queue.d.ts.map +1 -0
  131. package/resources/review-queue.js +49 -0
  132. package/resources/review-queue.js.map +1 -0
  133. package/resources/review-queue.mjs +45 -0
  134. package/resources/review-queue.mjs.map +1 -0
  135. package/resources/subscriptions.d.mts +14 -2
  136. package/resources/subscriptions.d.mts.map +1 -1
  137. package/resources/subscriptions.d.ts +14 -2
  138. package/resources/subscriptions.d.ts.map +1 -1
  139. package/resources/subscriptions.js +14 -2
  140. package/resources/subscriptions.js.map +1 -1
  141. package/resources/subscriptions.mjs +14 -2
  142. package/resources/subscriptions.mjs.map +1 -1
  143. package/resources/users.d.mts +62 -0
  144. package/resources/users.d.mts.map +1 -0
  145. package/resources/users.d.ts +62 -0
  146. package/resources/users.d.ts.map +1 -0
  147. package/resources/users.js +35 -0
  148. package/resources/users.js.map +1 -0
  149. package/resources/users.mjs +31 -0
  150. package/resources/users.mjs.map +1 -0
  151. package/src/client.ts +220 -3
  152. package/src/resources/buckets.ts +351 -0
  153. package/src/resources/entities/entities.ts +639 -0
  154. package/src/resources/entities/index.ts +20 -0
  155. package/src/resources/entities/synonyms.ts +140 -0
  156. package/src/resources/entities.ts +3 -0
  157. package/src/resources/entity-types/entity-types.ts +396 -0
  158. package/src/resources/entity-types/index.ts +19 -0
  159. package/src/resources/entity-types/reviewers.ts +175 -0
  160. package/src/resources/entity-types.ts +3 -0
  161. package/src/resources/eval/eval.ts +18 -0
  162. package/src/resources/eval/index.ts +7 -0
  163. package/src/resources/eval/score.ts +590 -0
  164. package/src/resources/fs.ts +30 -0
  165. package/src/resources/index.ts +40 -0
  166. package/src/resources/knowledge-graph.ts +153 -0
  167. package/src/resources/review-queue.ts +234 -0
  168. package/src/resources/subscriptions.ts +14 -2
  169. package/src/resources/users.ts +76 -0
  170. package/src/version.ts +1 -1
  171. package/version.d.mts +1 -1
  172. package/version.d.ts +1 -1
  173. package/version.js +1 -1
  174. package/version.mjs +1 -1
@@ -0,0 +1,486 @@
1
+ import { APIResource } from "../../core/resource.mjs";
2
+ import * as OutputsAPI from "../outputs.mjs";
3
+ import { APIPromise } from "../../core/api-promise.mjs";
4
+ import { RequestOptions } from "../../internal/request-options.mjs";
5
+ /**
6
+ * Monitor, evaluate, and iterate on the quality of every function in your
7
+ * environment. Function Accuracy bundles two complementary loops:
8
+ *
9
+ * ## Evaluations (`/v3/eval`)
10
+ *
11
+ * Trigger and retrieve per-transformation evaluations. Evaluations run
12
+ * asynchronously and score each transformation's output against the
13
+ * function's schema for confidence, per-field hallucination detection,
14
+ * and relevance. Supported for `extract`, `transform`, `analyze`, and
15
+ * `join` events.
16
+ *
17
+ * 1. **Trigger** — `POST /v3/eval` queues jobs for a batch of transformation IDs.
18
+ * 2. **Poll** — `GET /v3/eval/results` returns the current state of each
19
+ * requested ID, partitioned into `results`, `pending`, and `failed`.
20
+ * Accepts either `eventIDs` (preferred) or `transformationIDs` as a
21
+ * comma-separated query parameter, and always keys the response by
22
+ * event KSUID.
23
+ *
24
+ * Up to 100 IDs may be submitted per request.
25
+ *
26
+ * ## Metrics, review, regression (`/v3/functions/{metrics,review,regression,compare}`)
27
+ *
28
+ * Roll evaluation results and user corrections up into actionable
29
+ * function-level signal:
30
+ *
31
+ * - **`GET /v3/functions/metrics`** — aggregate accuracy, precision,
32
+ * recall, F1, and confusion-matrix counts per function.
33
+ * - **`POST /v3/functions/review`** — sample-size estimation,
34
+ * confidence-bucketed distribution, PR-AUC, and per-threshold
35
+ * confidence intervals (Wald or Wilson) for picking review cutoffs.
36
+ * - **`POST /v3/functions/regression`** — replay corrected historical
37
+ * inputs against a new function version, producing a labeled
38
+ * regression dataset.
39
+ * - **`POST /v3/functions/regression/corrections`** — propagate
40
+ * baseline corrections onto the regression dataset so it can be
41
+ * scored.
42
+ * - **`POST /v3/functions/compare`** — compute aggregate and
43
+ * field-level lift between any two versions, optionally scoped to
44
+ * the regression dataset.
45
+ *
46
+ * All five endpoints support `extract` end-to-end on both the vision
47
+ * and OCR paths, alongside the legacy `transform` / `analyze` / `join`
48
+ * types.
49
+ */
50
+ export declare class Score extends APIResource {
51
+ /**
52
+ * **Score a function against a list of (input, expected) pairs.**
53
+ *
54
+ * Submits a batch of `(input, expected)` pairs, runs the named function over each
55
+ * input, and returns per-pair + aggregate accuracy metrics comparing the
56
+ * function's actual output to the provided expected JSON.
57
+ *
58
+ * Scoring runs asynchronously. The response carries a `scoreRunID`; poll
59
+ * `GET /v3/eval/score/{scoreRunID}` until `status` is one of `completed`, `error`,
60
+ * or `cancelled`.
61
+ *
62
+ * `matchConfig` controls comparator behavior:
63
+ *
64
+ * - `numericTolerance`: relative tolerance for numeric fields (0 = exact)
65
+ * - `stringMatch`: `exact` (default) or `fuzzy` (Levenshtein ratio)
66
+ * - `arrayMatch`: `by-index` (default; only mode in P0)
67
+ * - `ignorePaths`: JSON Pointer paths to skip, supports `*` wildcards
68
+ *
69
+ * @example
70
+ * ```ts
71
+ * const score = await client.eval.score.create({
72
+ * functionName: 'functionName',
73
+ * pairs: [
74
+ * {
75
+ * expected: {},
76
+ * input: {
77
+ * inputContent: 'inputContent',
78
+ * inputType: 'csv',
79
+ * },
80
+ * },
81
+ * ],
82
+ * });
83
+ * ```
84
+ */
85
+ create(body: ScoreCreateParams, options?: RequestOptions): APIPromise<ScoreCreateResponse>;
86
+ /**
87
+ * **Get the status and per-pair results of a score run.**
88
+ *
89
+ * Returns `aggregate` only once `status` reaches `completed`. `perPair` is
90
+ * populated incrementally — each pair's `fieldResults` appears as its underlying
91
+ * function call terminates.
92
+ *
93
+ * @example
94
+ * ```ts
95
+ * const score = await client.eval.score.retrieve(
96
+ * 'scoreRunID',
97
+ * );
98
+ * ```
99
+ */
100
+ retrieve(scoreRunID: string, options?: RequestOptions): APIPromise<ScoreRetrieveResponse>;
101
+ /**
102
+ * **Cancel an in-flight score run.**
103
+ *
104
+ * Transitions the run to `cancelled`. Function calls already in flight are allowed
105
+ * to finish (best-effort cancellation via the job queue); results from completed
106
+ * pairs may still appear in subsequent GETs.
107
+ *
108
+ * @example
109
+ * ```ts
110
+ * const response = await client.eval.score.cancel(
111
+ * 'scoreRunID',
112
+ * );
113
+ * ```
114
+ */
115
+ cancel(scoreRunID: string, options?: RequestOptions): APIPromise<ScoreCancelResponse>;
116
+ }
117
+ /**
118
+ * Returned by `POST /v3/eval/score`.
119
+ */
120
+ export interface ScoreCreateResponse {
121
+ /**
122
+ * Run identifier. Use with `GET /v3/eval/score/{scoreRunID}`.
123
+ */
124
+ scoreRunID: string;
125
+ /**
126
+ * Status values for an eval-score run.
127
+ */
128
+ status: 'pending' | 'initializing' | 'running' | 'completed' | 'error' | 'cancelled';
129
+ }
130
+ /**
131
+ * Full status payload returned by `GET /v3/eval/score/{scoreRunID}`.
132
+ */
133
+ export interface ScoreRetrieveResponse {
134
+ functionName: string;
135
+ functionVersionNum: number;
136
+ /**
137
+ * Comparator configuration. All fields optional; conservative defaults.
138
+ */
139
+ matchConfig: ScoreRetrieveResponse.MatchConfig;
140
+ /**
141
+ * Per-pair results. `fieldResults` appears once a pair has been compared.
142
+ */
143
+ perPair: Array<ScoreRetrieveResponse.PerPair>;
144
+ /**
145
+ * Counts across all pairs.
146
+ */
147
+ progress: ScoreRetrieveResponse.Progress;
148
+ scoreRunID: string;
149
+ /**
150
+ * Status values for an eval-score run.
151
+ */
152
+ status: 'pending' | 'initializing' | 'running' | 'completed' | 'error' | 'cancelled';
153
+ /**
154
+ * Aggregate accuracy metrics.
155
+ */
156
+ aggregate?: ScoreRetrieveResponse.Aggregate;
157
+ }
158
+ export declare namespace ScoreRetrieveResponse {
159
+ /**
160
+ * Comparator configuration. All fields optional; conservative defaults.
161
+ */
162
+ interface MatchConfig {
163
+ /**
164
+ * P0 supports only `by-index`.
165
+ */
166
+ arrayMatch?: 'by-index';
167
+ /**
168
+ * Levenshtein-ratio threshold used when `stringMatch == "fuzzy"`. Range `[0, 1]`.
169
+ * Default `0.85`.
170
+ */
171
+ fuzzyThreshold?: number;
172
+ /**
173
+ * JSON Pointer paths to skip during comparison. The asterisk character matches
174
+ * arbitrary object keys / array indices.
175
+ *
176
+ * Example values: /metadata, /lineItems with asterisk segment, etc.
177
+ */
178
+ ignorePaths?: Array<string>;
179
+ /**
180
+ * Relative tolerance for numeric fields. `0` (default) means exact equality;
181
+ * `0.01` means ±1%.
182
+ */
183
+ numericTolerance?: number;
184
+ /**
185
+ * `exact` (default) or `fuzzy`.
186
+ */
187
+ stringMatch?: 'exact' | 'fuzzy';
188
+ }
189
+ /**
190
+ * Per-pair result.
191
+ */
192
+ interface PerPair {
193
+ pairIndex: number;
194
+ /**
195
+ * Per-pair status.
196
+ */
197
+ status: 'pending' | 'running' | 'completed' | 'failed';
198
+ /**
199
+ * The function call that produced the actual output, if any.
200
+ */
201
+ callID?: string;
202
+ /**
203
+ * Error message if the underlying function call failed.
204
+ */
205
+ errorMessage?: string;
206
+ /**
207
+ * Per-leaf comparator output. Present only after the pair has been compared.
208
+ */
209
+ fieldResults?: Array<PerPair.FieldResult>;
210
+ }
211
+ namespace PerPair {
212
+ /**
213
+ * One leaf in `expected ∪ actual`.
214
+ */
215
+ interface FieldResult {
216
+ /**
217
+ * Classification:
218
+ *
219
+ * - `exact`: both present and deep-equal
220
+ * - `within_tolerance`: both numbers, within configured tolerance
221
+ * - `fuzzy_match`: both strings, Levenshtein ratio above threshold
222
+ * - `miss`: expected present, actual absent or different
223
+ * - `extra`: actual present, expected absent
224
+ */
225
+ match: 'exact' | 'within_tolerance' | 'fuzzy_match' | 'miss' | 'extra';
226
+ /**
227
+ * JSON Pointer to the leaf.
228
+ */
229
+ path: string;
230
+ actual?: unknown;
231
+ /**
232
+ * Populated for numeric comparisons; `actual - expected`.
233
+ */
234
+ delta?: number;
235
+ expected?: unknown;
236
+ }
237
+ }
238
+ /**
239
+ * Counts across all pairs.
240
+ */
241
+ interface Progress {
242
+ completed: number;
243
+ failed: number;
244
+ total: number;
245
+ }
246
+ /**
247
+ * Aggregate accuracy metrics.
248
+ */
249
+ interface Aggregate {
250
+ exactMatches: number;
251
+ extras: number;
252
+ f1: number;
253
+ fuzzyMatches: number;
254
+ misses: number;
255
+ precision: number;
256
+ recall: number;
257
+ totalFieldsActual: number;
258
+ totalFieldsExpected: number;
259
+ withinTolerance: number;
260
+ }
261
+ }
262
+ /**
263
+ * Full status payload returned by `GET /v3/eval/score/{scoreRunID}`.
264
+ */
265
+ export interface ScoreCancelResponse {
266
+ functionName: string;
267
+ functionVersionNum: number;
268
+ /**
269
+ * Comparator configuration. All fields optional; conservative defaults.
270
+ */
271
+ matchConfig: ScoreCancelResponse.MatchConfig;
272
+ /**
273
+ * Per-pair results. `fieldResults` appears once a pair has been compared.
274
+ */
275
+ perPair: Array<ScoreCancelResponse.PerPair>;
276
+ /**
277
+ * Counts across all pairs.
278
+ */
279
+ progress: ScoreCancelResponse.Progress;
280
+ scoreRunID: string;
281
+ /**
282
+ * Status values for an eval-score run.
283
+ */
284
+ status: 'pending' | 'initializing' | 'running' | 'completed' | 'error' | 'cancelled';
285
+ /**
286
+ * Aggregate accuracy metrics.
287
+ */
288
+ aggregate?: ScoreCancelResponse.Aggregate;
289
+ }
290
+ export declare namespace ScoreCancelResponse {
291
+ /**
292
+ * Comparator configuration. All fields optional; conservative defaults.
293
+ */
294
+ interface MatchConfig {
295
+ /**
296
+ * P0 supports only `by-index`.
297
+ */
298
+ arrayMatch?: 'by-index';
299
+ /**
300
+ * Levenshtein-ratio threshold used when `stringMatch == "fuzzy"`. Range `[0, 1]`.
301
+ * Default `0.85`.
302
+ */
303
+ fuzzyThreshold?: number;
304
+ /**
305
+ * JSON Pointer paths to skip during comparison. The asterisk character matches
306
+ * arbitrary object keys / array indices.
307
+ *
308
+ * Example values: /metadata, /lineItems with asterisk segment, etc.
309
+ */
310
+ ignorePaths?: Array<string>;
311
+ /**
312
+ * Relative tolerance for numeric fields. `0` (default) means exact equality;
313
+ * `0.01` means ±1%.
314
+ */
315
+ numericTolerance?: number;
316
+ /**
317
+ * `exact` (default) or `fuzzy`.
318
+ */
319
+ stringMatch?: 'exact' | 'fuzzy';
320
+ }
321
+ /**
322
+ * Per-pair result.
323
+ */
324
+ interface PerPair {
325
+ pairIndex: number;
326
+ /**
327
+ * Per-pair status.
328
+ */
329
+ status: 'pending' | 'running' | 'completed' | 'failed';
330
+ /**
331
+ * The function call that produced the actual output, if any.
332
+ */
333
+ callID?: string;
334
+ /**
335
+ * Error message if the underlying function call failed.
336
+ */
337
+ errorMessage?: string;
338
+ /**
339
+ * Per-leaf comparator output. Present only after the pair has been compared.
340
+ */
341
+ fieldResults?: Array<PerPair.FieldResult>;
342
+ }
343
+ namespace PerPair {
344
+ /**
345
+ * One leaf in `expected ∪ actual`.
346
+ */
347
+ interface FieldResult {
348
+ /**
349
+ * Classification:
350
+ *
351
+ * - `exact`: both present and deep-equal
352
+ * - `within_tolerance`: both numbers, within configured tolerance
353
+ * - `fuzzy_match`: both strings, Levenshtein ratio above threshold
354
+ * - `miss`: expected present, actual absent or different
355
+ * - `extra`: actual present, expected absent
356
+ */
357
+ match: 'exact' | 'within_tolerance' | 'fuzzy_match' | 'miss' | 'extra';
358
+ /**
359
+ * JSON Pointer to the leaf.
360
+ */
361
+ path: string;
362
+ actual?: unknown;
363
+ /**
364
+ * Populated for numeric comparisons; `actual - expected`.
365
+ */
366
+ delta?: number;
367
+ expected?: unknown;
368
+ }
369
+ }
370
+ /**
371
+ * Counts across all pairs.
372
+ */
373
+ interface Progress {
374
+ completed: number;
375
+ failed: number;
376
+ total: number;
377
+ }
378
+ /**
379
+ * Aggregate accuracy metrics.
380
+ */
381
+ interface Aggregate {
382
+ exactMatches: number;
383
+ extras: number;
384
+ f1: number;
385
+ fuzzyMatches: number;
386
+ misses: number;
387
+ precision: number;
388
+ recall: number;
389
+ totalFieldsActual: number;
390
+ totalFieldsExpected: number;
391
+ withinTolerance: number;
392
+ }
393
+ }
394
+ export interface ScoreCreateParams {
395
+ /**
396
+ * Name of the function to score. Must be of type extract, transform, or analyze.
397
+ */
398
+ functionName: string;
399
+ /**
400
+ * Up to 1000 pairs per request.
401
+ */
402
+ pairs: Array<ScoreCreateParams.Pair>;
403
+ /**
404
+ * Optional version number to score against. P0: only the function's current
405
+ * version is accepted; passing a different version returns 422.
406
+ */
407
+ functionVersionNum?: number;
408
+ /**
409
+ * Comparator configuration. All fields optional; conservative defaults.
410
+ */
411
+ matchConfig?: ScoreCreateParams.MatchConfig;
412
+ }
413
+ export declare namespace ScoreCreateParams {
414
+ /**
415
+ * One `(input, expected)` pair.
416
+ */
417
+ interface Pair {
418
+ /**
419
+ * Expected output for this input, as a JSON value. The comparator walks
420
+ * `expected ∪ actual` and produces a per-leaf classification.
421
+ */
422
+ expected: unknown;
423
+ /**
424
+ * A single file input with base64-encoded content.
425
+ *
426
+ * When using the Bem CLI, use `@path/to/file` in the `inputContent` field to
427
+ * automatically read and base64-encode the file:
428
+ * `--input.single-file '{"inputContent": "@file.pdf", "inputType": "pdf"}' --wait`
429
+ */
430
+ input: Pair.Input;
431
+ }
432
+ namespace Pair {
433
+ /**
434
+ * A single file input with base64-encoded content.
435
+ *
436
+ * When using the Bem CLI, use `@path/to/file` in the `inputContent` field to
437
+ * automatically read and base64-encode the file:
438
+ * `--input.single-file '{"inputContent": "@file.pdf", "inputType": "pdf"}' --wait`
439
+ */
440
+ interface Input {
441
+ /**
442
+ * Base64-encoded file content. In the Bem CLI, use `@path/to/file` to embed file
443
+ * contents automatically.
444
+ */
445
+ inputContent: string;
446
+ /**
447
+ * The input type of the content you're sending for transformation.
448
+ */
449
+ inputType: OutputsAPI.InputType;
450
+ }
451
+ }
452
+ /**
453
+ * Comparator configuration. All fields optional; conservative defaults.
454
+ */
455
+ interface MatchConfig {
456
+ /**
457
+ * P0 supports only `by-index`.
458
+ */
459
+ arrayMatch?: 'by-index';
460
+ /**
461
+ * Levenshtein-ratio threshold used when `stringMatch == "fuzzy"`. Range `[0, 1]`.
462
+ * Default `0.85`.
463
+ */
464
+ fuzzyThreshold?: number;
465
+ /**
466
+ * JSON Pointer paths to skip during comparison. The asterisk character matches
467
+ * arbitrary object keys / array indices.
468
+ *
469
+ * Example values: /metadata, /lineItems with asterisk segment, etc.
470
+ */
471
+ ignorePaths?: Array<string>;
472
+ /**
473
+ * Relative tolerance for numeric fields. `0` (default) means exact equality;
474
+ * `0.01` means ±1%.
475
+ */
476
+ numericTolerance?: number;
477
+ /**
478
+ * `exact` (default) or `fuzzy`.
479
+ */
480
+ stringMatch?: 'exact' | 'fuzzy';
481
+ }
482
+ }
483
+ export declare namespace Score {
484
+ export { type ScoreCreateResponse as ScoreCreateResponse, type ScoreRetrieveResponse as ScoreRetrieveResponse, type ScoreCancelResponse as ScoreCancelResponse, type ScoreCreateParams as ScoreCreateParams, };
485
+ }
486
+ //# sourceMappingURL=score.d.mts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"score.d.mts","sourceRoot":"","sources":["../../src/resources/eval/score.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,WAAW,EAAE,gCAA4B;AAClD,OAAO,KAAK,UAAU,uBAAmB;AACzC,OAAO,EAAE,UAAU,EAAE,mCAA+B;AACpD,OAAO,EAAE,cAAc,EAAE,2CAAuC;AAGhE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4CG;AACH,qBAAa,KAAM,SAAQ,WAAW;IACpC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAiCG;IACH,MAAM,CAAC,IAAI,EAAE,iBAAiB,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,UAAU,CAAC,mBAAmB,CAAC;IAI1F;;;;;;;;;;;;;OAaG;IACH,QAAQ,CAAC,UAAU,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,UAAU,CAAC,qBAAqB,CAAC;IAIzF;;;;;;;;;;;;;OAaG;IACH,MAAM,CAAC,UAAU,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,UAAU,CAAC,mBAAmB,CAAC;CAGtF;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC;;OAEG;IACH,UAAU,EAAE,MAAM,CAAC;IAEnB;;OAEG;IACH,MAAM,EAAE,SAAS,GAAG,cAAc,GAAG,SAAS,GAAG,WAAW,GAAG,OAAO,GAAG,WAAW,CAAC;CACtF;AAED;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC,YAAY,EAAE,MAAM,CAAC;IAErB,kBAAkB,EAAE,MAAM,CAAC;IAE3B;;OAEG;IACH,WAAW,EAAE,qBAAqB,CAAC,WAAW,CAAC;IAE/C;;OAEG;IACH,OAAO,EAAE,KAAK,CAAC,qBAAqB,CAAC,OAAO,CAAC,CAAC;IAE9C;;OAEG;IACH,QAAQ,EAAE,qBAAqB,CAAC,QAAQ,CAAC;IAEzC,UAAU,EAAE,MAAM,CAAC;IAEnB;;OAEG;IACH,MAAM,EAAE,SAAS,GAAG,cAAc,GAAG,SAAS,GAAG,WAAW,GAAG,OAAO,GAAG,WAAW,CAAC;IAErF;;OAEG;IACH,SAAS,CAAC,EAAE,qBAAqB,CAAC,SAAS,CAAC;CAC7C;AAED,yBAAiB,qBAAqB,CAAC;IACrC;;OAEG;IACH,UAAiB,WAAW;QAC1B;;WAEG;QACH,UAAU,CAAC,EAAE,UAAU,CAAC;QAExB;;;WAGG;QACH,cAAc,CAAC,EAAE,MAAM,CAAC;QAExB;;;;;WAKG;QACH,WAAW,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;QAE5B;;;WAGG;QACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;QAE1B;;WAEG;QACH,WAAW,CAAC,EAAE,OAAO,GAAG,OAAO,CAAC;KACjC;IAED;;OAEG;IACH,UAAiB,OAAO;QACtB,SAAS,EAAE,MAAM,CAAC;QAElB;;WAEG;QACH,MAAM,EAAE,SAAS,GAAG,SAAS,GAAG,WAAW,GAAG,QAAQ,CAAC;QAEvD;;WAEG;QACH,MAAM,CAAC,EAAE,MAAM,CAAC;QAEhB;;WAEG;QACH,YAAY,CAAC,EAAE,MAAM,CAAC;QAEtB;;WAEG;QACH,YAAY,CAAC,EAAE,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;KAC3C;IAED,UAAiB,OAAO,CAAC;QACvB;;WAEG;QACH,UAAiB,WAAW;YAC1B;;;;;;;;eAQG;YACH,KAAK,EAAE,OAAO,GAAG,kBAAkB,GAAG,aAAa,GAAG,MAAM,GAAG,OAAO,CAAC;YAEvE;;eAEG;YACH,IAAI,EAAE,MAAM,CAAC;YAEb,MAAM,CAAC,EAAE,OAAO,CAAC;YAEjB;;eAEG;YACH,KAAK,CAAC,EAAE,MAAM,CAAC;YAEf,QAAQ,CAAC,EAAE,OAAO,CAAC;SACpB;KACF;IAED;;OAEG;IACH,UAAiB,QAAQ;QACvB,SAAS,EAAE,MAAM,CAAC;QAElB,MAAM,EAAE,MAAM,CAAC;QAEf,KAAK,EAAE,MAAM,CAAC;KACf;IAED;;OAEG;IACH,UAAiB,SAAS;QACxB,YAAY,EAAE,MAAM,CAAC;QAErB,MAAM,EAAE,MAAM,CAAC;QAEf,EAAE,EAAE,MAAM,CAAC;QAEX,YAAY,EAAE,MAAM,CAAC;QAErB,MAAM,EAAE,MAAM,CAAC;QAEf,SAAS,EAAE,MAAM,CAAC;QAElB,MAAM,EAAE,MAAM,CAAC;QAEf,iBAAiB,EAAE,MAAM,CAAC;QAE1B,mBAAmB,EAAE,MAAM,CAAC;QAE5B,eAAe,EAAE,MAAM,CAAC;KACzB;CACF;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,YAAY,EAAE,MAAM,CAAC;IAErB,kBAAkB,EAAE,MAAM,CAAC;IAE3B;;OAEG;IACH,WAAW,EAAE,mBAAmB,CAAC,WAAW,CAAC;IAE7C;;OAEG;IACH,OAAO,EAAE,KAAK,CAAC,mBAAmB,CAAC,OAAO,CAAC,CAAC;IAE5C;;OAEG;IACH,QAAQ,EAAE,mBAAmB,CAAC,QAAQ,CAAC;IAEvC,UAAU,EAAE,MAAM,CAAC;IAEnB;;OAEG;IACH,MAAM,EAAE,SAAS,GAAG,cAAc,GAAG,SAAS,GAAG,WAAW,GAAG,OAAO,GAAG,WAAW,CAAC;IAErF;;OAEG;IACH,SAAS,CAAC,EAAE,mBAAmB,CAAC,SAAS,CAAC;CAC3C;AAED,yBAAiB,mBAAmB,CAAC;IACnC;;OAEG;IACH,UAAiB,WAAW;QAC1B;;WAEG;QACH,UAAU,CAAC,EAAE,UAAU,CAAC;QAExB;;;WAGG;QACH,cAAc,CAAC,EAAE,MAAM,CAAC;QAExB;;;;;WAKG;QACH,WAAW,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;QAE5B;;;WAGG;QACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;QAE1B;;WAEG;QACH,WAAW,CAAC,EAAE,OAAO,GAAG,OAAO,CAAC;KACjC;IAED;;OAEG;IACH,UAAiB,OAAO;QACtB,SAAS,EAAE,MAAM,CAAC;QAElB;;WAEG;QACH,MAAM,EAAE,SAAS,GAAG,SAAS,GAAG,WAAW,GAAG,QAAQ,CAAC;QAEvD;;WAEG;QACH,MAAM,CAAC,EAAE,MAAM,CAAC;QAEhB;;WAEG;QACH,YAAY,CAAC,EAAE,MAAM,CAAC;QAEtB;;WAEG;QACH,YAAY,CAAC,EAAE,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;KAC3C;IAED,UAAiB,OAAO,CAAC;QACvB;;WAEG;QACH,UAAiB,WAAW;YAC1B;;;;;;;;eAQG;YACH,KAAK,EAAE,OAAO,GAAG,kBAAkB,GAAG,aAAa,GAAG,MAAM,GAAG,OAAO,CAAC;YAEvE;;eAEG;YACH,IAAI,EAAE,MAAM,CAAC;YAEb,MAAM,CAAC,EAAE,OAAO,CAAC;YAEjB;;eAEG;YACH,KAAK,CAAC,EAAE,MAAM,CAAC;YAEf,QAAQ,CAAC,EAAE,OAAO,CAAC;SACpB;KACF;IAED;;OAEG;IACH,UAAiB,QAAQ;QACvB,SAAS,EAAE,MAAM,CAAC;QAElB,MAAM,EAAE,MAAM,CAAC;QAEf,KAAK,EAAE,MAAM,CAAC;KACf;IAED;;OAEG;IACH,UAAiB,SAAS;QACxB,YAAY,EAAE,MAAM,CAAC;QAErB,MAAM,EAAE,MAAM,CAAC;QAEf,EAAE,EAAE,MAAM,CAAC;QAEX,YAAY,EAAE,MAAM,CAAC;QAErB,MAAM,EAAE,MAAM,CAAC;QAEf,SAAS,EAAE,MAAM,CAAC;QAElB,MAAM,EAAE,MAAM,CAAC;QAEf,iBAAiB,EAAE,MAAM,CAAC;QAE1B,mBAAmB,EAAE,MAAM,CAAC;QAE5B,eAAe,EAAE,MAAM,CAAC;KACzB;CACF;AAED,MAAM,WAAW,iBAAiB;IAChC;;OAEG;IACH,YAAY,EAAE,MAAM,CAAC;IAErB;;OAEG;IACH,KAAK,EAAE,KAAK,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC;IAErC;;;OAGG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAE5B;;OAEG;IACH,WAAW,CAAC,EAAE,iBAAiB,CAAC,WAAW,CAAC;CAC7C;AAED,yBAAiB,iBAAiB,CAAC;IACjC;;OAEG;IACH,UAAiB,IAAI;QACnB;;;WAGG;QACH,QAAQ,EAAE,OAAO,CAAC;QAElB;;;;;;WAMG;QACH,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC;KACnB;IAED,UAAiB,IAAI,CAAC;QACpB;;;;;;WAMG;QACH,UAAiB,KAAK;YACpB;;;eAGG;YACH,YAAY,EAAE,MAAM,CAAC;YAErB;;eAEG;YACH,SAAS,EAAE,UAAU,CAAC,SAAS,CAAC;SACjC;KACF;IAED;;OAEG;IACH,UAAiB,WAAW;QAC1B;;WAEG;QACH,UAAU,CAAC,EAAE,UAAU,CAAC;QAExB;;;WAGG;QACH,cAAc,CAAC,EAAE,MAAM,CAAC;QAExB;;;;;WAKG;QACH,WAAW,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;QAE5B;;;WAGG;QACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;QAE1B;;WAEG;QACH,WAAW,CAAC,EAAE,OAAO,GAAG,OAAO,CAAC;KACjC;CACF;AAED,MAAM,CAAC,OAAO,WAAW,KAAK,CAAC;IAC7B,OAAO,EACL,KAAK,mBAAmB,IAAI,mBAAmB,EAC/C,KAAK,qBAAqB,IAAI,qBAAqB,EACnD,KAAK,mBAAmB,IAAI,mBAAmB,EAC/C,KAAK,iBAAiB,IAAI,iBAAiB,GAC5C,CAAC;CACH"}