bem-ai-sdk 0.26.0 → 0.27.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +14 -0
- package/client.d.mts +127 -2
- package/client.d.mts.map +1 -1
- package/client.d.ts +127 -2
- package/client.d.ts.map +1 -1
- package/client.js +136 -3
- package/client.js.map +1 -1
- package/client.mjs +136 -3
- package/client.mjs.map +1 -1
- package/package.json +1 -1
- package/resources/buckets.d.mts +272 -0
- package/resources/buckets.d.mts.map +1 -0
- package/resources/buckets.d.ts +272 -0
- package/resources/buckets.d.ts.map +1 -0
- package/resources/buckets.js +93 -0
- package/resources/buckets.js.map +1 -0
- package/resources/buckets.mjs +89 -0
- package/resources/buckets.mjs.map +1 -0
- package/resources/entities/entities.d.mts +516 -0
- package/resources/entities/entities.d.mts.map +1 -0
- package/resources/entities/entities.d.ts +516 -0
- package/resources/entities/entities.d.ts.map +1 -0
- package/resources/entities/entities.js +93 -0
- package/resources/entities/entities.js.map +1 -0
- package/resources/entities/entities.mjs +88 -0
- package/resources/entities/entities.mjs.map +1 -0
- package/resources/entities/index.d.mts +3 -0
- package/resources/entities/index.d.mts.map +1 -0
- package/resources/entities/index.d.ts +3 -0
- package/resources/entities/index.d.ts.map +1 -0
- package/resources/entities/index.js +9 -0
- package/resources/entities/index.js.map +1 -0
- package/resources/entities/index.mjs +4 -0
- package/resources/entities/index.mjs.map +1 -0
- package/resources/entities/synonyms.d.mts +109 -0
- package/resources/entities/synonyms.d.mts.map +1 -0
- package/resources/entities/synonyms.d.ts +109 -0
- package/resources/entities/synonyms.d.ts.map +1 -0
- package/resources/entities/synonyms.js +61 -0
- package/resources/entities/synonyms.js.map +1 -0
- package/resources/entities/synonyms.mjs +57 -0
- package/resources/entities/synonyms.mjs.map +1 -0
- package/resources/entities.d.mts +2 -0
- package/resources/entities.d.mts.map +1 -0
- package/resources/entities.d.ts +2 -0
- package/resources/entities.d.ts.map +1 -0
- package/resources/entities.js +6 -0
- package/resources/entities.js.map +1 -0
- package/resources/entities.mjs +3 -0
- package/resources/entities.mjs.map +1 -0
- package/resources/entity-types/entity-types.d.mts +302 -0
- package/resources/entity-types/entity-types.d.mts.map +1 -0
- package/resources/entity-types/entity-types.d.ts +302 -0
- package/resources/entity-types/entity-types.d.ts.map +1 -0
- package/resources/entity-types/entity-types.js +104 -0
- package/resources/entity-types/entity-types.js.map +1 -0
- package/resources/entity-types/entity-types.mjs +99 -0
- package/resources/entity-types/entity-types.mjs.map +1 -0
- package/resources/entity-types/index.d.mts +3 -0
- package/resources/entity-types/index.d.mts.map +1 -0
- package/resources/entity-types/index.d.ts +3 -0
- package/resources/entity-types/index.d.ts.map +1 -0
- package/resources/entity-types/index.js +9 -0
- package/resources/entity-types/index.js.map +1 -0
- package/resources/entity-types/index.mjs +4 -0
- package/resources/entity-types/index.mjs.map +1 -0
- package/resources/entity-types/reviewers.d.mts +136 -0
- package/resources/entity-types/reviewers.d.mts.map +1 -0
- package/resources/entity-types/reviewers.d.ts +136 -0
- package/resources/entity-types/reviewers.d.ts.map +1 -0
- package/resources/entity-types/reviewers.js +74 -0
- package/resources/entity-types/reviewers.js.map +1 -0
- package/resources/entity-types/reviewers.mjs +70 -0
- package/resources/entity-types/reviewers.mjs.map +1 -0
- package/resources/entity-types.d.mts +2 -0
- package/resources/entity-types.d.mts.map +1 -0
- package/resources/entity-types.d.ts +2 -0
- package/resources/entity-types.d.ts.map +1 -0
- package/resources/entity-types.js +6 -0
- package/resources/entity-types.js.map +1 -0
- package/resources/entity-types.mjs +3 -0
- package/resources/entity-types.mjs.map +1 -0
- package/resources/eval/eval.d.mts +4 -0
- package/resources/eval/eval.d.mts.map +1 -1
- package/resources/eval/eval.d.ts +4 -0
- package/resources/eval/eval.d.ts.map +1 -1
- package/resources/eval/eval.js +4 -0
- package/resources/eval/eval.js.map +1 -1
- package/resources/eval/eval.mjs +4 -0
- package/resources/eval/eval.mjs.map +1 -1
- package/resources/eval/index.d.mts +1 -0
- package/resources/eval/index.d.mts.map +1 -1
- package/resources/eval/index.d.ts +1 -0
- package/resources/eval/index.d.ts.map +1 -1
- package/resources/eval/index.js +3 -1
- package/resources/eval/index.js.map +1 -1
- package/resources/eval/index.mjs +1 -0
- package/resources/eval/index.mjs.map +1 -1
- package/resources/eval/score.d.mts +486 -0
- package/resources/eval/score.d.mts.map +1 -0
- package/resources/eval/score.d.ts +486 -0
- package/resources/eval/score.d.ts.map +1 -0
- package/resources/eval/score.js +126 -0
- package/resources/eval/score.js.map +1 -0
- package/resources/eval/score.mjs +122 -0
- package/resources/eval/score.mjs.map +1 -0
- package/resources/fs.d.mts +28 -0
- package/resources/fs.d.mts.map +1 -1
- package/resources/fs.d.ts +28 -0
- package/resources/fs.d.ts.map +1 -1
- package/resources/index.d.mts +6 -0
- package/resources/index.d.mts.map +1 -1
- package/resources/index.d.ts +6 -0
- package/resources/index.d.ts.map +1 -1
- package/resources/index.js +13 -1
- package/resources/index.js.map +1 -1
- package/resources/index.mjs +6 -0
- package/resources/index.mjs.map +1 -1
- package/resources/knowledge-graph.d.mts +125 -0
- package/resources/knowledge-graph.d.mts.map +1 -0
- package/resources/knowledge-graph.d.ts +125 -0
- package/resources/knowledge-graph.d.ts.map +1 -0
- package/resources/knowledge-graph.js +34 -0
- package/resources/knowledge-graph.js.map +1 -0
- package/resources/knowledge-graph.mjs +30 -0
- package/resources/knowledge-graph.mjs.map +1 -0
- package/resources/review-queue.d.mts +192 -0
- package/resources/review-queue.d.mts.map +1 -0
- package/resources/review-queue.d.ts +192 -0
- package/resources/review-queue.d.ts.map +1 -0
- package/resources/review-queue.js +49 -0
- package/resources/review-queue.js.map +1 -0
- package/resources/review-queue.mjs +45 -0
- package/resources/review-queue.mjs.map +1 -0
- package/resources/subscriptions.d.mts +14 -2
- package/resources/subscriptions.d.mts.map +1 -1
- package/resources/subscriptions.d.ts +14 -2
- package/resources/subscriptions.d.ts.map +1 -1
- package/resources/subscriptions.js +14 -2
- package/resources/subscriptions.js.map +1 -1
- package/resources/subscriptions.mjs +14 -2
- package/resources/subscriptions.mjs.map +1 -1
- package/resources/users.d.mts +62 -0
- package/resources/users.d.mts.map +1 -0
- package/resources/users.d.ts +62 -0
- package/resources/users.d.ts.map +1 -0
- package/resources/users.js +35 -0
- package/resources/users.js.map +1 -0
- package/resources/users.mjs +31 -0
- package/resources/users.mjs.map +1 -0
- package/src/client.ts +220 -3
- package/src/resources/buckets.ts +351 -0
- package/src/resources/entities/entities.ts +639 -0
- package/src/resources/entities/index.ts +20 -0
- package/src/resources/entities/synonyms.ts +140 -0
- package/src/resources/entities.ts +3 -0
- package/src/resources/entity-types/entity-types.ts +396 -0
- package/src/resources/entity-types/index.ts +19 -0
- package/src/resources/entity-types/reviewers.ts +175 -0
- package/src/resources/entity-types.ts +3 -0
- package/src/resources/eval/eval.ts +18 -0
- package/src/resources/eval/index.ts +7 -0
- package/src/resources/eval/score.ts +590 -0
- package/src/resources/fs.ts +30 -0
- package/src/resources/index.ts +40 -0
- package/src/resources/knowledge-graph.ts +153 -0
- package/src/resources/review-queue.ts +234 -0
- package/src/resources/subscriptions.ts +14 -2
- package/src/resources/users.ts +76 -0
- package/src/version.ts +1 -1
- package/version.d.mts +1 -1
- package/version.d.ts +1 -1
- package/version.js +1 -1
- package/version.mjs +1 -1
|
@@ -0,0 +1,486 @@
|
|
|
1
|
+
import { APIResource } from "../../core/resource.mjs";
|
|
2
|
+
import * as OutputsAPI from "../outputs.mjs";
|
|
3
|
+
import { APIPromise } from "../../core/api-promise.mjs";
|
|
4
|
+
import { RequestOptions } from "../../internal/request-options.mjs";
|
|
5
|
+
/**
|
|
6
|
+
* Monitor, evaluate, and iterate on the quality of every function in your
|
|
7
|
+
* environment. Function Accuracy bundles two complementary loops:
|
|
8
|
+
*
|
|
9
|
+
* ## Evaluations (`/v3/eval`)
|
|
10
|
+
*
|
|
11
|
+
* Trigger and retrieve per-transformation evaluations. Evaluations run
|
|
12
|
+
* asynchronously and score each transformation's output against the
|
|
13
|
+
* function's schema for confidence, per-field hallucination detection,
|
|
14
|
+
* and relevance. Supported for `extract`, `transform`, `analyze`, and
|
|
15
|
+
* `join` events.
|
|
16
|
+
*
|
|
17
|
+
* 1. **Trigger** — `POST /v3/eval` queues jobs for a batch of transformation IDs.
|
|
18
|
+
* 2. **Poll** — `GET /v3/eval/results` returns the current state of each
|
|
19
|
+
* requested ID, partitioned into `results`, `pending`, and `failed`.
|
|
20
|
+
* Accepts either `eventIDs` (preferred) or `transformationIDs` as a
|
|
21
|
+
* comma-separated query parameter, and always keys the response by
|
|
22
|
+
* event KSUID.
|
|
23
|
+
*
|
|
24
|
+
* Up to 100 IDs may be submitted per request.
|
|
25
|
+
*
|
|
26
|
+
* ## Metrics, review, regression (`/v3/functions/{metrics,review,regression,compare}`)
|
|
27
|
+
*
|
|
28
|
+
* Roll evaluation results and user corrections up into actionable
|
|
29
|
+
* function-level signal:
|
|
30
|
+
*
|
|
31
|
+
* - **`GET /v3/functions/metrics`** — aggregate accuracy, precision,
|
|
32
|
+
* recall, F1, and confusion-matrix counts per function.
|
|
33
|
+
* - **`POST /v3/functions/review`** — sample-size estimation,
|
|
34
|
+
* confidence-bucketed distribution, PR-AUC, and per-threshold
|
|
35
|
+
* confidence intervals (Wald or Wilson) for picking review cutoffs.
|
|
36
|
+
* - **`POST /v3/functions/regression`** — replay corrected historical
|
|
37
|
+
* inputs against a new function version, producing a labeled
|
|
38
|
+
* regression dataset.
|
|
39
|
+
* - **`POST /v3/functions/regression/corrections`** — propagate
|
|
40
|
+
* baseline corrections onto the regression dataset so it can be
|
|
41
|
+
* scored.
|
|
42
|
+
* - **`POST /v3/functions/compare`** — compute aggregate and
|
|
43
|
+
* field-level lift between any two versions, optionally scoped to
|
|
44
|
+
* the regression dataset.
|
|
45
|
+
*
|
|
46
|
+
* All five endpoints support `extract` end-to-end on both the vision
|
|
47
|
+
* and OCR paths, alongside the legacy `transform` / `analyze` / `join`
|
|
48
|
+
* types.
|
|
49
|
+
*/
|
|
50
|
+
export declare class Score extends APIResource {
|
|
51
|
+
/**
|
|
52
|
+
* **Score a function against a list of (input, expected) pairs.**
|
|
53
|
+
*
|
|
54
|
+
* Submits a batch of `(input, expected)` pairs, runs the named function over each
|
|
55
|
+
* input, and returns per-pair + aggregate accuracy metrics comparing the
|
|
56
|
+
* function's actual output to the provided expected JSON.
|
|
57
|
+
*
|
|
58
|
+
* Scoring runs asynchronously. The response carries a `scoreRunID`; poll
|
|
59
|
+
* `GET /v3/eval/score/{scoreRunID}` until `status` is one of `completed`, `error`,
|
|
60
|
+
* or `cancelled`.
|
|
61
|
+
*
|
|
62
|
+
* `matchConfig` controls comparator behavior:
|
|
63
|
+
*
|
|
64
|
+
* - `numericTolerance`: relative tolerance for numeric fields (0 = exact)
|
|
65
|
+
* - `stringMatch`: `exact` (default) or `fuzzy` (Levenshtein ratio)
|
|
66
|
+
* - `arrayMatch`: `by-index` (default; only mode in P0)
|
|
67
|
+
* - `ignorePaths`: JSON Pointer paths to skip, supports `*` wildcards
|
|
68
|
+
*
|
|
69
|
+
* @example
|
|
70
|
+
* ```ts
|
|
71
|
+
* const score = await client.eval.score.create({
|
|
72
|
+
* functionName: 'functionName',
|
|
73
|
+
* pairs: [
|
|
74
|
+
* {
|
|
75
|
+
* expected: {},
|
|
76
|
+
* input: {
|
|
77
|
+
* inputContent: 'inputContent',
|
|
78
|
+
* inputType: 'csv',
|
|
79
|
+
* },
|
|
80
|
+
* },
|
|
81
|
+
* ],
|
|
82
|
+
* });
|
|
83
|
+
* ```
|
|
84
|
+
*/
|
|
85
|
+
create(body: ScoreCreateParams, options?: RequestOptions): APIPromise<ScoreCreateResponse>;
|
|
86
|
+
/**
|
|
87
|
+
* **Get the status and per-pair results of a score run.**
|
|
88
|
+
*
|
|
89
|
+
* Returns `aggregate` only once `status` reaches `completed`. `perPair` is
|
|
90
|
+
* populated incrementally — each pair's `fieldResults` appears as its underlying
|
|
91
|
+
* function call terminates.
|
|
92
|
+
*
|
|
93
|
+
* @example
|
|
94
|
+
* ```ts
|
|
95
|
+
* const score = await client.eval.score.retrieve(
|
|
96
|
+
* 'scoreRunID',
|
|
97
|
+
* );
|
|
98
|
+
* ```
|
|
99
|
+
*/
|
|
100
|
+
retrieve(scoreRunID: string, options?: RequestOptions): APIPromise<ScoreRetrieveResponse>;
|
|
101
|
+
/**
|
|
102
|
+
* **Cancel an in-flight score run.**
|
|
103
|
+
*
|
|
104
|
+
* Transitions the run to `cancelled`. Function calls already in flight are allowed
|
|
105
|
+
* to finish (best-effort cancellation via the job queue); results from completed
|
|
106
|
+
* pairs may still appear in subsequent GETs.
|
|
107
|
+
*
|
|
108
|
+
* @example
|
|
109
|
+
* ```ts
|
|
110
|
+
* const response = await client.eval.score.cancel(
|
|
111
|
+
* 'scoreRunID',
|
|
112
|
+
* );
|
|
113
|
+
* ```
|
|
114
|
+
*/
|
|
115
|
+
cancel(scoreRunID: string, options?: RequestOptions): APIPromise<ScoreCancelResponse>;
|
|
116
|
+
}
|
|
117
|
+
/**
|
|
118
|
+
* Returned by `POST /v3/eval/score`.
|
|
119
|
+
*/
|
|
120
|
+
export interface ScoreCreateResponse {
|
|
121
|
+
/**
|
|
122
|
+
* Run identifier. Use with `GET /v3/eval/score/{scoreRunID}`.
|
|
123
|
+
*/
|
|
124
|
+
scoreRunID: string;
|
|
125
|
+
/**
|
|
126
|
+
* Status values for an eval-score run.
|
|
127
|
+
*/
|
|
128
|
+
status: 'pending' | 'initializing' | 'running' | 'completed' | 'error' | 'cancelled';
|
|
129
|
+
}
|
|
130
|
+
/**
|
|
131
|
+
* Full status payload returned by `GET /v3/eval/score/{scoreRunID}`.
|
|
132
|
+
*/
|
|
133
|
+
export interface ScoreRetrieveResponse {
|
|
134
|
+
functionName: string;
|
|
135
|
+
functionVersionNum: number;
|
|
136
|
+
/**
|
|
137
|
+
* Comparator configuration. All fields optional; conservative defaults.
|
|
138
|
+
*/
|
|
139
|
+
matchConfig: ScoreRetrieveResponse.MatchConfig;
|
|
140
|
+
/**
|
|
141
|
+
* Per-pair results. `fieldResults` appears once a pair has been compared.
|
|
142
|
+
*/
|
|
143
|
+
perPair: Array<ScoreRetrieveResponse.PerPair>;
|
|
144
|
+
/**
|
|
145
|
+
* Counts across all pairs.
|
|
146
|
+
*/
|
|
147
|
+
progress: ScoreRetrieveResponse.Progress;
|
|
148
|
+
scoreRunID: string;
|
|
149
|
+
/**
|
|
150
|
+
* Status values for an eval-score run.
|
|
151
|
+
*/
|
|
152
|
+
status: 'pending' | 'initializing' | 'running' | 'completed' | 'error' | 'cancelled';
|
|
153
|
+
/**
|
|
154
|
+
* Aggregate accuracy metrics.
|
|
155
|
+
*/
|
|
156
|
+
aggregate?: ScoreRetrieveResponse.Aggregate;
|
|
157
|
+
}
|
|
158
|
+
export declare namespace ScoreRetrieveResponse {
|
|
159
|
+
/**
|
|
160
|
+
* Comparator configuration. All fields optional; conservative defaults.
|
|
161
|
+
*/
|
|
162
|
+
interface MatchConfig {
|
|
163
|
+
/**
|
|
164
|
+
* P0 supports only `by-index`.
|
|
165
|
+
*/
|
|
166
|
+
arrayMatch?: 'by-index';
|
|
167
|
+
/**
|
|
168
|
+
* Levenshtein-ratio threshold used when `stringMatch == "fuzzy"`. Range `[0, 1]`.
|
|
169
|
+
* Default `0.85`.
|
|
170
|
+
*/
|
|
171
|
+
fuzzyThreshold?: number;
|
|
172
|
+
/**
|
|
173
|
+
* JSON Pointer paths to skip during comparison. The asterisk character matches
|
|
174
|
+
* arbitrary object keys / array indices.
|
|
175
|
+
*
|
|
176
|
+
* Example values: /metadata, /lineItems with asterisk segment, etc.
|
|
177
|
+
*/
|
|
178
|
+
ignorePaths?: Array<string>;
|
|
179
|
+
/**
|
|
180
|
+
* Relative tolerance for numeric fields. `0` (default) means exact equality;
|
|
181
|
+
* `0.01` means ±1%.
|
|
182
|
+
*/
|
|
183
|
+
numericTolerance?: number;
|
|
184
|
+
/**
|
|
185
|
+
* `exact` (default) or `fuzzy`.
|
|
186
|
+
*/
|
|
187
|
+
stringMatch?: 'exact' | 'fuzzy';
|
|
188
|
+
}
|
|
189
|
+
/**
|
|
190
|
+
* Per-pair result.
|
|
191
|
+
*/
|
|
192
|
+
interface PerPair {
|
|
193
|
+
pairIndex: number;
|
|
194
|
+
/**
|
|
195
|
+
* Per-pair status.
|
|
196
|
+
*/
|
|
197
|
+
status: 'pending' | 'running' | 'completed' | 'failed';
|
|
198
|
+
/**
|
|
199
|
+
* The function call that produced the actual output, if any.
|
|
200
|
+
*/
|
|
201
|
+
callID?: string;
|
|
202
|
+
/**
|
|
203
|
+
* Error message if the underlying function call failed.
|
|
204
|
+
*/
|
|
205
|
+
errorMessage?: string;
|
|
206
|
+
/**
|
|
207
|
+
* Per-leaf comparator output. Present only after the pair has been compared.
|
|
208
|
+
*/
|
|
209
|
+
fieldResults?: Array<PerPair.FieldResult>;
|
|
210
|
+
}
|
|
211
|
+
namespace PerPair {
|
|
212
|
+
/**
|
|
213
|
+
* One leaf in `expected ∪ actual`.
|
|
214
|
+
*/
|
|
215
|
+
interface FieldResult {
|
|
216
|
+
/**
|
|
217
|
+
* Classification:
|
|
218
|
+
*
|
|
219
|
+
* - `exact`: both present and deep-equal
|
|
220
|
+
* - `within_tolerance`: both numbers, within configured tolerance
|
|
221
|
+
* - `fuzzy_match`: both strings, Levenshtein ratio above threshold
|
|
222
|
+
* - `miss`: expected present, actual absent or different
|
|
223
|
+
* - `extra`: actual present, expected absent
|
|
224
|
+
*/
|
|
225
|
+
match: 'exact' | 'within_tolerance' | 'fuzzy_match' | 'miss' | 'extra';
|
|
226
|
+
/**
|
|
227
|
+
* JSON Pointer to the leaf.
|
|
228
|
+
*/
|
|
229
|
+
path: string;
|
|
230
|
+
actual?: unknown;
|
|
231
|
+
/**
|
|
232
|
+
* Populated for numeric comparisons; `actual - expected`.
|
|
233
|
+
*/
|
|
234
|
+
delta?: number;
|
|
235
|
+
expected?: unknown;
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
/**
|
|
239
|
+
* Counts across all pairs.
|
|
240
|
+
*/
|
|
241
|
+
interface Progress {
|
|
242
|
+
completed: number;
|
|
243
|
+
failed: number;
|
|
244
|
+
total: number;
|
|
245
|
+
}
|
|
246
|
+
/**
|
|
247
|
+
* Aggregate accuracy metrics.
|
|
248
|
+
*/
|
|
249
|
+
interface Aggregate {
|
|
250
|
+
exactMatches: number;
|
|
251
|
+
extras: number;
|
|
252
|
+
f1: number;
|
|
253
|
+
fuzzyMatches: number;
|
|
254
|
+
misses: number;
|
|
255
|
+
precision: number;
|
|
256
|
+
recall: number;
|
|
257
|
+
totalFieldsActual: number;
|
|
258
|
+
totalFieldsExpected: number;
|
|
259
|
+
withinTolerance: number;
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
/**
|
|
263
|
+
* Full status payload returned by `GET /v3/eval/score/{scoreRunID}`.
|
|
264
|
+
*/
|
|
265
|
+
export interface ScoreCancelResponse {
|
|
266
|
+
functionName: string;
|
|
267
|
+
functionVersionNum: number;
|
|
268
|
+
/**
|
|
269
|
+
* Comparator configuration. All fields optional; conservative defaults.
|
|
270
|
+
*/
|
|
271
|
+
matchConfig: ScoreCancelResponse.MatchConfig;
|
|
272
|
+
/**
|
|
273
|
+
* Per-pair results. `fieldResults` appears once a pair has been compared.
|
|
274
|
+
*/
|
|
275
|
+
perPair: Array<ScoreCancelResponse.PerPair>;
|
|
276
|
+
/**
|
|
277
|
+
* Counts across all pairs.
|
|
278
|
+
*/
|
|
279
|
+
progress: ScoreCancelResponse.Progress;
|
|
280
|
+
scoreRunID: string;
|
|
281
|
+
/**
|
|
282
|
+
* Status values for an eval-score run.
|
|
283
|
+
*/
|
|
284
|
+
status: 'pending' | 'initializing' | 'running' | 'completed' | 'error' | 'cancelled';
|
|
285
|
+
/**
|
|
286
|
+
* Aggregate accuracy metrics.
|
|
287
|
+
*/
|
|
288
|
+
aggregate?: ScoreCancelResponse.Aggregate;
|
|
289
|
+
}
|
|
290
|
+
export declare namespace ScoreCancelResponse {
|
|
291
|
+
/**
|
|
292
|
+
* Comparator configuration. All fields optional; conservative defaults.
|
|
293
|
+
*/
|
|
294
|
+
interface MatchConfig {
|
|
295
|
+
/**
|
|
296
|
+
* P0 supports only `by-index`.
|
|
297
|
+
*/
|
|
298
|
+
arrayMatch?: 'by-index';
|
|
299
|
+
/**
|
|
300
|
+
* Levenshtein-ratio threshold used when `stringMatch == "fuzzy"`. Range `[0, 1]`.
|
|
301
|
+
* Default `0.85`.
|
|
302
|
+
*/
|
|
303
|
+
fuzzyThreshold?: number;
|
|
304
|
+
/**
|
|
305
|
+
* JSON Pointer paths to skip during comparison. The asterisk character matches
|
|
306
|
+
* arbitrary object keys / array indices.
|
|
307
|
+
*
|
|
308
|
+
* Example values: /metadata, /lineItems with asterisk segment, etc.
|
|
309
|
+
*/
|
|
310
|
+
ignorePaths?: Array<string>;
|
|
311
|
+
/**
|
|
312
|
+
* Relative tolerance for numeric fields. `0` (default) means exact equality;
|
|
313
|
+
* `0.01` means ±1%.
|
|
314
|
+
*/
|
|
315
|
+
numericTolerance?: number;
|
|
316
|
+
/**
|
|
317
|
+
* `exact` (default) or `fuzzy`.
|
|
318
|
+
*/
|
|
319
|
+
stringMatch?: 'exact' | 'fuzzy';
|
|
320
|
+
}
|
|
321
|
+
/**
|
|
322
|
+
* Per-pair result.
|
|
323
|
+
*/
|
|
324
|
+
interface PerPair {
|
|
325
|
+
pairIndex: number;
|
|
326
|
+
/**
|
|
327
|
+
* Per-pair status.
|
|
328
|
+
*/
|
|
329
|
+
status: 'pending' | 'running' | 'completed' | 'failed';
|
|
330
|
+
/**
|
|
331
|
+
* The function call that produced the actual output, if any.
|
|
332
|
+
*/
|
|
333
|
+
callID?: string;
|
|
334
|
+
/**
|
|
335
|
+
* Error message if the underlying function call failed.
|
|
336
|
+
*/
|
|
337
|
+
errorMessage?: string;
|
|
338
|
+
/**
|
|
339
|
+
* Per-leaf comparator output. Present only after the pair has been compared.
|
|
340
|
+
*/
|
|
341
|
+
fieldResults?: Array<PerPair.FieldResult>;
|
|
342
|
+
}
|
|
343
|
+
namespace PerPair {
|
|
344
|
+
/**
|
|
345
|
+
* One leaf in `expected ∪ actual`.
|
|
346
|
+
*/
|
|
347
|
+
interface FieldResult {
|
|
348
|
+
/**
|
|
349
|
+
* Classification:
|
|
350
|
+
*
|
|
351
|
+
* - `exact`: both present and deep-equal
|
|
352
|
+
* - `within_tolerance`: both numbers, within configured tolerance
|
|
353
|
+
* - `fuzzy_match`: both strings, Levenshtein ratio above threshold
|
|
354
|
+
* - `miss`: expected present, actual absent or different
|
|
355
|
+
* - `extra`: actual present, expected absent
|
|
356
|
+
*/
|
|
357
|
+
match: 'exact' | 'within_tolerance' | 'fuzzy_match' | 'miss' | 'extra';
|
|
358
|
+
/**
|
|
359
|
+
* JSON Pointer to the leaf.
|
|
360
|
+
*/
|
|
361
|
+
path: string;
|
|
362
|
+
actual?: unknown;
|
|
363
|
+
/**
|
|
364
|
+
* Populated for numeric comparisons; `actual - expected`.
|
|
365
|
+
*/
|
|
366
|
+
delta?: number;
|
|
367
|
+
expected?: unknown;
|
|
368
|
+
}
|
|
369
|
+
}
|
|
370
|
+
/**
|
|
371
|
+
* Counts across all pairs.
|
|
372
|
+
*/
|
|
373
|
+
interface Progress {
|
|
374
|
+
completed: number;
|
|
375
|
+
failed: number;
|
|
376
|
+
total: number;
|
|
377
|
+
}
|
|
378
|
+
/**
|
|
379
|
+
* Aggregate accuracy metrics.
|
|
380
|
+
*/
|
|
381
|
+
interface Aggregate {
|
|
382
|
+
exactMatches: number;
|
|
383
|
+
extras: number;
|
|
384
|
+
f1: number;
|
|
385
|
+
fuzzyMatches: number;
|
|
386
|
+
misses: number;
|
|
387
|
+
precision: number;
|
|
388
|
+
recall: number;
|
|
389
|
+
totalFieldsActual: number;
|
|
390
|
+
totalFieldsExpected: number;
|
|
391
|
+
withinTolerance: number;
|
|
392
|
+
}
|
|
393
|
+
}
|
|
394
|
+
export interface ScoreCreateParams {
|
|
395
|
+
/**
|
|
396
|
+
* Name of the function to score. Must be of type extract, transform, or analyze.
|
|
397
|
+
*/
|
|
398
|
+
functionName: string;
|
|
399
|
+
/**
|
|
400
|
+
* Up to 1000 pairs per request.
|
|
401
|
+
*/
|
|
402
|
+
pairs: Array<ScoreCreateParams.Pair>;
|
|
403
|
+
/**
|
|
404
|
+
* Optional version number to score against. P0: only the function's current
|
|
405
|
+
* version is accepted; passing a different version returns 422.
|
|
406
|
+
*/
|
|
407
|
+
functionVersionNum?: number;
|
|
408
|
+
/**
|
|
409
|
+
* Comparator configuration. All fields optional; conservative defaults.
|
|
410
|
+
*/
|
|
411
|
+
matchConfig?: ScoreCreateParams.MatchConfig;
|
|
412
|
+
}
|
|
413
|
+
export declare namespace ScoreCreateParams {
|
|
414
|
+
/**
|
|
415
|
+
* One `(input, expected)` pair.
|
|
416
|
+
*/
|
|
417
|
+
interface Pair {
|
|
418
|
+
/**
|
|
419
|
+
* Expected output for this input, as a JSON value. The comparator walks
|
|
420
|
+
* `expected ∪ actual` and produces a per-leaf classification.
|
|
421
|
+
*/
|
|
422
|
+
expected: unknown;
|
|
423
|
+
/**
|
|
424
|
+
* A single file input with base64-encoded content.
|
|
425
|
+
*
|
|
426
|
+
* When using the Bem CLI, use `@path/to/file` in the `inputContent` field to
|
|
427
|
+
* automatically read and base64-encode the file:
|
|
428
|
+
* `--input.single-file '{"inputContent": "@file.pdf", "inputType": "pdf"}' --wait`
|
|
429
|
+
*/
|
|
430
|
+
input: Pair.Input;
|
|
431
|
+
}
|
|
432
|
+
namespace Pair {
|
|
433
|
+
/**
|
|
434
|
+
* A single file input with base64-encoded content.
|
|
435
|
+
*
|
|
436
|
+
* When using the Bem CLI, use `@path/to/file` in the `inputContent` field to
|
|
437
|
+
* automatically read and base64-encode the file:
|
|
438
|
+
* `--input.single-file '{"inputContent": "@file.pdf", "inputType": "pdf"}' --wait`
|
|
439
|
+
*/
|
|
440
|
+
interface Input {
|
|
441
|
+
/**
|
|
442
|
+
* Base64-encoded file content. In the Bem CLI, use `@path/to/file` to embed file
|
|
443
|
+
* contents automatically.
|
|
444
|
+
*/
|
|
445
|
+
inputContent: string;
|
|
446
|
+
/**
|
|
447
|
+
* The input type of the content you're sending for transformation.
|
|
448
|
+
*/
|
|
449
|
+
inputType: OutputsAPI.InputType;
|
|
450
|
+
}
|
|
451
|
+
}
|
|
452
|
+
/**
|
|
453
|
+
* Comparator configuration. All fields optional; conservative defaults.
|
|
454
|
+
*/
|
|
455
|
+
interface MatchConfig {
|
|
456
|
+
/**
|
|
457
|
+
* P0 supports only `by-index`.
|
|
458
|
+
*/
|
|
459
|
+
arrayMatch?: 'by-index';
|
|
460
|
+
/**
|
|
461
|
+
* Levenshtein-ratio threshold used when `stringMatch == "fuzzy"`. Range `[0, 1]`.
|
|
462
|
+
* Default `0.85`.
|
|
463
|
+
*/
|
|
464
|
+
fuzzyThreshold?: number;
|
|
465
|
+
/**
|
|
466
|
+
* JSON Pointer paths to skip during comparison. The asterisk character matches
|
|
467
|
+
* arbitrary object keys / array indices.
|
|
468
|
+
*
|
|
469
|
+
* Example values: /metadata, /lineItems with asterisk segment, etc.
|
|
470
|
+
*/
|
|
471
|
+
ignorePaths?: Array<string>;
|
|
472
|
+
/**
|
|
473
|
+
* Relative tolerance for numeric fields. `0` (default) means exact equality;
|
|
474
|
+
* `0.01` means ±1%.
|
|
475
|
+
*/
|
|
476
|
+
numericTolerance?: number;
|
|
477
|
+
/**
|
|
478
|
+
* `exact` (default) or `fuzzy`.
|
|
479
|
+
*/
|
|
480
|
+
stringMatch?: 'exact' | 'fuzzy';
|
|
481
|
+
}
|
|
482
|
+
}
|
|
483
|
+
export declare namespace Score {
|
|
484
|
+
export { type ScoreCreateResponse as ScoreCreateResponse, type ScoreRetrieveResponse as ScoreRetrieveResponse, type ScoreCancelResponse as ScoreCancelResponse, type ScoreCreateParams as ScoreCreateParams, };
|
|
485
|
+
}
|
|
486
|
+
//# sourceMappingURL=score.d.mts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"score.d.mts","sourceRoot":"","sources":["../../src/resources/eval/score.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,WAAW,EAAE,gCAA4B;AAClD,OAAO,KAAK,UAAU,uBAAmB;AACzC,OAAO,EAAE,UAAU,EAAE,mCAA+B;AACpD,OAAO,EAAE,cAAc,EAAE,2CAAuC;AAGhE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4CG;AACH,qBAAa,KAAM,SAAQ,WAAW;IACpC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAiCG;IACH,MAAM,CAAC,IAAI,EAAE,iBAAiB,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,UAAU,CAAC,mBAAmB,CAAC;IAI1F;;;;;;;;;;;;;OAaG;IACH,QAAQ,CAAC,UAAU,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,UAAU,CAAC,qBAAqB,CAAC;IAIzF;;;;;;;;;;;;;OAaG;IACH,MAAM,CAAC,UAAU,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,UAAU,CAAC,mBAAmB,CAAC;CAGtF;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC;;OAEG;IACH,UAAU,EAAE,MAAM,CAAC;IAEnB;;OAEG;IACH,MAAM,EAAE,SAAS,GAAG,cAAc,GAAG,SAAS,GAAG,WAAW,GAAG,OAAO,GAAG,WAAW,CAAC;CACtF;AAED;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC,YAAY,EAAE,MAAM,CAAC;IAErB,kBAAkB,EAAE,MAAM,CAAC;IAE3B;;OAEG;IACH,WAAW,EAAE,qBAAqB,CAAC,WAAW,CAAC;IAE/C;;OAEG;IACH,OAAO,EAAE,KAAK,CAAC,qBAAqB,CAAC,OAAO,CAAC,CAAC;IAE9C;;OAEG;IACH,QAAQ,EAAE,qBAAqB,CAAC,QAAQ,CAAC;IAEzC,UAAU,EAAE,MAAM,CAAC;IAEnB;;OAEG;IACH,MAAM,EAAE,SAAS,GAAG,cAAc,GAAG,SAAS,GAAG,WAAW,GAAG,OAAO,GAAG,WAAW,CAAC;IAErF;;OAEG;IACH,SAAS,CAAC,EAAE,qBAAqB,CAAC,SAAS,CAAC;CAC7C;AAED,yBAAiB,qBAAqB,CAAC;IACrC;;OAEG;IACH,UAAiB,WAAW;QAC1B;;WAEG;QACH,UAAU,CAAC,EAAE,UAAU,CAAC;QAExB;;;WAGG;QACH,cAAc,CAAC,EAAE,MAAM,CAAC;QAExB;;;;;WAKG;QACH,WAAW,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;QAE5B;;;WAGG;QACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;QAE1B;;WAEG;QACH,WAAW,CAAC,EAAE,OAAO,GAAG,OAAO,CAAC;KACjC;IAED;;OAEG;IACH,UAAiB,OAAO;QACtB,SAAS,EAAE,MAAM,CAAC;QAElB;;WAEG;QACH,MAAM,EAAE,SAAS,GAAG,SAAS,GAAG,WAAW,GAAG,QAAQ,CAAC;QAEvD;;WAEG;QACH,MAAM,CAAC,EAAE,MAAM,CAAC;QAEhB;;WAEG;QACH,YAAY,CAAC,EAAE,MAAM,CAAC;QAEtB;;WAEG;QACH,YAAY,CAAC,EAAE,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;KAC3C;IAED,UAAiB,OAAO,CAAC;QACvB;;WAEG;QACH,UAAiB,WAAW;YAC1B;;;;;;;;eAQG;YACH,KAAK,EAAE,OAAO,GAAG,kBAAkB,GAAG,aAAa,GAAG,MAAM,GAAG,OAAO,CAAC;YAEvE;;eAEG;YACH,IAAI,EAAE,MAAM,CAAC;YAEb,MAAM,CAAC,EAAE,OAAO,CAAC;YAEjB;;eAEG;YACH,KAAK,CAAC,EAAE,MAAM,CAAC;YAEf,QAAQ,CAAC,EAAE,OAAO,CAAC;SACpB;KACF;IAED;;OAEG;IACH,UAAiB,QAAQ;QACvB,SAAS,EAAE,MAAM,CAAC;QAElB,MAAM,EAAE,MAAM,CAAC;QAEf,KAAK,EAAE,MAAM,CAAC;KACf;IAED;;OAEG;IACH,UAAiB,SAAS;QACxB,YAAY,EAAE,MAAM,CAAC;QAErB,MAAM,EAAE,MAAM,CAAC;QAEf,EAAE,EAAE,MAAM,CAAC;QAEX,YAAY,EAAE,MAAM,CAAC;QAErB,MAAM,EAAE,MAAM,CAAC;QAEf,SAAS,EAAE,MAAM,CAAC;QAElB,MAAM,EAAE,MAAM,CAAC;QAEf,iBAAiB,EAAE,MAAM,CAAC;QAE1B,mBAAmB,EAAE,MAAM,CAAC;QAE5B,eAAe,EAAE,MAAM,CAAC;KACzB;CACF;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,YAAY,EAAE,MAAM,CAAC;IAErB,kBAAkB,EAAE,MAAM,CAAC;IAE3B;;OAEG;IACH,WAAW,EAAE,mBAAmB,CAAC,WAAW,CAAC;IAE7C;;OAEG;IACH,OAAO,EAAE,KAAK,CAAC,mBAAmB,CAAC,OAAO,CAAC,CAAC;IAE5C;;OAEG;IACH,QAAQ,EAAE,mBAAmB,CAAC,QAAQ,CAAC;IAEvC,UAAU,EAAE,MAAM,CAAC;IAEnB;;OAEG;IACH,MAAM,EAAE,SAAS,GAAG,cAAc,GAAG,SAAS,GAAG,WAAW,GAAG,OAAO,GAAG,WAAW,CAAC;IAErF;;OAEG;IACH,SAAS,CAAC,EAAE,mBAAmB,CAAC,SAAS,CAAC;CAC3C;AAED,yBAAiB,mBAAmB,CAAC;IACnC;;OAEG;IACH,UAAiB,WAAW;QAC1B;;WAEG;QACH,UAAU,CAAC,EAAE,UAAU,CAAC;QAExB;;;WAGG;QACH,cAAc,CAAC,EAAE,MAAM,CAAC;QAExB;;;;;WAKG;QACH,WAAW,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;QAE5B;;;WAGG;QACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;QAE1B;;WAEG;QACH,WAAW,CAAC,EAAE,OAAO,GAAG,OAAO,CAAC;KACjC;IAED;;OAEG;IACH,UAAiB,OAAO;QACtB,SAAS,EAAE,MAAM,CAAC;QAElB;;WAEG;QACH,MAAM,EAAE,SAAS,GAAG,SAAS,GAAG,WAAW,GAAG,QAAQ,CAAC;QAEvD;;WAEG;QACH,MAAM,CAAC,EAAE,MAAM,CAAC;QAEhB;;WAEG;QACH,YAAY,CAAC,EAAE,MAAM,CAAC;QAEtB;;WAEG;QACH,YAAY,CAAC,EAAE,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;KAC3C;IAED,UAAiB,OAAO,CAAC;QACvB;;WAEG;QACH,UAAiB,WAAW;YAC1B;;;;;;;;eAQG;YACH,KAAK,EAAE,OAAO,GAAG,kBAAkB,GAAG,aAAa,GAAG,MAAM,GAAG,OAAO,CAAC;YAEvE;;eAEG;YACH,IAAI,EAAE,MAAM,CAAC;YAEb,MAAM,CAAC,EAAE,OAAO,CAAC;YAEjB;;eAEG;YACH,KAAK,CAAC,EAAE,MAAM,CAAC;YAEf,QAAQ,CAAC,EAAE,OAAO,CAAC;SACpB;KACF;IAED;;OAEG;IACH,UAAiB,QAAQ;QACvB,SAAS,EAAE,MAAM,CAAC;QAElB,MAAM,EAAE,MAAM,CAAC;QAEf,KAAK,EAAE,MAAM,CAAC;KACf;IAED;;OAEG;IACH,UAAiB,SAAS;QACxB,YAAY,EAAE,MAAM,CAAC;QAErB,MAAM,EAAE,MAAM,CAAC;QAEf,EAAE,EAAE,MAAM,CAAC;QAEX,YAAY,EAAE,MAAM,CAAC;QAErB,MAAM,EAAE,MAAM,CAAC;QAEf,SAAS,EAAE,MAAM,CAAC;QAElB,MAAM,EAAE,MAAM,CAAC;QAEf,iBAAiB,EAAE,MAAM,CAAC;QAE1B,mBAAmB,EAAE,MAAM,CAAC;QAE5B,eAAe,EAAE,MAAM,CAAC;KACzB;CACF;AAED,MAAM,WAAW,iBAAiB;IAChC;;OAEG;IACH,YAAY,EAAE,MAAM,CAAC;IAErB;;OAEG;IACH,KAAK,EAAE,KAAK,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC;IAErC;;;OAGG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAE5B;;OAEG;IACH,WAAW,CAAC,EAAE,iBAAiB,CAAC,WAAW,CAAC;CAC7C;AAED,yBAAiB,iBAAiB,CAAC;IACjC;;OAEG;IACH,UAAiB,IAAI;QACnB;;;WAGG;QACH,QAAQ,EAAE,OAAO,CAAC;QAElB;;;;;;WAMG;QACH,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC;KACnB;IAED,UAAiB,IAAI,CAAC;QACpB;;;;;;WAMG;QACH,UAAiB,KAAK;YACpB;;;eAGG;YACH,YAAY,EAAE,MAAM,CAAC;YAErB;;eAEG;YACH,SAAS,EAAE,UAAU,CAAC,SAAS,CAAC;SACjC;KACF;IAED;;OAEG;IACH,UAAiB,WAAW;QAC1B;;WAEG;QACH,UAAU,CAAC,EAAE,UAAU,CAAC;QAExB;;;WAGG;QACH,cAAc,CAAC,EAAE,MAAM,CAAC;QAExB;;;;;WAKG;QACH,WAAW,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;QAE5B;;;WAGG;QACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;QAE1B;;WAEG;QACH,WAAW,CAAC,EAAE,OAAO,GAAG,OAAO,CAAC;KACjC;CACF;AAED,MAAM,CAAC,OAAO,WAAW,KAAK,CAAC;IAC7B,OAAO,EACL,KAAK,mBAAmB,IAAI,mBAAmB,EAC/C,KAAK,qBAAqB,IAAI,qBAAqB,EACnD,KAAK,mBAAmB,IAAI,mBAAmB,EAC/C,KAAK,iBAAiB,IAAI,iBAAiB,GAC5C,CAAC;CACH"}
|