@eigenpal/cli 0.16.1 → 0.16.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -0
- package/dist/cli.js +447 -19
- package/dist/docs-bundle.json +33 -12
- package/dist/skill/reference/dataset-format.md +274 -1
- package/dist/skill/reference/step-types.md +510 -5
- package/dist/skill/reference/workflow-yaml.md +96 -0
- package/dist/templates/human-review-confidence-exception/workflow.yaml +60 -0
- package/dist/templates/human-review-explicit-critical/workflow.yaml +62 -0
- package/package.json +1 -1
|
@@ -21,7 +21,7 @@ dataset/
|
|
|
21
21
|
│ ├── expected.json OPTIONAL — expected output or { "$error": ... }
|
|
22
22
|
│ ├── expected/
|
|
23
23
|
│ │ └── Invoice.docx referenced by expected.json
|
|
24
|
-
│ └── meta.json OPTIONAL — { rowOrder?, annotation?, overrides? }
|
|
24
|
+
│ └── meta.json OPTIONAL — { rowOrder?, annotation?, overrides?, review? }
|
|
25
25
|
│
|
|
26
26
|
└── unsupported-format/
|
|
27
27
|
├── input.json
|
|
@@ -97,6 +97,163 @@ reference is resolved into the S3 file descriptor the worker consumes.
|
|
|
97
97
|
|
|
98
98
|
Each file is uploaded to S3 and the original filename is preserved.
|
|
99
99
|
|
|
100
|
+
## Human review fixtures (`meta.json`)
|
|
101
|
+
|
|
102
|
+
When a workflow includes `control.human_review`, attach optional `review`
|
|
103
|
+
fixtures to `examples/<name>/meta.json`. Evaluation runs simulate review and
|
|
104
|
+
never pause — fixtures assert selection routing and optional simulated edits.
|
|
105
|
+
|
|
106
|
+
```json
|
|
107
|
+
{
|
|
108
|
+
"review": {
|
|
109
|
+
"version": 1,
|
|
110
|
+
"steps": {
|
|
111
|
+
"review-invoice": {
|
|
112
|
+
"fields": {
|
|
113
|
+
"/total": {
|
|
114
|
+
"expectedRoute": "review",
|
|
115
|
+
"expectedReason": "low_confidence",
|
|
116
|
+
"expectedValue": 42
|
|
117
|
+
}
|
|
118
|
+
},
|
|
119
|
+
"simulate": {
|
|
120
|
+
"outcome": "approved",
|
|
121
|
+
"edits": { "/total": 42 }
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
Keys under `steps` match the human review step name. `expectedRoute` is `review`
|
|
130
|
+
or `skip`. `expectedReason` matches selection reason codes such as
|
|
131
|
+
`low_confidence`, `always`, or `threshold_met`. `simulate.outcome` is currently
|
|
132
|
+
`approved`; `simulate.edits` applies reviewer corrections during the simulated
|
|
133
|
+
path. Omit `simulate` to assert routing only.
|
|
134
|
+
|
|
135
|
+
<!-- GENERATED:DATASET_META_REVIEW START -->
|
|
136
|
+
Schema for `meta.json` → `review` (from `HumanReviewEvaluationFixtureSchema` in `@eigenpal/types`):
|
|
137
|
+
|
|
138
|
+
```json
|
|
139
|
+
{
|
|
140
|
+
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
141
|
+
"type": "object",
|
|
142
|
+
"properties": {
|
|
143
|
+
"version": {
|
|
144
|
+
"type": "number",
|
|
145
|
+
"const": 1
|
|
146
|
+
},
|
|
147
|
+
"steps": {
|
|
148
|
+
"type": "object",
|
|
149
|
+
"propertyNames": {
|
|
150
|
+
"type": "string",
|
|
151
|
+
"minLength": 1
|
|
152
|
+
},
|
|
153
|
+
"additionalProperties": {
|
|
154
|
+
"type": "object",
|
|
155
|
+
"properties": {
|
|
156
|
+
"fields": {
|
|
157
|
+
"default": {},
|
|
158
|
+
"type": "object",
|
|
159
|
+
"propertyNames": {
|
|
160
|
+
"type": "string"
|
|
161
|
+
},
|
|
162
|
+
"additionalProperties": {
|
|
163
|
+
"type": "object",
|
|
164
|
+
"properties": {
|
|
165
|
+
"expectedRoute": {
|
|
166
|
+
"type": "string",
|
|
167
|
+
"enum": [
|
|
168
|
+
"review",
|
|
169
|
+
"skip"
|
|
170
|
+
]
|
|
171
|
+
},
|
|
172
|
+
"expectedReason": {
|
|
173
|
+
"type": "string",
|
|
174
|
+
"enum": [
|
|
175
|
+
"always",
|
|
176
|
+
"explicit",
|
|
177
|
+
"low_confidence",
|
|
178
|
+
"missing_confidence",
|
|
179
|
+
"all",
|
|
180
|
+
"reviewer_edit",
|
|
181
|
+
"threshold_met",
|
|
182
|
+
"never",
|
|
183
|
+
"excluded",
|
|
184
|
+
"unmatched",
|
|
185
|
+
"missing_confidence_skip"
|
|
186
|
+
]
|
|
187
|
+
},
|
|
188
|
+
"expectedValue": {
|
|
189
|
+
"anyOf": [
|
|
190
|
+
{
|
|
191
|
+
"type": "string"
|
|
192
|
+
},
|
|
193
|
+
{
|
|
194
|
+
"type": "number"
|
|
195
|
+
},
|
|
196
|
+
{
|
|
197
|
+
"type": "boolean"
|
|
198
|
+
},
|
|
199
|
+
{
|
|
200
|
+
"type": "null"
|
|
201
|
+
}
|
|
202
|
+
]
|
|
203
|
+
}
|
|
204
|
+
},
|
|
205
|
+
"additionalProperties": false
|
|
206
|
+
}
|
|
207
|
+
},
|
|
208
|
+
"simulate": {
|
|
209
|
+
"type": "object",
|
|
210
|
+
"properties": {
|
|
211
|
+
"outcome": {
|
|
212
|
+
"type": "string",
|
|
213
|
+
"const": "approved"
|
|
214
|
+
},
|
|
215
|
+
"edits": {
|
|
216
|
+
"type": "object",
|
|
217
|
+
"propertyNames": {
|
|
218
|
+
"type": "string"
|
|
219
|
+
},
|
|
220
|
+
"additionalProperties": {
|
|
221
|
+
"anyOf": [
|
|
222
|
+
{
|
|
223
|
+
"type": "string"
|
|
224
|
+
},
|
|
225
|
+
{
|
|
226
|
+
"type": "number"
|
|
227
|
+
},
|
|
228
|
+
{
|
|
229
|
+
"type": "boolean"
|
|
230
|
+
},
|
|
231
|
+
{
|
|
232
|
+
"type": "null"
|
|
233
|
+
}
|
|
234
|
+
]
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
},
|
|
238
|
+
"additionalProperties": false
|
|
239
|
+
}
|
|
240
|
+
},
|
|
241
|
+
"required": [
|
|
242
|
+
"fields"
|
|
243
|
+
],
|
|
244
|
+
"additionalProperties": false
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
},
|
|
248
|
+
"required": [
|
|
249
|
+
"version",
|
|
250
|
+
"steps"
|
|
251
|
+
],
|
|
252
|
+
"additionalProperties": false
|
|
253
|
+
}
|
|
254
|
+
```
|
|
255
|
+
<!-- GENERATED:DATASET_META_REVIEW END -->
|
|
256
|
+
|
|
100
257
|
## Validate before pushing
|
|
101
258
|
|
|
102
259
|
```bash
|
|
@@ -193,6 +350,7 @@ Failure-expected examples store a single top-level `$error` object. At least one
|
|
|
193
350
|
| `rowOrder` | integer | no | | Non-negative display order hint for this example. |
|
|
194
351
|
| `annotation` | string | no | | Free-form example note, limited to 2000 characters. |
|
|
195
352
|
| `overrides` | record<string, unknown> | no | | Per-step output overrides as `{ "steps": { "<stepName>": <outputObject> } }`; overridden steps are skipped or partially merged during evaluation. |
|
|
353
|
+
| `review` | object | no | | Optional human-review evaluation fixture: expected field routes and simulated approvals or edits for this example. |
|
|
196
354
|
|
|
197
355
|
|
|
198
356
|
### Import mode (`DatasetImportModeSchema`)
|
|
@@ -238,6 +396,121 @@ Allowed values: `"append"` \| `"replace"`
|
|
|
238
396
|
"type": "string"
|
|
239
397
|
},
|
|
240
398
|
"additionalProperties": {}
|
|
399
|
+
},
|
|
400
|
+
"review": {
|
|
401
|
+
"description": "Optional human-review evaluation fixture: expected field routes and simulated approvals or edits for this example.",
|
|
402
|
+
"type": "object",
|
|
403
|
+
"properties": {
|
|
404
|
+
"version": {
|
|
405
|
+
"type": "number",
|
|
406
|
+
"const": 1
|
|
407
|
+
},
|
|
408
|
+
"steps": {
|
|
409
|
+
"type": "object",
|
|
410
|
+
"propertyNames": {
|
|
411
|
+
"type": "string",
|
|
412
|
+
"minLength": 1
|
|
413
|
+
},
|
|
414
|
+
"additionalProperties": {
|
|
415
|
+
"type": "object",
|
|
416
|
+
"properties": {
|
|
417
|
+
"fields": {
|
|
418
|
+
"default": {},
|
|
419
|
+
"type": "object",
|
|
420
|
+
"propertyNames": {
|
|
421
|
+
"type": "string"
|
|
422
|
+
},
|
|
423
|
+
"additionalProperties": {
|
|
424
|
+
"type": "object",
|
|
425
|
+
"properties": {
|
|
426
|
+
"expectedRoute": {
|
|
427
|
+
"type": "string",
|
|
428
|
+
"enum": [
|
|
429
|
+
"review",
|
|
430
|
+
"skip"
|
|
431
|
+
]
|
|
432
|
+
},
|
|
433
|
+
"expectedReason": {
|
|
434
|
+
"type": "string",
|
|
435
|
+
"enum": [
|
|
436
|
+
"always",
|
|
437
|
+
"explicit",
|
|
438
|
+
"low_confidence",
|
|
439
|
+
"missing_confidence",
|
|
440
|
+
"all",
|
|
441
|
+
"reviewer_edit",
|
|
442
|
+
"threshold_met",
|
|
443
|
+
"never",
|
|
444
|
+
"excluded",
|
|
445
|
+
"unmatched",
|
|
446
|
+
"missing_confidence_skip"
|
|
447
|
+
]
|
|
448
|
+
},
|
|
449
|
+
"expectedValue": {
|
|
450
|
+
"anyOf": [
|
|
451
|
+
{
|
|
452
|
+
"type": "string"
|
|
453
|
+
},
|
|
454
|
+
{
|
|
455
|
+
"type": "number"
|
|
456
|
+
},
|
|
457
|
+
{
|
|
458
|
+
"type": "boolean"
|
|
459
|
+
},
|
|
460
|
+
{
|
|
461
|
+
"type": "null"
|
|
462
|
+
}
|
|
463
|
+
]
|
|
464
|
+
}
|
|
465
|
+
},
|
|
466
|
+
"additionalProperties": false
|
|
467
|
+
}
|
|
468
|
+
},
|
|
469
|
+
"simulate": {
|
|
470
|
+
"type": "object",
|
|
471
|
+
"properties": {
|
|
472
|
+
"outcome": {
|
|
473
|
+
"type": "string",
|
|
474
|
+
"const": "approved"
|
|
475
|
+
},
|
|
476
|
+
"edits": {
|
|
477
|
+
"type": "object",
|
|
478
|
+
"propertyNames": {
|
|
479
|
+
"type": "string"
|
|
480
|
+
},
|
|
481
|
+
"additionalProperties": {
|
|
482
|
+
"anyOf": [
|
|
483
|
+
{
|
|
484
|
+
"type": "string"
|
|
485
|
+
},
|
|
486
|
+
{
|
|
487
|
+
"type": "number"
|
|
488
|
+
},
|
|
489
|
+
{
|
|
490
|
+
"type": "boolean"
|
|
491
|
+
},
|
|
492
|
+
{
|
|
493
|
+
"type": "null"
|
|
494
|
+
}
|
|
495
|
+
]
|
|
496
|
+
}
|
|
497
|
+
}
|
|
498
|
+
},
|
|
499
|
+
"additionalProperties": false
|
|
500
|
+
}
|
|
501
|
+
},
|
|
502
|
+
"required": [
|
|
503
|
+
"fields"
|
|
504
|
+
],
|
|
505
|
+
"additionalProperties": false
|
|
506
|
+
}
|
|
507
|
+
}
|
|
508
|
+
},
|
|
509
|
+
"required": [
|
|
510
|
+
"version",
|
|
511
|
+
"steps"
|
|
512
|
+
],
|
|
513
|
+
"additionalProperties": false
|
|
241
514
|
}
|
|
242
515
|
},
|
|
243
516
|
"additionalProperties": false
|