@lesliechan721/agent-skill-eval 0.1.0-beta.7 → 0.1.0-beta.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (98) hide show
  1. package/dist/bin.js +149 -39
  2. package/dist/bin.js.map +1 -1
  3. package/dist/codex-eval-app-server.d.ts +199 -0
  4. package/dist/codex-eval-app-server.js +1366 -0
  5. package/dist/codex-eval-app-server.js.map +1 -0
  6. package/dist/codex-eval-cases.d.ts +43 -0
  7. package/dist/codex-eval-cases.js +327 -0
  8. package/dist/codex-eval-cases.js.map +1 -0
  9. package/dist/codex-eval-isolation.d.ts +61 -0
  10. package/dist/codex-eval-isolation.js +318 -0
  11. package/dist/codex-eval-isolation.js.map +1 -0
  12. package/dist/codex-eval-process-tree.d.ts +38 -0
  13. package/dist/codex-eval-process-tree.js +803 -0
  14. package/dist/codex-eval-process-tree.js.map +1 -0
  15. package/dist/codex-eval-runtime-config.d.ts +43 -0
  16. package/dist/codex-eval-runtime-config.js +607 -0
  17. package/dist/codex-eval-runtime-config.js.map +1 -0
  18. package/dist/codex-trigger-eval.d.ts +82 -0
  19. package/dist/codex-trigger-eval.js +789 -0
  20. package/dist/codex-trigger-eval.js.map +1 -0
  21. package/dist/codex-workflow.d.ts +188 -0
  22. package/dist/codex-workflow.js +1790 -0
  23. package/dist/codex-workflow.js.map +1 -0
  24. package/dist/credentials.d.ts +5 -0
  25. package/dist/credentials.js +30 -0
  26. package/dist/credentials.js.map +1 -0
  27. package/dist/custom-process.d.ts +58 -0
  28. package/dist/custom-process.js +409 -0
  29. package/dist/custom-process.js.map +1 -0
  30. package/dist/eval-audit.d.ts +38 -0
  31. package/dist/eval-audit.js +71 -0
  32. package/dist/eval-audit.js.map +1 -0
  33. package/dist/eval-file.d.ts +9 -0
  34. package/dist/eval-file.js +15 -0
  35. package/dist/eval-file.js.map +1 -0
  36. package/dist/eval-result.d.ts +64 -0
  37. package/dist/eval-result.js +144 -0
  38. package/dist/eval-result.js.map +1 -0
  39. package/dist/eval-run.d.ts +23 -0
  40. package/dist/eval-run.js +539 -0
  41. package/dist/eval-run.js.map +1 -0
  42. package/dist/eval-spec.d.ts +57 -0
  43. package/dist/eval-spec.js +120 -0
  44. package/dist/eval-spec.js.map +1 -0
  45. package/dist/eval-suite-binding.d.ts +4 -0
  46. package/dist/eval-suite-binding.js +19 -0
  47. package/dist/eval-suite-binding.js.map +1 -0
  48. package/dist/index.d.ts +34 -9
  49. package/dist/index.js +18 -6
  50. package/dist/index.js.map +1 -1
  51. package/dist/json-schema.js +75 -5
  52. package/dist/json-schema.js.map +1 -1
  53. package/dist/output-path.js +2 -2
  54. package/dist/process-tree.d.ts +19 -1
  55. package/dist/process-tree.js +321 -66
  56. package/dist/process-tree.js.map +1 -1
  57. package/dist/process.d.ts +0 -2
  58. package/dist/process.js +76 -25
  59. package/dist/process.js.map +1 -1
  60. package/dist/provider-doctor.d.ts +33 -0
  61. package/dist/provider-doctor.js +86 -0
  62. package/dist/provider-doctor.js.map +1 -0
  63. package/dist/providers.d.ts +82 -0
  64. package/dist/providers.js +630 -0
  65. package/dist/providers.js.map +1 -0
  66. package/dist/run-paths.d.ts +8 -0
  67. package/dist/run-paths.js +71 -0
  68. package/dist/run-paths.js.map +1 -0
  69. package/dist/runtime-skill.d.ts +6 -0
  70. package/dist/runtime-skill.js +87 -3
  71. package/dist/runtime-skill.js.map +1 -1
  72. package/dist/schemas/digests.json +3 -3
  73. package/dist/schemas/eval-audit.schema.json +64 -0
  74. package/dist/schemas/eval-result.schema.json +202 -0
  75. package/dist/schemas/eval-spec.schema.json +109 -0
  76. package/dist/schemas.d.ts +1 -3
  77. package/dist/schemas.js +0 -17
  78. package/dist/schemas.js.map +1 -1
  79. package/dist/selected-cases.d.ts +30 -0
  80. package/dist/selected-cases.js +388 -0
  81. package/dist/selected-cases.js.map +1 -0
  82. package/dist/types.d.ts +2 -34
  83. package/package.json +6 -1
  84. package/dist/bundle.d.ts +0 -93
  85. package/dist/bundle.js +0 -216
  86. package/dist/bundle.js.map +0 -1
  87. package/dist/capabilities.d.ts +0 -2
  88. package/dist/capabilities.js +0 -38
  89. package/dist/capabilities.js.map +0 -1
  90. package/dist/plan.d.ts +0 -79
  91. package/dist/plan.js +0 -285
  92. package/dist/plan.js.map +0 -1
  93. package/dist/run.d.ts +0 -2
  94. package/dist/run.js +0 -592
  95. package/dist/run.js.map +0 -1
  96. package/dist/schemas/aw-skill-eval-execution-plan.schema.json +0 -220
  97. package/dist/schemas/eval-run-bundle.schema.json +0 -408
  98. package/dist/schemas/eval-runtime-capabilities.schema.json +0 -85
package/dist/schemas/aw-skill-eval-execution-plan.schema.json DELETED
@@ -1,220 +0,0 @@
1
- {
2
- "$schema": "https://json-schema.org/draft/2020-12/schema",
3
- "$id": "https://agent-skill-toolkit.invalid/schemas/aw-skill-eval-execution-plan.schema.json",
4
- "title": "AwSkillEvalExecutionPlan",
5
- "type": "object",
6
- "required": [
7
- "schemaVersion",
8
- "planId",
9
- "target",
10
- "cases",
11
- "provider",
12
- "configurations",
13
- "runsPerCase",
14
- "timeoutMs",
15
- "outputRoot",
16
- "capture"
17
- ],
18
- "properties": {
19
- "schemaVersion": {
20
- "const": 3
21
- },
22
- "planId": {
23
- "type": "string",
24
- "minLength": 1,
25
- "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$"
26
- },
27
- "minimumEvidenceLevel": {
28
- "enum": [
29
- "executed",
30
- "comparative"
31
- ]
32
- },
33
- "target": {
34
- "type": "object",
35
- "required": [
36
- "name",
37
- "path",
38
- "digest"
39
- ],
40
- "properties": {
41
- "name": {
42
- "type": "string",
43
- "minLength": 1
44
- },
45
- "path": {
46
- "type": "string",
47
- "minLength": 1
48
- },
49
- "digest": {
50
- "type": "string",
51
- "pattern": "^sha256:[0-9a-f]{64}$"
52
- }
53
- },
54
- "additionalProperties": false
55
- },
56
- "cases": {
57
- "type": "object",
58
- "required": [
59
- "paths",
60
- "corpusDigest"
61
- ],
62
- "properties": {
63
- "paths": {
64
- "type": "array",
65
- "minItems": 1,
66
- "items": {
67
- "type": "string",
68
- "minLength": 1
69
- },
70
- "uniqueItems": true
71
- },
72
- "corpusDigest": {
73
- "type": "string",
74
- "pattern": "^sha256:[0-9a-f]{64}$"
75
- }
76
- },
77
- "additionalProperties": false
78
- },
79
- "provider": {
80
- "type": "object",
81
- "required": [
82
- "id",
83
- "command",
84
- "capabilitiesPath",
85
- "capabilitiesDigest",
86
- "inheritEnvironment"
87
- ],
88
- "properties": {
89
- "id": {
90
- "type": "string",
91
- "minLength": 1
92
- },
93
- "command": {
94
- "type": "array",
95
- "minItems": 1,
96
- "items": {
97
- "type": "string",
98
- "minLength": 1
99
- }
100
- },
101
- "cwd": {
102
- "type": [
103
- "string",
104
- "null"
105
- ]
106
- },
107
- "env": {
108
- "type": "object",
109
- "additionalProperties": {
110
- "type": "string"
111
- }
112
- },
113
- "capabilitiesPath": {
114
- "type": "string",
115
- "minLength": 1
116
- },
117
- "capabilitiesDigest": {
118
- "type": "string",
119
- "pattern": "^sha256:[0-9a-f]{64}$"
120
- },
121
- "inheritEnvironment": {
122
- "type": "boolean"
123
- }
124
- },
125
- "additionalProperties": false
126
- },
127
- "configurations": {
128
- "type": "array",
129
- "minItems": 1,
130
- "items": {
131
- "type": "object",
132
- "required": [
133
- "id",
134
- "skillMode",
135
- "comparisonId"
136
- ],
137
- "properties": {
138
- "id": {
139
- "type": "string",
140
- "minLength": 1,
141
- "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$"
142
- },
143
- "skillMode": {
144
- "enum": [
145
- "with-skill",
146
- "without-skill",
147
- "previous-version"
148
- ]
149
- },
150
- "comparisonId": {
151
- "type": [
152
- "string",
153
- "null"
154
- ],
155
- "minLength": 1,
156
- "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$"
157
- },
158
- "skillPath": {
159
- "type": [
160
- "string",
161
- "null"
162
- ]
163
- },
164
- "skillDigest": {
165
- "type": [
166
- "string",
167
- "null"
168
- ],
169
- "pattern": "^sha256:[0-9a-f]{64}$"
170
- },
171
- "env": {
172
- "type": "object",
173
- "additionalProperties": {
174
- "type": "string"
175
- }
176
- }
177
- },
178
- "additionalProperties": false
179
- }
180
- },
181
- "runsPerCase": {
182
- "type": "integer",
183
- "minimum": 1
184
- },
185
- "timeoutMs": {
186
- "type": "integer",
187
- "minimum": 1000
188
- },
189
- "outputRoot": {
190
- "type": "string",
191
- "minLength": 1,
192
- "pattern": "^(?!/)(?!.*(?:^|/)\\.\\.(?:/|$))(?!.*\\\\).+$"
193
- },
194
- "capture": {
195
- "type": "object",
196
- "required": [
197
- "outputs",
198
- "trace",
199
- "timing",
200
- "usage"
201
- ],
202
- "properties": {
203
- "outputs": {
204
- "type": "boolean"
205
- },
206
- "trace": {
207
- "type": "boolean"
208
- },
209
- "timing": {
210
- "type": "boolean"
211
- },
212
- "usage": {
213
- "type": "boolean"
214
- }
215
- },
216
- "additionalProperties": false
217
- }
218
- },
219
- "additionalProperties": false
220
- }
package/dist/schemas/eval-run-bundle.schema.json DELETED
@@ -1,408 +0,0 @@
1
- {
2
- "$schema": "https://json-schema.org/draft/2020-12/schema",
3
- "$id": "https://agent-skill-toolkit.invalid/schemas/eval-run-bundle.schema.json",
4
- "title": "EvalRunBundle",
5
- "type": "object",
6
- "required": [
7
- "schemaVersion",
8
- "runner",
9
- "independentContext",
10
- "provider",
11
- "planDigest",
12
- "targetSkillDigest",
13
- "caseCorpusDigest",
14
- "comparisonPlanned",
15
- "evidenceLevel",
16
- "runs",
17
- "aggregate",
18
- "limitations"
19
- ],
20
- "properties": {
21
- "schemaVersion": {
22
- "const": 3
23
- },
24
- "runner": {
25
- "type": "object",
26
- "required": [
27
- "id",
28
- "version"
29
- ],
30
- "properties": {
31
- "id": {
32
- "type": "string",
33
- "minLength": 1
34
- },
35
- "version": {
36
- "type": "string",
37
- "minLength": 1
38
- }
39
- },
40
- "additionalProperties": false
41
- },
42
- "independentContext": {
43
- "type": "boolean"
44
- },
45
- "provider": {
46
- "type": "object",
47
- "required": [
48
- "id",
49
- "version"
50
- ],
51
- "properties": {
52
- "id": {
53
- "type": "string",
54
- "minLength": 1
55
- },
56
- "version": {
57
- "type": "string",
58
- "minLength": 1
59
- }
60
- },
61
- "additionalProperties": false
62
- },
63
- "planDigest": {
64
- "type": "string",
65
- "pattern": "^sha256:[0-9a-f]{64}$"
66
- },
67
- "targetSkillDigest": {
68
- "type": "string",
69
- "pattern": "^sha256:[0-9a-f]{64}$"
70
- },
71
- "caseCorpusDigest": {
72
- "type": "string",
73
- "pattern": "^sha256:[0-9a-f]{64}$"
74
- },
75
- "comparisonPlanned": {
76
- "type": "boolean"
77
- },
78
- "evidenceLevel": {
79
- "enum": [
80
- "executed",
81
- "comparative"
82
- ]
83
- },
84
- "evidenceRequirement": {
85
- "type": "object",
86
- "required": [
87
- "minimumEvidenceLevel",
88
- "plannedComparisons",
89
- "comparableComparisons",
90
- "satisfied"
91
- ],
92
- "properties": {
93
- "minimumEvidenceLevel": {
94
- "enum": [
95
- "executed",
96
- "comparative"
97
- ]
98
- },
99
- "plannedComparisons": {
100
- "type": "integer",
101
- "minimum": 0
102
- },
103
- "comparableComparisons": {
104
- "type": "integer",
105
- "minimum": 0
106
- },
107
- "satisfied": {
108
- "type": "boolean"
109
- }
110
- },
111
- "additionalProperties": false
112
- },
113
- "processManagement": {
114
- "type": "object",
115
- "required": [
116
- "mode",
117
- "descendantCleanupGuaranteed"
118
- ],
119
- "properties": {
120
- "mode": {
121
- "enum": [
122
- "managed",
123
- "verified-observed",
124
- "observed"
125
- ]
126
- },
127
- "descendantCleanupGuaranteed": {
128
- "type": "boolean"
129
- },
130
- "limitation": {
131
- "type": "string",
132
- "minLength": 1
133
- }
134
- },
135
- "allOf": [
136
- {
137
- "if": {
138
- "properties": {
139
- "mode": {
140
- "enum": [
141
- "verified-observed",
142
- "observed"
143
- ]
144
- }
145
- },
146
- "required": [
147
- "mode"
148
- ]
149
- },
150
- "then": {
151
- "required": [
152
- "limitation"
153
- ]
154
- }
155
- }
156
- ],
157
- "additionalProperties": false
158
- },
159
- "runs": {
160
- "type": "array",
161
- "minItems": 1,
162
- "items": {
163
- "type": "object",
164
- "required": [
165
- "caseId",
166
- "configuration",
167
- "comparisonId",
168
- "skillMode",
169
- "skillDigest",
170
- "runIndex",
171
- "status",
172
- "exitCode",
173
- "durationMs",
174
- "stdoutPath",
175
- "stdoutDigest",
176
- "stderrPath",
177
- "stderrDigest",
178
- "usage",
179
- "grading",
180
- "artifacts"
181
- ],
182
- "properties": {
183
- "caseId": {
184
- "type": "string",
185
- "minLength": 1
186
- },
187
- "configuration": {
188
- "type": "string",
189
- "minLength": 1
190
- },
191
- "comparisonId": {
192
- "type": [
193
- "string",
194
- "null"
195
- ],
196
- "minLength": 1
197
- },
198
- "skillMode": {
199
- "enum": [
200
- "with-skill",
201
- "without-skill",
202
- "previous-version"
203
- ]
204
- },
205
- "skillDigest": {
206
- "type": [
207
- "string",
208
- "null"
209
- ],
210
- "pattern": "^sha256:[0-9a-f]{64}$"
211
- },
212
- "runIndex": {
213
- "type": "integer",
214
- "minimum": 1
215
- },
216
- "status": {
217
- "enum": [
218
- "completed",
219
- "process-failed",
220
- "timeout",
221
- "error"
222
- ]
223
- },
224
- "exitCode": {
225
- "type": [
226
- "integer",
227
- "null"
228
- ]
229
- },
230
- "durationMs": {
231
- "type": "integer",
232
- "minimum": 0
233
- },
234
- "stdoutPath": {
235
- "type": "string",
236
- "minLength": 1
237
- },
238
- "stdoutDigest": {
239
- "type": "string",
240
- "pattern": "^sha256:[0-9a-f]{64}$"
241
- },
242
- "stderrPath": {
243
- "type": "string",
244
- "minLength": 1
245
- },
246
- "stderrDigest": {
247
- "type": "string",
248
- "pattern": "^sha256:[0-9a-f]{64}$"
249
- },
250
- "usage": {
251
- "type": [
252
- "object",
253
- "null"
254
- ]
255
- },
256
- "grading": {
257
- "type": [
258
- "object",
259
- "null"
260
- ],
261
- "required": [
262
- "grader",
263
- "result"
264
- ],
265
- "properties": {
266
- "grader": {
267
- "type": "object",
268
- "required": [
269
- "id",
270
- "version"
271
- ],
272
- "properties": {
273
- "id": {
274
- "type": "string",
275
- "minLength": 1
276
- },
277
- "version": {
278
- "type": "string",
279
- "minLength": 1
280
- }
281
- },
282
- "additionalProperties": true
283
- },
284
- "result": {
285
- "type": "object"
286
- }
287
- },
288
- "additionalProperties": true
289
- },
290
- "artifacts": {
291
- "type": "array",
292
- "items": {
293
- "type": "object",
294
- "required": [
295
- "role",
296
- "path",
297
- "digest"
298
- ],
299
- "properties": {
300
- "role": {
301
- "enum": [
302
- "trace",
303
- "report",
304
- "output",
305
- "grading",
306
- "other"
307
- ]
308
- },
309
- "path": {
310
- "type": "string",
311
- "minLength": 1
312
- },
313
- "digest": {
314
- "type": "string",
315
- "pattern": "^sha256:[0-9a-f]{64}$"
316
- }
317
- },
318
- "additionalProperties": false
319
- }
320
- }
321
- },
322
- "additionalProperties": false
323
- }
324
- },
325
- "aggregate": {
326
- "type": "object",
327
- "required": [
328
- "cases",
329
- "configurations",
330
- "runs",
331
- "completed",
332
- "processFailed",
333
- "timedOut",
334
- "comparisons"
335
- ],
336
- "properties": {
337
- "cases": {
338
- "type": "integer",
339
- "minimum": 1
340
- },
341
- "configurations": {
342
- "type": "integer",
343
- "minimum": 1
344
- },
345
- "runs": {
346
- "type": "integer",
347
- "minimum": 1
348
- },
349
- "completed": {
350
- "type": "integer",
351
- "minimum": 0
352
- },
353
- "processFailed": {
354
- "type": "integer",
355
- "minimum": 0
356
- },
357
- "timedOut": {
358
- "type": "integer",
359
- "minimum": 0
360
- },
361
- "comparisons": {
362
- "type": "array",
363
- "items": {
364
- "type": "object",
365
- "required": [
366
- "comparisonId",
367
- "caseId",
368
- "runIndex",
369
- "paired",
370
- "comparable"
371
- ],
372
- "properties": {
373
- "comparisonId": {
374
- "type": "string",
375
- "minLength": 1
376
- },
377
- "caseId": {
378
- "type": "string",
379
- "minLength": 1
380
- },
381
- "runIndex": {
382
- "type": "integer",
383
- "minimum": 1
384
- },
385
- "paired": {
386
- "type": "boolean"
387
- },
388
- "comparable": {
389
- "type": "boolean"
390
- }
391
- },
392
- "additionalProperties": false
393
- }
394
- }
395
- },
396
- "additionalProperties": false
397
- },
398
- "limitations": {
399
- "type": "array",
400
- "items": {
401
- "type": "string",
402
- "minLength": 1
403
- },
404
- "uniqueItems": true
405
- }
406
- },
407
- "additionalProperties": false
408
- }
package/dist/schemas/eval-runtime-capabilities.schema.json DELETED
@@ -1,85 +0,0 @@
1
- {
2
- "$schema": "https://json-schema.org/draft/2020-12/schema",
3
- "$id": "https://agent-skill-toolkit.invalid/schemas/eval-runtime-capabilities.schema.json",
4
- "title": "EvalRuntimeCapabilities",
5
- "type": "object",
6
- "required": [
7
- "schemaVersion",
8
- "provider",
9
- "providerVersion",
10
- "capabilities",
11
- "limitations"
12
- ],
13
- "properties": {
14
- "schemaVersion": {
15
- "const": 1
16
- },
17
- "provider": {
18
- "type": "string",
19
- "minLength": 1
20
- },
21
- "providerVersion": {
22
- "type": "string",
23
- "minLength": 1
24
- },
25
- "capabilities": {
26
- "type": "object",
27
- "required": [
28
- "triggerDiscovery",
29
- "skillPathExecution",
30
- "previousVersionBaseline",
31
- "withoutSkillBaseline",
32
- "cleanContext",
33
- "parallelRuns",
34
- "toolTrace",
35
- "tokenUsage",
36
- "duration",
37
- "sandbox",
38
- "materialization"
39
- ],
40
- "properties": {
41
- "triggerDiscovery": {
42
- "type": "boolean"
43
- },
44
- "skillPathExecution": {
45
- "type": "boolean"
46
- },
47
- "previousVersionBaseline": {
48
- "type": "boolean"
49
- },
50
- "withoutSkillBaseline": {
51
- "type": "boolean"
52
- },
53
- "cleanContext": {
54
- "type": "boolean"
55
- },
56
- "parallelRuns": {
57
- "type": "boolean"
58
- },
59
- "toolTrace": {
60
- "type": "boolean"
61
- },
62
- "tokenUsage": {
63
- "type": "boolean"
64
- },
65
- "duration": {
66
- "type": "boolean"
67
- },
68
- "sandbox": {
69
- "type": "boolean"
70
- },
71
- "materialization": {
72
- "type": "boolean"
73
- }
74
- },
75
- "additionalProperties": false
76
- },
77
- "limitations": {
78
- "type": "array",
79
- "items": {
80
- "type": "string"
81
- }
82
- }
83
- },
84
- "additionalProperties": false
85
- }