@fre4x/benchmark 1.1.0-beta.1 → 1.1.0-beta.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +122 -18
- package/dist/fallback-catalog.json +613 -0
- package/dist/index.js +1218 -343
- package/package.json +3 -3
|
@@ -0,0 +1,613 @@
|
|
|
1
|
+
[
|
|
2
|
+
{
|
|
3
|
+
"challenge_id": "code_ci_repair_drill",
|
|
4
|
+
"benchmark_id": "deterministic_mix",
|
|
5
|
+
"family": "code",
|
|
6
|
+
"runner_kind": "code_runner",
|
|
7
|
+
"title": "Code CI Repair Drill",
|
|
8
|
+
"description": "Read incident assets and answer with deterministic, machine-checkable outputs.",
|
|
9
|
+
"version": "builtin-v2",
|
|
10
|
+
"source": "builtin",
|
|
11
|
+
"tasks": [
|
|
12
|
+
{
|
|
13
|
+
"task_id": "code-ci-1",
|
|
14
|
+
"title": "Classify the CI failure",
|
|
15
|
+
"prompt": "Read the attached incident report and failing test summary.\nReturn compact JSON with exactly these fields:\n{\"root_cause\":\"...\",\"failing_test_count\":number}\nThe root_cause must be one of: cache_key, timeout_config, path_join.",
|
|
16
|
+
"response_format": "json",
|
|
17
|
+
"difficulty": 1,
|
|
18
|
+
"assets": [
|
|
19
|
+
{
|
|
20
|
+
"asset_id": "code-ci-incident",
|
|
21
|
+
"name": "incident_report.md",
|
|
22
|
+
"mime_type": "text/markdown",
|
|
23
|
+
"transport": "inline",
|
|
24
|
+
"materialization": "inline",
|
|
25
|
+
"encoding": "utf8",
|
|
26
|
+
"description": "Postmortem summary for the broken CI run.",
|
|
27
|
+
"content": "# Incident report\n\n- Regression introduced after lowering `timeout_ms` from 1500 to 500.\n- Remote artifact fetch now fails on slower runners.\n- No cache key or path joining regressions were found."
|
|
28
|
+
},
|
|
29
|
+
{
|
|
30
|
+
"asset_id": "code-ci-tests",
|
|
31
|
+
"name": "failing_tests.txt",
|
|
32
|
+
"mime_type": "text/plain",
|
|
33
|
+
"transport": "inline",
|
|
34
|
+
"materialization": "inline",
|
|
35
|
+
"encoding": "utf8",
|
|
36
|
+
"description": "Names of the failing tests from CI.",
|
|
37
|
+
"content": "FAIL retry fetches artifact after timeout\nFAIL increases timeout for cold runner\nFAIL preserves timeout override in deploy step"
|
|
38
|
+
}
|
|
39
|
+
],
|
|
40
|
+
"checkers": [
|
|
41
|
+
{
|
|
42
|
+
"checker_id": "code-ci-root-cause",
|
|
43
|
+
"kind": "json_field_equals",
|
|
44
|
+
"field": "root_cause",
|
|
45
|
+
"expected": "timeout_config"
|
|
46
|
+
},
|
|
47
|
+
{
|
|
48
|
+
"checker_id": "code-ci-failing-count",
|
|
49
|
+
"kind": "json_field_equals",
|
|
50
|
+
"field": "failing_test_count",
|
|
51
|
+
"expected": 3
|
|
52
|
+
},
|
|
53
|
+
{
|
|
54
|
+
"checker_id": "code-ci-materialized-assets",
|
|
55
|
+
"kind": "runner_fact_equals",
|
|
56
|
+
"fact": "materialized_asset_count",
|
|
57
|
+
"expected": 0
|
|
58
|
+
},
|
|
59
|
+
{
|
|
60
|
+
"checker_id": "code-ci-log",
|
|
61
|
+
"kind": "runner_log_contains_text",
|
|
62
|
+
"terms": ["prepared code workspace inputs"]
|
|
63
|
+
}
|
|
64
|
+
]
|
|
65
|
+
},
|
|
66
|
+
{
|
|
67
|
+
"task_id": "code-ci-2",
|
|
68
|
+
"title": "Name the config key to patch",
|
|
69
|
+
"prompt": "Read the patch note and return only the configuration key that must be raised.\nDo not return prose.",
|
|
70
|
+
"response_format": "text",
|
|
71
|
+
"difficulty": 1,
|
|
72
|
+
"assets": [
|
|
73
|
+
{
|
|
74
|
+
"asset_id": "code-ci-patch-note",
|
|
75
|
+
"name": "patch_note.txt",
|
|
76
|
+
"mime_type": "text/plain",
|
|
77
|
+
"transport": "inline",
|
|
78
|
+
"materialization": "inline",
|
|
79
|
+
"encoding": "utf8",
|
|
80
|
+
"description": "Patch hint for the CI timeout regression.",
|
|
81
|
+
"content": "Increase timeout_ms to 1500 so slow runners can finish the artifact fetch."
|
|
82
|
+
}
|
|
83
|
+
],
|
|
84
|
+
"checkers": [
|
|
85
|
+
{
|
|
86
|
+
"checker_id": "code-ci-config-key",
|
|
87
|
+
"kind": "exact_text",
|
|
88
|
+
"expected": "timeout_ms",
|
|
89
|
+
"case_sensitive": true
|
|
90
|
+
}
|
|
91
|
+
]
|
|
92
|
+
}
|
|
93
|
+
]
|
|
94
|
+
},
|
|
95
|
+
{
|
|
96
|
+
"challenge_id": "web_ops_order_reconciliation",
|
|
97
|
+
"benchmark_id": "deterministic_mix",
|
|
98
|
+
"family": "web",
|
|
99
|
+
"runner_kind": "browser_runner",
|
|
100
|
+
"title": "Web Ops Order Reconciliation",
|
|
101
|
+
"description": "Parse a simulated operations dashboard and return deterministic JSON fields.",
|
|
102
|
+
"version": "builtin-v2",
|
|
103
|
+
"source": "builtin",
|
|
104
|
+
"tasks": [
|
|
105
|
+
{
|
|
106
|
+
"task_id": "web-ops-1",
|
|
107
|
+
"title": "Extract dashboard status",
|
|
108
|
+
"prompt": "Inspect the attached HTML snapshot.\nReturn compact JSON with exactly these fields:\n{\"order_id\":\"...\",\"status\":\"...\",\"eta\":\"...\"}",
|
|
109
|
+
"response_format": "json",
|
|
110
|
+
"difficulty": 2,
|
|
111
|
+
"assets": [
|
|
112
|
+
{
|
|
113
|
+
"asset_id": "web-ops-dashboard",
|
|
114
|
+
"name": "dashboard.html",
|
|
115
|
+
"mime_type": "text/html",
|
|
116
|
+
"transport": "inline",
|
|
117
|
+
"materialization": "workspace_file",
|
|
118
|
+
"encoding": "utf8",
|
|
119
|
+
"description": "Static DOM snapshot of the fulfillment dashboard.",
|
|
120
|
+
"content": "<section id=\"orders\">\n <article data-order-id=\"SO-2048\">\n <h2>Shipment SO-2048</h2>\n <p class=\"status\">Delayed</p>\n <p class=\"eta\">ETA: 2026-05-14</p>\n </article>\n</section>"
|
|
121
|
+
}
|
|
122
|
+
],
|
|
123
|
+
"checkers": [
|
|
124
|
+
{
|
|
125
|
+
"checker_id": "web-order-id",
|
|
126
|
+
"kind": "json_field_equals",
|
|
127
|
+
"field": "order_id",
|
|
128
|
+
"expected": "SO-2048"
|
|
129
|
+
},
|
|
130
|
+
{
|
|
131
|
+
"checker_id": "web-status",
|
|
132
|
+
"kind": "json_field_equals",
|
|
133
|
+
"field": "status",
|
|
134
|
+
"expected": "Delayed"
|
|
135
|
+
},
|
|
136
|
+
{
|
|
137
|
+
"checker_id": "web-eta",
|
|
138
|
+
"kind": "json_field_equals",
|
|
139
|
+
"field": "eta",
|
|
140
|
+
"expected": "2026-05-14"
|
|
141
|
+
},
|
|
142
|
+
{
|
|
143
|
+
"checker_id": "web-materialized-assets",
|
|
144
|
+
"kind": "runner_fact_equals",
|
|
145
|
+
"fact": "materialized_asset_count",
|
|
146
|
+
"expected": 1
|
|
147
|
+
},
|
|
148
|
+
{
|
|
149
|
+
"checker_id": "web-log",
|
|
150
|
+
"kind": "runner_log_contains_text",
|
|
151
|
+
"terms": ["prepared browser snapshot workspace"]
|
|
152
|
+
}
|
|
153
|
+
]
|
|
154
|
+
}
|
|
155
|
+
]
|
|
156
|
+
},
|
|
157
|
+
{
|
|
158
|
+
"challenge_id": "os_release_gate_review",
|
|
159
|
+
"benchmark_id": "deterministic_mix",
|
|
160
|
+
"family": "os",
|
|
161
|
+
"runner_kind": "os_runner",
|
|
162
|
+
"title": "OS Release Gate Review",
|
|
163
|
+
"description": "Read filesystem-style assets and answer with deterministic release decisions.",
|
|
164
|
+
"version": "builtin-v2",
|
|
165
|
+
"source": "builtin",
|
|
166
|
+
"tasks": [
|
|
167
|
+
{
|
|
168
|
+
"task_id": "os-gate-1",
|
|
169
|
+
"title": "Decide the release action",
|
|
170
|
+
"prompt": "Read the attached release notes and audit log.\nReturn only the final action in lowercase.\nExpected format: two words.",
|
|
171
|
+
"response_format": "text",
|
|
172
|
+
"difficulty": 2,
|
|
173
|
+
"assets": [
|
|
174
|
+
{
|
|
175
|
+
"asset_id": "os-release-notes",
|
|
176
|
+
"name": "release_notes.txt",
|
|
177
|
+
"mime_type": "text/plain",
|
|
178
|
+
"transport": "inline",
|
|
179
|
+
"materialization": "workspace_file",
|
|
180
|
+
"encoding": "utf8",
|
|
181
|
+
"description": "Release checklist summary.",
|
|
182
|
+
"content": "release=2026.05.14\ndatabase_migration=pending\nsecurity_scan=pass\nrollback_bundle=ready"
|
|
183
|
+
},
|
|
184
|
+
{
|
|
185
|
+
"asset_id": "os-audit-log",
|
|
186
|
+
"name": "audit.log",
|
|
187
|
+
"mime_type": "text/plain",
|
|
188
|
+
"transport": "inline",
|
|
189
|
+
"materialization": "workspace_file",
|
|
190
|
+
"encoding": "utf8",
|
|
191
|
+
"description": "Audit events from the release gate.",
|
|
192
|
+
"content": "[warn] migration step not executed\n[info] rollback artifacts verified\n[decision] rollback release"
|
|
193
|
+
}
|
|
194
|
+
],
|
|
195
|
+
"checkers": [
|
|
196
|
+
{
|
|
197
|
+
"checker_id": "os-release-action",
|
|
198
|
+
"kind": "normalized_text",
|
|
199
|
+
"expected": "rollback release"
|
|
200
|
+
},
|
|
201
|
+
{
|
|
202
|
+
"checker_id": "os-materialized-assets",
|
|
203
|
+
"kind": "runner_fact_equals",
|
|
204
|
+
"fact": "materialized_asset_count",
|
|
205
|
+
"expected": 2
|
|
206
|
+
},
|
|
207
|
+
{
|
|
208
|
+
"checker_id": "os-log",
|
|
209
|
+
"kind": "runner_log_contains_text",
|
|
210
|
+
"terms": ["prepared os workspace inventory"]
|
|
211
|
+
}
|
|
212
|
+
]
|
|
213
|
+
}
|
|
214
|
+
]
|
|
215
|
+
},
|
|
216
|
+
{
|
|
217
|
+
"challenge_id": "code_repository_incident_pack",
|
|
218
|
+
"benchmark_id": "external_mix",
|
|
219
|
+
"family": "code",
|
|
220
|
+
"runner_kind": "code_runner",
|
|
221
|
+
"title": "Code Repository Incident Pack",
|
|
222
|
+
"description": "Repository incident triage tasks for deterministic code evaluation.",
|
|
223
|
+
"version": "external-v1",
|
|
224
|
+
"source": "external-example",
|
|
225
|
+
"tasks": [
|
|
226
|
+
{
|
|
227
|
+
"task_id": "code-repo-1",
|
|
228
|
+
"title": "Identify regression owner",
|
|
229
|
+
"prompt": "Read the incident brief and return compact JSON with exactly these fields: {\"repository\":\"...\",\"root_cause\":\"...\"}. The root_cause must be one of: retry_budget, timeout_config, cache_key.",
|
|
230
|
+
"response_format": "json",
|
|
231
|
+
"difficulty": 1,
|
|
232
|
+
"assets": [
|
|
233
|
+
{
|
|
234
|
+
"asset_id": "code-repo-1-brief",
|
|
235
|
+
"name": "incident.md",
|
|
236
|
+
"mime_type": "text/markdown",
|
|
237
|
+
"transport": "inline",
|
|
238
|
+
"materialization": "inline",
|
|
239
|
+
"encoding": "utf8",
|
|
240
|
+
"description": "Repository incident summary.",
|
|
241
|
+
"content": "# Repo incident\n\nRepository: api-gateway\nRegression traced to timeout_config drift after a config cleanup.\nNo retry_budget or cache_key regressions were found."
|
|
242
|
+
}
|
|
243
|
+
],
|
|
244
|
+
"checkers": [
|
|
245
|
+
{
|
|
246
|
+
"checker_id": "code-repo-1-repository",
|
|
247
|
+
"kind": "json_field_equals",
|
|
248
|
+
"field": "repository",
|
|
249
|
+
"expected": "api-gateway"
|
|
250
|
+
},
|
|
251
|
+
{
|
|
252
|
+
"checker_id": "code-repo-1-cause",
|
|
253
|
+
"kind": "json_field_equals",
|
|
254
|
+
"field": "root_cause",
|
|
255
|
+
"expected": "timeout_config"
|
|
256
|
+
},
|
|
257
|
+
{
|
|
258
|
+
"checker_id": "code-repo-1-log",
|
|
259
|
+
"kind": "runner_log_contains_text",
|
|
260
|
+
"terms": ["prepared code workspace inputs"]
|
|
261
|
+
}
|
|
262
|
+
]
|
|
263
|
+
},
|
|
264
|
+
{
|
|
265
|
+
"task_id": "code-repo-2",
|
|
266
|
+
"title": "Return the hotfix branch",
|
|
267
|
+
"prompt": "Read the handoff note and return only the hotfix branch name.",
|
|
268
|
+
"response_format": "text",
|
|
269
|
+
"difficulty": 1,
|
|
270
|
+
"assets": [
|
|
271
|
+
{
|
|
272
|
+
"asset_id": "code-repo-2-note",
|
|
273
|
+
"name": "handoff.txt",
|
|
274
|
+
"mime_type": "text/plain",
|
|
275
|
+
"transport": "inline",
|
|
276
|
+
"materialization": "inline",
|
|
277
|
+
"encoding": "utf8",
|
|
278
|
+
"description": "Hotfix branch handoff.",
|
|
279
|
+
"content": "Create hotfix/timeout-rollback before restarting deployment."
|
|
280
|
+
}
|
|
281
|
+
],
|
|
282
|
+
"checkers": [
|
|
283
|
+
{
|
|
284
|
+
"checker_id": "code-repo-2-branch",
|
|
285
|
+
"kind": "exact_text",
|
|
286
|
+
"expected": "hotfix/timeout-rollback",
|
|
287
|
+
"case_sensitive": true
|
|
288
|
+
}
|
|
289
|
+
]
|
|
290
|
+
},
|
|
291
|
+
{
|
|
292
|
+
"task_id": "code-repo-3",
|
|
293
|
+
"title": "Extract severity metadata",
|
|
294
|
+
"prompt": "Inspect the release memo and return compact JSON with exactly these fields: {\"owner\":\"...\",\"severity\":\"...\"}.",
|
|
295
|
+
"response_format": "json",
|
|
296
|
+
"difficulty": 2,
|
|
297
|
+
"assets": [
|
|
298
|
+
{
|
|
299
|
+
"asset_id": "code-repo-3-memo",
|
|
300
|
+
"name": "release_memo.txt",
|
|
301
|
+
"mime_type": "text/plain",
|
|
302
|
+
"transport": "inline",
|
|
303
|
+
"materialization": "workspace_file",
|
|
304
|
+
"encoding": "utf8",
|
|
305
|
+
"description": "Release severity memo.",
|
|
306
|
+
"content": "Owner: platform-release\nSeverity: high\nRollback status: ready"
|
|
307
|
+
}
|
|
308
|
+
],
|
|
309
|
+
"checkers": [
|
|
310
|
+
{
|
|
311
|
+
"checker_id": "code-repo-3-owner",
|
|
312
|
+
"kind": "json_field_equals",
|
|
313
|
+
"field": "owner",
|
|
314
|
+
"expected": "platform-release"
|
|
315
|
+
},
|
|
316
|
+
{
|
|
317
|
+
"checker_id": "code-repo-3-severity",
|
|
318
|
+
"kind": "json_field_equals",
|
|
319
|
+
"field": "severity",
|
|
320
|
+
"expected": "high"
|
|
321
|
+
},
|
|
322
|
+
{
|
|
323
|
+
"checker_id": "code-repo-3-assets",
|
|
324
|
+
"kind": "runner_fact_equals",
|
|
325
|
+
"fact": "materialized_asset_count",
|
|
326
|
+
"expected": 1
|
|
327
|
+
}
|
|
328
|
+
]
|
|
329
|
+
}
|
|
330
|
+
]
|
|
331
|
+
},
|
|
332
|
+
{
|
|
333
|
+
"challenge_id": "web_customer_portal_pack",
|
|
334
|
+
"benchmark_id": "external_mix",
|
|
335
|
+
"family": "web",
|
|
336
|
+
"runner_kind": "browser_runner",
|
|
337
|
+
"title": "Web Customer Portal Pack",
|
|
338
|
+
"description": "Portal extraction tasks using deterministic DOM snapshots.",
|
|
339
|
+
"version": "external-v1",
|
|
340
|
+
"source": "external-example",
|
|
341
|
+
"tasks": [
|
|
342
|
+
{
|
|
343
|
+
"task_id": "web-portal-1",
|
|
344
|
+
"title": "Extract queue ownership",
|
|
345
|
+
"prompt": "Read the HTML snapshot and return compact JSON with exactly these fields: {\"ticket_id\":\"...\",\"priority\":\"...\",\"owner\":\"...\"}.",
|
|
346
|
+
"response_format": "json",
|
|
347
|
+
"difficulty": 2,
|
|
348
|
+
"assets": [
|
|
349
|
+
{
|
|
350
|
+
"asset_id": "web-portal-1-html",
|
|
351
|
+
"name": "queue.html",
|
|
352
|
+
"mime_type": "text/html",
|
|
353
|
+
"transport": "inline",
|
|
354
|
+
"materialization": "workspace_file",
|
|
355
|
+
"encoding": "utf8",
|
|
356
|
+
"description": "Support queue HTML snapshot.",
|
|
357
|
+
"content": "<div class=\"ticket\" data-ticket-id=\"T-4401\"><span class=\"priority\">urgent</span><span class=\"owner\">mina.choi</span></div>"
|
|
358
|
+
}
|
|
359
|
+
],
|
|
360
|
+
"checkers": [
|
|
361
|
+
{
|
|
362
|
+
"checker_id": "web-portal-1-ticket",
|
|
363
|
+
"kind": "json_field_equals",
|
|
364
|
+
"field": "ticket_id",
|
|
365
|
+
"expected": "T-4401"
|
|
366
|
+
},
|
|
367
|
+
{
|
|
368
|
+
"checker_id": "web-portal-1-priority",
|
|
369
|
+
"kind": "json_field_equals",
|
|
370
|
+
"field": "priority",
|
|
371
|
+
"expected": "urgent"
|
|
372
|
+
},
|
|
373
|
+
{
|
|
374
|
+
"checker_id": "web-portal-1-owner",
|
|
375
|
+
"kind": "json_field_equals",
|
|
376
|
+
"field": "owner",
|
|
377
|
+
"expected": "mina.choi"
|
|
378
|
+
},
|
|
379
|
+
{
|
|
380
|
+
"checker_id": "web-portal-1-log",
|
|
381
|
+
"kind": "runner_log_contains_text",
|
|
382
|
+
"terms": ["prepared browser snapshot workspace"]
|
|
383
|
+
}
|
|
384
|
+
]
|
|
385
|
+
},
|
|
386
|
+
{
|
|
387
|
+
"task_id": "web-portal-2",
|
|
388
|
+
"title": "Return the account state",
|
|
389
|
+
"prompt": "Inspect the account banner and return only the account state phrase in lowercase.",
|
|
390
|
+
"response_format": "text",
|
|
391
|
+
"difficulty": 1,
|
|
392
|
+
"assets": [
|
|
393
|
+
{
|
|
394
|
+
"asset_id": "web-portal-2-banner",
|
|
395
|
+
"name": "banner.html",
|
|
396
|
+
"mime_type": "text/html",
|
|
397
|
+
"transport": "inline",
|
|
398
|
+
"materialization": "inline",
|
|
399
|
+
"encoding": "utf8",
|
|
400
|
+
"description": "Account banner snapshot.",
|
|
401
|
+
"content": "<aside class=\"banner\">Account state: billing hold</aside>"
|
|
402
|
+
}
|
|
403
|
+
],
|
|
404
|
+
"checkers": [
|
|
405
|
+
{
|
|
406
|
+
"checker_id": "web-portal-2-state",
|
|
407
|
+
"kind": "normalized_text",
|
|
408
|
+
"expected": "billing hold"
|
|
409
|
+
}
|
|
410
|
+
]
|
|
411
|
+
},
|
|
412
|
+
{
|
|
413
|
+
"task_id": "web-portal-3",
|
|
414
|
+
"title": "Extract invoice status",
|
|
415
|
+
"prompt": "Return compact JSON with exactly these fields after reading the invoice card: {\"invoice_id\":\"...\",\"status\":\"...\"}.",
|
|
416
|
+
"response_format": "json",
|
|
417
|
+
"difficulty": 1,
|
|
418
|
+
"assets": [
|
|
419
|
+
{
|
|
420
|
+
"asset_id": "web-portal-3-card",
|
|
421
|
+
"name": "invoice.html",
|
|
422
|
+
"mime_type": "text/html",
|
|
423
|
+
"transport": "inline",
|
|
424
|
+
"materialization": "workspace_file",
|
|
425
|
+
"encoding": "utf8",
|
|
426
|
+
"description": "Invoice card HTML.",
|
|
427
|
+
"content": "<article class=\"invoice\" data-invoice-id=\"INV-991\"><span class=\"status\">Escalated</span></article>"
|
|
428
|
+
}
|
|
429
|
+
],
|
|
430
|
+
"checkers": [
|
|
431
|
+
{
|
|
432
|
+
"checker_id": "web-portal-3-id",
|
|
433
|
+
"kind": "json_field_equals",
|
|
434
|
+
"field": "invoice_id",
|
|
435
|
+
"expected": "INV-991"
|
|
436
|
+
},
|
|
437
|
+
{
|
|
438
|
+
"checker_id": "web-portal-3-status",
|
|
439
|
+
"kind": "json_field_equals",
|
|
440
|
+
"field": "status",
|
|
441
|
+
"expected": "Escalated"
|
|
442
|
+
},
|
|
443
|
+
{
|
|
444
|
+
"checker_id": "web-portal-3-assets",
|
|
445
|
+
"kind": "runner_fact_equals",
|
|
446
|
+
"fact": "materialized_asset_count",
|
|
447
|
+
"expected": 1
|
|
448
|
+
}
|
|
449
|
+
]
|
|
450
|
+
}
|
|
451
|
+
]
|
|
452
|
+
},
|
|
453
|
+
{
|
|
454
|
+
"challenge_id": "os_release_ops_pack",
|
|
455
|
+
"benchmark_id": "external_mix",
|
|
456
|
+
"family": "os",
|
|
457
|
+
"runner_kind": "os_runner",
|
|
458
|
+
"title": "OS Release Ops Pack",
|
|
459
|
+
"description": "Filesystem and audit-log based deterministic operations tasks.",
|
|
460
|
+
"version": "external-v1",
|
|
461
|
+
"source": "external-example",
|
|
462
|
+
"tasks": [
|
|
463
|
+
{
|
|
464
|
+
"task_id": "os-ops-1",
|
|
465
|
+
"title": "Decide deployment action",
|
|
466
|
+
"prompt": "Read the release note and return only the final deployment action in lowercase.",
|
|
467
|
+
"response_format": "text",
|
|
468
|
+
"difficulty": 2,
|
|
469
|
+
"assets": [
|
|
470
|
+
{
|
|
471
|
+
"asset_id": "os-ops-1-release",
|
|
472
|
+
"name": "release.txt",
|
|
473
|
+
"mime_type": "text/plain",
|
|
474
|
+
"transport": "inline",
|
|
475
|
+
"materialization": "workspace_file",
|
|
476
|
+
"encoding": "utf8",
|
|
477
|
+
"description": "Release action summary.",
|
|
478
|
+
"content": "deploy_window=open\nmigration=blocked\naction=pause deploy"
|
|
479
|
+
}
|
|
480
|
+
],
|
|
481
|
+
"checkers": [
|
|
482
|
+
{
|
|
483
|
+
"checker_id": "os-ops-1-action",
|
|
484
|
+
"kind": "normalized_text",
|
|
485
|
+
"expected": "pause deploy"
|
|
486
|
+
},
|
|
487
|
+
{
|
|
488
|
+
"checker_id": "os-ops-1-assets",
|
|
489
|
+
"kind": "runner_fact_equals",
|
|
490
|
+
"fact": "materialized_asset_count",
|
|
491
|
+
"expected": 1
|
|
492
|
+
},
|
|
493
|
+
{
|
|
494
|
+
"checker_id": "os-ops-1-log",
|
|
495
|
+
"kind": "runner_log_contains_text",
|
|
496
|
+
"terms": ["prepared os workspace inventory"]
|
|
497
|
+
}
|
|
498
|
+
]
|
|
499
|
+
},
|
|
500
|
+
{
|
|
501
|
+
"task_id": "os-ops-2",
|
|
502
|
+
"title": "Extract cleanup order",
|
|
503
|
+
"prompt": "Return compact JSON with exactly these fields after reading the session audit: {\"cleanup_action\":\"...\",\"stale_sessions\":number}.",
|
|
504
|
+
"response_format": "json",
|
|
505
|
+
"difficulty": 2,
|
|
506
|
+
"assets": [
|
|
507
|
+
{
|
|
508
|
+
"asset_id": "os-ops-2-audit",
|
|
509
|
+
"name": "session_audit.log",
|
|
510
|
+
"mime_type": "text/plain",
|
|
511
|
+
"transport": "inline",
|
|
512
|
+
"materialization": "workspace_file",
|
|
513
|
+
"encoding": "utf8",
|
|
514
|
+
"description": "Session cleanup audit log.",
|
|
515
|
+
"content": "cleanup_action=rotate tokens\nstale_sessions=4\nrollback_ready=yes"
|
|
516
|
+
}
|
|
517
|
+
],
|
|
518
|
+
"checkers": [
|
|
519
|
+
{
|
|
520
|
+
"checker_id": "os-ops-2-action",
|
|
521
|
+
"kind": "json_field_equals",
|
|
522
|
+
"field": "cleanup_action",
|
|
523
|
+
"expected": "rotate tokens"
|
|
524
|
+
},
|
|
525
|
+
{
|
|
526
|
+
"checker_id": "os-ops-2-count",
|
|
527
|
+
"kind": "json_field_equals",
|
|
528
|
+
"field": "stale_sessions",
|
|
529
|
+
"expected": 4
|
|
530
|
+
}
|
|
531
|
+
]
|
|
532
|
+
}
|
|
533
|
+
]
|
|
534
|
+
},
|
|
535
|
+
{
|
|
536
|
+
"challenge_id": "code_data_quality_pack",
|
|
537
|
+
"benchmark_id": "external_mix",
|
|
538
|
+
"family": "code",
|
|
539
|
+
"runner_kind": "code_runner",
|
|
540
|
+
"title": "Code Data Quality Pack",
|
|
541
|
+
"description": "Data pipeline and schema audit tasks for deterministic code-oriented evaluation.",
|
|
542
|
+
"version": "external-v1",
|
|
543
|
+
"source": "external-example",
|
|
544
|
+
"tasks": [
|
|
545
|
+
{
|
|
546
|
+
"task_id": "code-data-1",
|
|
547
|
+
"title": "Extract failed row count",
|
|
548
|
+
"prompt": "Read the pipeline report and return compact JSON with exactly these fields: {\"pipeline\":\"...\",\"failed_rows\":number}.",
|
|
549
|
+
"response_format": "json",
|
|
550
|
+
"difficulty": 1,
|
|
551
|
+
"assets": [
|
|
552
|
+
{
|
|
553
|
+
"asset_id": "code-data-1-report",
|
|
554
|
+
"name": "pipeline_report.txt",
|
|
555
|
+
"mime_type": "text/plain",
|
|
556
|
+
"transport": "inline",
|
|
557
|
+
"materialization": "inline",
|
|
558
|
+
"encoding": "utf8",
|
|
559
|
+
"description": "Pipeline quality report.",
|
|
560
|
+
"content": "pipeline=nightly-orders\nfailed_rows=17\nschema_state=compatible"
|
|
561
|
+
}
|
|
562
|
+
],
|
|
563
|
+
"checkers": [
|
|
564
|
+
{
|
|
565
|
+
"checker_id": "code-data-1-pipeline",
|
|
566
|
+
"kind": "json_field_equals",
|
|
567
|
+
"field": "pipeline",
|
|
568
|
+
"expected": "nightly-orders"
|
|
569
|
+
},
|
|
570
|
+
{
|
|
571
|
+
"checker_id": "code-data-1-rows",
|
|
572
|
+
"kind": "json_field_equals",
|
|
573
|
+
"field": "failed_rows",
|
|
574
|
+
"expected": 17
|
|
575
|
+
}
|
|
576
|
+
]
|
|
577
|
+
},
|
|
578
|
+
{
|
|
579
|
+
"task_id": "code-data-2",
|
|
580
|
+
"title": "Return schema key",
|
|
581
|
+
"prompt": "Inspect the migration note and return only the schema key that must be pinned.",
|
|
582
|
+
"response_format": "text",
|
|
583
|
+
"difficulty": 1,
|
|
584
|
+
"assets": [
|
|
585
|
+
{
|
|
586
|
+
"asset_id": "code-data-2-note",
|
|
587
|
+
"name": "migration_note.md",
|
|
588
|
+
"mime_type": "text/markdown",
|
|
589
|
+
"transport": "inline",
|
|
590
|
+
"materialization": "workspace_file",
|
|
591
|
+
"encoding": "utf8",
|
|
592
|
+
"description": "Migration note for schema pinning.",
|
|
593
|
+
"content": "Pin schema_version before replaying the backlog."
|
|
594
|
+
}
|
|
595
|
+
],
|
|
596
|
+
"checkers": [
|
|
597
|
+
{
|
|
598
|
+
"checker_id": "code-data-2-key",
|
|
599
|
+
"kind": "exact_text",
|
|
600
|
+
"expected": "schema_version",
|
|
601
|
+
"case_sensitive": true
|
|
602
|
+
},
|
|
603
|
+
{
|
|
604
|
+
"checker_id": "code-data-2-assets",
|
|
605
|
+
"kind": "runner_fact_equals",
|
|
606
|
+
"fact": "materialized_asset_count",
|
|
607
|
+
"expected": 1
|
|
608
|
+
}
|
|
609
|
+
]
|
|
610
|
+
}
|
|
611
|
+
]
|
|
612
|
+
}
|
|
613
|
+
]
|