modelrig 0.11.1 → 0.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (182) hide show
  1. package/CHANGELOG.md +34 -0
  2. package/dist/adapters/anthropic.d.ts +4 -4
  3. package/dist/adapters/anthropic.d.ts.map +1 -1
  4. package/dist/adapters/anthropic.js +8 -1
  5. package/dist/adapters/deepseek.d.ts +2 -4
  6. package/dist/adapters/deepseek.d.ts.map +1 -1
  7. package/dist/adapters/deepseek.js +9 -5
  8. package/dist/adapters/gemini.d.ts +4 -1
  9. package/dist/adapters/gemini.d.ts.map +1 -1
  10. package/dist/adapters/gemini.js +16 -1
  11. package/dist/adapters/grok.d.ts +2 -4
  12. package/dist/adapters/grok.d.ts.map +1 -1
  13. package/dist/adapters/grok.js +8 -4
  14. package/dist/adapters/history-mapping.d.ts +126 -0
  15. package/dist/adapters/history-mapping.d.ts.map +1 -0
  16. package/dist/adapters/history-mapping.js +230 -0
  17. package/dist/adapters/openai-compat.d.ts +2 -4
  18. package/dist/adapters/openai-compat.d.ts.map +1 -1
  19. package/dist/adapters/openai-compat.js +9 -5
  20. package/dist/adapters/openai.d.ts +2 -4
  21. package/dist/adapters/openai.d.ts.map +1 -1
  22. package/dist/adapters/openai.js +8 -4
  23. package/dist/adapters/responses-api.d.ts.map +1 -1
  24. package/dist/adapters/responses-api.js +9 -4
  25. package/dist/artifacts/artifact.d.ts.map +1 -1
  26. package/dist/artifacts/artifact.js +79 -0
  27. package/dist/artifacts/conversation.d.ts +128 -0
  28. package/dist/artifacts/conversation.d.ts.map +1 -0
  29. package/dist/artifacts/conversation.js +172 -0
  30. package/dist/artifacts/index.d.ts +1 -1
  31. package/dist/artifacts/index.d.ts.map +1 -1
  32. package/dist/artifacts/media.d.ts +46 -0
  33. package/dist/artifacts/media.d.ts.map +1 -0
  34. package/dist/artifacts/media.js +88 -0
  35. package/dist/artifacts/run-context.d.ts +61 -2
  36. package/dist/artifacts/run-context.d.ts.map +1 -1
  37. package/dist/artifacts/run-context.js +183 -12
  38. package/dist/artifacts/scrub.d.ts +3 -0
  39. package/dist/artifacts/scrub.d.ts.map +1 -1
  40. package/dist/bakeoff/engine.d.ts.map +1 -1
  41. package/dist/bakeoff/engine.js +34 -7
  42. package/dist/bakeoff/eval-cases.d.ts +5 -0
  43. package/dist/bakeoff/eval-cases.d.ts.map +1 -1
  44. package/dist/bakeoff/eval-cases.js +10 -1
  45. package/dist/bakeoff/replay.d.ts +15 -0
  46. package/dist/bakeoff/replay.d.ts.map +1 -1
  47. package/dist/bakeoff/replay.js +19 -1
  48. package/dist/bakeoff/report.d.ts.map +1 -1
  49. package/dist/bakeoff/report.js +9 -2
  50. package/dist/bakeoff/turn-replay.d.ts +93 -0
  51. package/dist/bakeoff/turn-replay.d.ts.map +1 -0
  52. package/dist/bakeoff/turn-replay.js +150 -0
  53. package/dist/bakeoff/types.d.ts +12 -0
  54. package/dist/bakeoff/types.d.ts.map +1 -1
  55. package/dist/feedback/grade.d.ts +7 -2
  56. package/dist/feedback/grade.d.ts.map +1 -1
  57. package/dist/feedback/grade.js +14 -0
  58. package/dist/feedback/outcome.d.ts +103 -0
  59. package/dist/feedback/outcome.d.ts.map +1 -0
  60. package/dist/feedback/outcome.js +175 -0
  61. package/dist/gateway/history-bridge.d.ts +32 -0
  62. package/dist/gateway/history-bridge.d.ts.map +1 -0
  63. package/dist/gateway/history-bridge.js +64 -0
  64. package/dist/gateway/serve-client.d.ts.map +1 -1
  65. package/dist/gateway/serve-client.js +6 -0
  66. package/dist/index.d.ts +44 -1
  67. package/dist/index.d.ts.map +1 -1
  68. package/dist/index.js +106 -6
  69. package/dist/init/observe.d.ts.map +1 -1
  70. package/dist/init/observe.js +3 -3
  71. package/dist/registry/build.d.ts +4 -0
  72. package/dist/registry/build.d.ts.map +1 -1
  73. package/dist/registry/generation-models.d.ts +68 -0
  74. package/dist/registry/generation-models.d.ts.map +1 -0
  75. package/dist/registry/generation-models.js +168 -0
  76. package/dist/registry/model-kind.d.ts +34 -0
  77. package/dist/registry/model-kind.d.ts.map +1 -0
  78. package/dist/registry/model-kind.js +40 -0
  79. package/dist/registry/pricing-shape.d.ts +29 -1
  80. package/dist/registry/pricing-shape.d.ts.map +1 -1
  81. package/dist/registry/pricing-shape.js +9 -0
  82. package/dist/registry/pricing.d.ts +26 -0
  83. package/dist/registry/pricing.d.ts.map +1 -1
  84. package/dist/registry/pricing.js +29 -0
  85. package/dist/registry/rows.d.ts.map +1 -1
  86. package/dist/registry/rows.js +3 -1
  87. package/dist/registry/unit-pricing.d.ts +32 -0
  88. package/dist/registry/unit-pricing.d.ts.map +1 -0
  89. package/dist/registry/unit-pricing.js +62 -0
  90. package/dist/routes/input-loader.d.ts +38 -0
  91. package/dist/routes/input-loader.d.ts.map +1 -0
  92. package/dist/routes/input-loader.js +80 -0
  93. package/dist/routes/loader.d.ts.map +1 -1
  94. package/dist/routes/loader.js +3 -0
  95. package/dist/routes/resolved-bundle.d.ts +2 -0
  96. package/dist/routes/resolved-bundle.d.ts.map +1 -1
  97. package/dist/routes/resolved-bundle.js +3 -0
  98. package/dist/run/cost.d.ts +28 -0
  99. package/dist/run/cost.d.ts.map +1 -1
  100. package/dist/run/cost.js +74 -0
  101. package/dist/run/dispatch.d.ts +4 -1
  102. package/dist/run/dispatch.d.ts.map +1 -1
  103. package/dist/run/dispatch.js +3 -0
  104. package/dist/run/limits.d.ts +88 -0
  105. package/dist/run/limits.d.ts.map +1 -0
  106. package/dist/run/limits.js +155 -0
  107. package/dist/run/messages.d.ts +199 -0
  108. package/dist/run/messages.d.ts.map +1 -0
  109. package/dist/run/messages.js +430 -0
  110. package/dist/run/raw-finalize.d.ts +5 -2
  111. package/dist/run/raw-finalize.d.ts.map +1 -1
  112. package/dist/run/raw-finalize.js +8 -5
  113. package/dist/run/raw-request.d.ts.map +1 -1
  114. package/dist/run/raw-request.js +2 -1
  115. package/dist/run/raw.d.ts +10 -4
  116. package/dist/run/raw.d.ts.map +1 -1
  117. package/dist/run/raw.js +11 -3
  118. package/dist/run/record-attempt.d.ts.map +1 -1
  119. package/dist/run/record-attempt.js +16 -6
  120. package/dist/run/record-call.d.ts +13 -2
  121. package/dist/run/record-call.d.ts.map +1 -1
  122. package/dist/run/record-call.js +50 -7
  123. package/dist/run/run.d.ts.map +1 -1
  124. package/dist/run/run.js +12 -8
  125. package/dist/telemetry/attempt-meta.d.ts +49 -1
  126. package/dist/telemetry/attempt-meta.d.ts.map +1 -1
  127. package/dist/telemetry/attempt-meta.js +45 -1
  128. package/dist/telemetry/exporter.d.ts.map +1 -1
  129. package/dist/telemetry/exporter.js +2 -1
  130. package/dist/telemetry/grade-mappers.d.ts +1 -1
  131. package/dist/telemetry/grade-mappers.d.ts.map +1 -1
  132. package/dist/telemetry/grade-mappers.js +7 -1
  133. package/dist/telemetry/grade-task.d.ts +49 -0
  134. package/dist/telemetry/grade-task.d.ts.map +1 -0
  135. package/dist/telemetry/grade-task.js +85 -0
  136. package/dist/telemetry/grades-sql.d.ts +16 -1
  137. package/dist/telemetry/grades-sql.d.ts.map +1 -1
  138. package/dist/telemetry/grades-sql.js +64 -4
  139. package/dist/telemetry/ingest-protocol.d.ts +24 -1
  140. package/dist/telemetry/ingest-protocol.d.ts.map +1 -1
  141. package/dist/telemetry/ingest-protocol.js +72 -43
  142. package/dist/telemetry/otel-content.d.ts +31 -0
  143. package/dist/telemetry/otel-content.d.ts.map +1 -0
  144. package/dist/telemetry/otel-content.js +106 -0
  145. package/dist/telemetry/otel-hierarchy.d.ts +57 -0
  146. package/dist/telemetry/otel-hierarchy.d.ts.map +1 -0
  147. package/dist/telemetry/otel-hierarchy.js +155 -0
  148. package/dist/telemetry/otel-normalize.d.ts +15 -1
  149. package/dist/telemetry/otel-normalize.d.ts.map +1 -1
  150. package/dist/telemetry/otel-normalize.js +24 -39
  151. package/dist/telemetry/sinks.d.ts +4 -1
  152. package/dist/telemetry/sinks.d.ts.map +1 -1
  153. package/dist/telemetry/sinks.js +53 -34
  154. package/dist/telemetry/store-migrations.d.ts.map +1 -1
  155. package/dist/telemetry/store-migrations.js +8 -0
  156. package/dist/telemetry/store.d.ts +9 -0
  157. package/dist/telemetry/store.d.ts.map +1 -1
  158. package/dist/telemetry/store.js +10 -0
  159. package/dist/telemetry/uniform-rows.d.ts +43 -0
  160. package/dist/telemetry/uniform-rows.d.ts.map +1 -0
  161. package/dist/telemetry/uniform-rows.js +139 -0
  162. package/dist/types/failures-teaching.d.ts.map +1 -1
  163. package/dist/types/failures-teaching.js +11 -0
  164. package/dist/types/failures.d.ts +1 -1
  165. package/dist/types/failures.d.ts.map +1 -1
  166. package/dist/types/failures.js +2 -0
  167. package/dist/types/provider.d.ts +12 -0
  168. package/dist/types/provider.d.ts.map +1 -1
  169. package/dist/types/route.d.ts +13 -0
  170. package/dist/types/route.d.ts.map +1 -1
  171. package/dist/types/telemetry.d.ts +14 -1
  172. package/dist/types/telemetry.d.ts.map +1 -1
  173. package/docs/agent-skill/SKILL.md +16 -3
  174. package/docs/how-it-fits.md +3 -3
  175. package/docs/instrumentation.md +15 -6
  176. package/docs/llms.txt +55 -7
  177. package/docs/mcp-oracle.md +15 -4
  178. package/docs/quickstart.md +6 -5
  179. package/package.json +4 -2
  180. package/registry/registry.json +456 -0
  181. package/templates/creative-approval.json +81 -0
  182. package/templates/support-conversation.json +82 -0
@@ -21,9 +21,12 @@
21
21
  * v1 wire, zero new rows, zero behaviour change (spec §1.4).
22
22
  */
23
23
  Object.defineProperty(exports, "__esModule", { value: true });
24
+ exports.OPEN_CONVERSATION_IDLE_MS = exports.MAX_OPEN_CONVERSATIONS = void 0;
24
25
  exports.resetRunMetaWarningsForTest = resetRunMetaWarningsForTest;
25
26
  exports.withRunIdTag = withRunIdTag;
26
27
  exports.getActiveRunContext = getActiveRunContext;
28
+ exports.runLimitRefusal = runLimitRefusal;
29
+ exports.enforceRunLimits = enforceRunLimits;
27
30
  exports.isValidStepKey = isValidStepKey;
28
31
  exports.resolveStepKey = resolveStepKey;
29
32
  exports.resetStepKeyWarningsForTest = resetStepKeyWarningsForTest;
@@ -33,6 +36,9 @@ const node_crypto_1 = require("node:crypto");
33
36
  const otel_map_1 = require("../telemetry/otel-map");
34
37
  const attempt_meta_1 = require("../telemetry/attempt-meta");
35
38
  const attempt_meta_2 = require("../telemetry/attempt-meta");
39
+ const conversation_1 = require("./conversation");
40
+ const limits_1 = require("../run/limits");
41
+ const failures_1 = require("../types/failures");
36
42
  /** Once-per-process latch for the run-envelope sanitize warning (Ruling A,
37
43
  * PR #262). Mirrors {@link warnedStepKeys}: a dropped field is named ONCE, so a
38
44
  * mislabelled `actor` teaches without spamming the run loop. */
@@ -82,9 +88,13 @@ const storage = new node_async_hooks_1.AsyncLocalStorage();
82
88
  * (what the `/projects` tree joins on) or, without one, the run's id. Never
83
89
  * overrides a caller-supplied `run_id`; no context ⇒ tags unchanged. */
84
90
  function withRunIdTag(tags, ctx) {
85
- if (ctx === null || (typeof tags.run_id === "string" && tags.run_id !== ""))
91
+ if (ctx === null)
86
92
  return tags;
87
- return { ...tags, run_id: ctx.episodeKey ?? ctx.runId };
93
+ // ARC-1: a conversation's user/subject tags sit UNDER the caller's (caller wins).
94
+ const withDefaults = ctx.defaultTags !== undefined ? { ...ctx.defaultTags, ...tags } : tags;
95
+ if (typeof tags.run_id === "string" && tags.run_id !== "")
96
+ return withDefaults;
97
+ return { ...withDefaults, run_id: ctx.episodeKey ?? ctx.runId };
88
98
  }
89
99
  /** The active run context, or null when none is set (namespace disabled, or no
90
100
  * run.start on this async path). Read by the run loop's auto-step seam. */
@@ -92,6 +102,46 @@ function getActiveRunContext() {
92
102
  const ctx = storage.getStore();
93
103
  return ctx !== undefined && !ctx.ended ? ctx : null;
94
104
  }
105
+ /** ARC-5: the limits recorded on a stored run's RunMeta (snake → camel), or null. */
106
+ function limitsFromMeta(meta) {
107
+ const l = meta?.limits;
108
+ if (l === undefined)
109
+ return null;
110
+ return (0, limits_1.validateRunLimits)({
111
+ ...(l.max_usd !== undefined ? { maxUsd: l.max_usd } : {}),
112
+ ...(l.max_calls !== undefined ? { maxCalls: l.max_calls } : {}),
113
+ ...(l.max_wall_ms !== undefined ? { maxWallMs: l.max_wall_ms } : {}),
114
+ ...(l.max_repeat_tool_call !== undefined ? { maxRepeatToolCall: l.max_repeat_tool_call } : {}),
115
+ });
116
+ }
117
+ /**
118
+ * ARC-5 pre-dispatch check — the ONE call `run()` and `runRaw()` make before a
119
+ * provider dispatch. Reads the ambient run (INCLUDING one already stopped at a
120
+ * limit, so it keeps refusing on that path). No run, or a run with no limits ⇒
121
+ * null, zero work. On a fresh breach it ends the run `failed` with
122
+ * `failure_reason: limit:<which>` and returns the typed error to throw.
123
+ * `ctx` overrides the ambient lookup (the raw stream captures it at entry).
124
+ */
125
+ function runLimitRefusal(ctx) {
126
+ const state = (ctx === undefined ? storage.getStore() : (ctx ?? undefined));
127
+ if (state === undefined || state.limits === null || state.limits === undefined)
128
+ return null;
129
+ if (state.limitStop !== undefined)
130
+ return new limits_1.RunLimitExceeded(state.limitStop, state.runId);
131
+ if (state.ended)
132
+ return null;
133
+ const breach = (0, limits_1.checkRunLimits)(state.limitState, state.limits, Date.now());
134
+ if (breach === null)
135
+ return null;
136
+ state.stopForLimit(breach);
137
+ return new limits_1.RunLimitExceeded(breach, state.runId);
138
+ }
139
+ /** Throwing form of {@link runLimitRefusal} — `run()` / `runRaw()` call this. */
140
+ function enforceRunLimits(ctx) {
141
+ const refusal = runLimitRefusal(ctx);
142
+ if (refusal !== null)
143
+ throw refusal;
144
+ }
95
145
  /** Valid ground-truth step key (G1 spec §3): a caller-supplied `stepKey` (either
96
146
  * lane) or a `tags.step` used as one. Auto-minted keys (`label#seq`) may contain
97
147
  * a `/` from a route slug or `provider/model` label and are NOT validated — the
@@ -128,10 +178,19 @@ function warnOnce(scope, err) {
128
178
  console.warn(`[modelrig artifacts] ${scope} buffer write failed (dropped, run unaffected): ` +
129
179
  `${err instanceof Error ? err.message : String(err)}`);
130
180
  }
181
+ /** ARC-1 (fix round 1, M4): the in-process open-conversation registry's bounds.
182
+ * An entry idle longer than the TTL (the server's default idle-close window) or
183
+ * past the LRU cap is EVICTED — not ended: the run stays `running` in the local
184
+ * store, so a later `rig.conversation(id)` re-enters it through `findOpenRun`. */
185
+ exports.MAX_OPEN_CONVERSATIONS = 10000;
186
+ exports.OPEN_CONVERSATION_IDLE_MS = 30 * 60000;
131
187
  function createRunNamespace(deps) {
132
188
  const { buffer, enabled } = deps;
189
+ const maxOpen = deps.maxOpenConversations ?? exports.MAX_OPEN_CONVERSATIONS;
190
+ const idleMs = deps.conversationIdleMs ?? exports.OPEN_CONVERSATION_IDLE_MS;
191
+ const clock = deps.now ?? Date.now;
133
192
  const inertHandle = { runId: "", end() { } };
134
- function buildContext(startRow, traceId, zeroRetention) {
193
+ function buildContext(startRow, traceId, zeroRetention, conversation, limits = null) {
135
194
  const runId = startRow.id;
136
195
  const episodeKey = startRow.episodeKey;
137
196
  const attemptByStep = new Map();
@@ -140,7 +199,7 @@ function createRunNamespace(deps) {
140
199
  // The ambient step key (G1): the last step minted or emitted in this run,
141
200
  // which an artifact.save without an explicit stepKey attaches to.
142
201
  let lastStepKey = null;
143
- return {
202
+ const state = {
144
203
  runId,
145
204
  episodeKey,
146
205
  traceId,
@@ -148,6 +207,15 @@ function createRunNamespace(deps) {
148
207
  ended: false,
149
208
  buffer,
150
209
  startRow,
210
+ limits,
211
+ limitState: (0, limits_1.initialLimitState)(Date.parse(startRow.startedAt) || Date.now()),
212
+ stopForLimit(breach) {
213
+ state.limitStop = breach;
214
+ endContext(state, "failed", { failureReason: (0, limits_1.limitFailureReason)(breach.which) });
215
+ },
216
+ ...(conversation !== undefined
217
+ ? { turns: conversation.turns, conversationKey: conversation.key, ...(Object.keys(conversation.tags).length > 0 ? { defaultTags: conversation.tags } : {}) }
218
+ : {}),
151
219
  noteLoad(artifactId) {
152
220
  loaded.add(artifactId);
153
221
  },
@@ -157,6 +225,12 @@ function createRunNamespace(deps) {
157
225
  return ids;
158
226
  },
159
227
  nextStepKey(label) {
228
+ // ARC-1: inside a conversation the auto key is the turn key `turn:<n>.<k>`
229
+ // (an explicit caller stepKey never reaches here — resolveStepKey).
230
+ if (conversation !== undefined) {
231
+ lastStepKey = conversation.turns.mint();
232
+ return lastStepKey;
233
+ }
160
234
  stepSeq += 1;
161
235
  lastStepKey = `${label}#${stepSeq}`;
162
236
  return lastStepKey;
@@ -165,9 +239,20 @@ function createRunNamespace(deps) {
165
239
  return lastStepKey;
166
240
  },
167
241
  emitGroundTruthStep(input) {
242
+ // M4: activity keeps a conversation fresh in the in-process registry.
243
+ if (state.conversationKey !== undefined && openConversations.has(state.conversationKey))
244
+ touch(state.conversationKey, state);
168
245
  // A caller-supplied key on the raw lane never minted through nextStepKey;
169
246
  // record it as the ambient step so artifacts saved around this call join it.
170
247
  lastStepKey = input.stepKey;
248
+ // ARC-5: fold this attempt into the limit tally (pure; no-op without limits).
249
+ if (state.limits !== null) {
250
+ state.limitState = (0, limits_1.accumulate)(state.limitState, {
251
+ costUsd: input.costUsd ?? 0,
252
+ notDispatched: input.failureClass !== null && failures_1.TERMINAL_CLASSES.has(input.failureClass),
253
+ ...(input.toolCalls !== undefined ? { toolCalls: input.toolCalls } : {}),
254
+ });
255
+ }
171
256
  const attemptNo = (attemptByStep.get(input.stepKey) ?? 0) + 1;
172
257
  attemptByStep.set(input.stepKey, attemptNo);
173
258
  try {
@@ -190,10 +275,67 @@ function createRunNamespace(deps) {
190
275
  }
191
276
  },
192
277
  };
278
+ return state;
279
+ }
280
+ function seededCounter(stepKeys) {
281
+ const { turn, call } = (0, conversation_1.turnStateOf)(stepKeys);
282
+ return new conversation_1.TurnCounter(turn, call);
283
+ }
284
+ /** ARC-1: the open conversations of THIS namespace, by `agent \0 id`, in LRU
285
+ * order (a touch re-inserts), each with its last-touch time (M4). */
286
+ const openConversations = new Map();
287
+ function touch(key, ctx) {
288
+ openConversations.delete(key);
289
+ openConversations.set(key, { ctx, touchedAt: clock() });
290
+ }
291
+ /** Evict idle-past-TTL entries and anything past the cap (oldest first). */
292
+ function evict() {
293
+ const t = clock();
294
+ for (const [key, entry] of openConversations) {
295
+ if (openConversations.size > maxOpen || t - entry.touchedAt >= idleMs || entry.ctx.ended)
296
+ openConversations.delete(key);
297
+ else
298
+ break; // LRU order: the rest are newer
299
+ }
300
+ }
301
+ /** Test seam: the registry's live size. */
302
+ function openConversationCount() {
303
+ return openConversations.size;
304
+ }
305
+ /** Find (in-process, then local store) or start a conversation's run. */
306
+ function openOrResume(id, opts) {
307
+ const key = `${opts.agent}\u0000${id}`;
308
+ evict();
309
+ const live = openConversations.get(key)?.ctx;
310
+ if (live !== undefined && !live.ended) {
311
+ touch(key, live);
312
+ return { ctx: live, resumed: true };
313
+ }
314
+ const tags = (0, conversation_1.conversationTags)(opts);
315
+ let stored = null;
316
+ try {
317
+ stored = buffer.findOpenRun?.(opts.agent, id) ?? null;
318
+ }
319
+ catch (err) {
320
+ warnOnce("conversation lookup", err); // fail-open: start a fresh run
321
+ }
322
+ const ctx = stored !== null
323
+ ? buildContext(stored.row, null, false, { turns: seededCounter(stored.stepKeys), key, tags }, limitsFromMeta(stored.row.meta) ?? (0, limits_1.validateRunLimits)(opts.limits))
324
+ : newRun({
325
+ pipeline: opts.agent,
326
+ episodeKey: id,
327
+ ...(opts.environment !== undefined ? { environment: opts.environment } : {}),
328
+ meta: { v: 1, trigger: "conversation" },
329
+ ...(opts.limits !== undefined ? { limits: opts.limits } : {}),
330
+ }, { turns: new conversation_1.TurnCounter(0), key, tags });
331
+ touch(key, ctx);
332
+ evict();
333
+ return { ctx, resumed: stored !== null };
193
334
  }
194
335
  /** Build the context + record the `running` row. Shared by start() and
195
336
  * scope() so the two entry points agree on identity + buffering. */
196
- function newRun(opts) {
337
+ function newRun(opts, conversation) {
338
+ const limits = (0, limits_1.validateRunLimits)(opts.limits);
197
339
  const startRow = {
198
340
  id: (0, node_crypto_1.randomUUID)(),
199
341
  pipeline: opts.pipeline,
@@ -205,12 +347,12 @@ function createRunNamespace(deps) {
205
347
  status: "running",
206
348
  // WI-1 (spec §3): the run envelope, git-sha auto-filled. endContext spreads
207
349
  // ...ctx.startRow, so the terminal projection re-exports the same meta.
208
- meta: buildRunMeta(opts.meta),
350
+ meta: buildRunMeta(limits !== null ? { v: 1, ...(opts.meta ?? {}), limits: (0, limits_1.limitsForMeta)(limits) } : opts.meta),
209
351
  };
210
352
  const traceId = opts.traceparent !== undefined
211
353
  ? ((0, otel_map_1.parseTraceparent)(opts.traceparent)?.traceId ?? null)
212
354
  : null;
213
- const ctx = buildContext(startRow, traceId, opts.zeroRetention === true);
355
+ const ctx = buildContext(startRow, traceId, opts.zeroRetention === true, conversation, limits);
214
356
  try {
215
357
  buffer.recordRun(startRow);
216
358
  }
@@ -227,13 +369,20 @@ function createRunNamespace(deps) {
227
369
  storage.enterWith(ctx);
228
370
  return {
229
371
  runId: ctx.runId,
230
- end: (status) => endContext(ctx, status),
372
+ end: (status, opts) => endContext(ctx, status, opts),
231
373
  };
232
374
  },
233
- end(status) {
375
+ end(status, opts) {
234
376
  const ctx = storage.getStore();
235
377
  if (ctx !== undefined)
236
- endContext(ctx, status);
378
+ endContext(ctx, status, opts);
379
+ },
380
+ setFailureReason(reason) {
381
+ if (!enabled)
382
+ return;
383
+ const ctx = storage.getStore();
384
+ if (ctx !== undefined && !ctx.ended)
385
+ ctx.failureReason = reason;
237
386
  },
238
387
  async scope(opts, fn) {
239
388
  if (!enabled)
@@ -254,11 +403,32 @@ function createRunNamespace(deps) {
254
403
  }
255
404
  });
256
405
  },
406
+ openConversationCount,
407
+ conversation(id, opts) {
408
+ return (0, conversation_1.createConversation)({
409
+ enabled,
410
+ openOrResume,
411
+ runIdOf: (ctx) => ctx.runId,
412
+ counterOf: (ctx) => ctx.turns,
413
+ enter: (ctx) => storage.enterWith(ctx),
414
+ runIn: (ctx, fn) => storage.run(ctx, fn),
415
+ end: (ctx, status, endOpts) => endContext(ctx, status, endOpts),
416
+ }, id, opts);
417
+ },
257
418
  };
258
- function endContext(ctx, status) {
419
+ function endContext(ctx, status, opts) {
259
420
  if (ctx.ended)
260
421
  return;
261
422
  ctx.ended = true;
423
+ if (ctx.conversationKey !== undefined)
424
+ openConversations.delete(ctx.conversationKey);
425
+ const terminal = status ?? "succeeded";
426
+ const reason = opts?.failureReason ?? ctx.failureReason;
427
+ // R9: a failed run carries its reason in runs.meta (sanitized like every
428
+ // envelope field — a free-text reason is dropped, never smuggled).
429
+ const meta = terminal === "failed" && reason !== undefined
430
+ ? buildRunMeta({ ...(ctx.startRow.meta ?? { v: 1 }), failure_reason: reason })
431
+ : ctx.startRow.meta;
262
432
  // Update the projection in place: same identity as the start row, now with
263
433
  // an ended_at and a terminal status. recordRun bumps export_seq so this
264
434
  // terminal state re-exports past the `:runs` watermark. Emitting the full
@@ -267,8 +437,9 @@ function createRunNamespace(deps) {
267
437
  try {
268
438
  buffer.recordRun({
269
439
  ...ctx.startRow,
440
+ meta,
270
441
  endedAt: new Date().toISOString(),
271
- status: status ?? "succeeded",
442
+ status: terminal,
272
443
  });
273
444
  }
274
445
  catch (err) {
@@ -55,6 +55,9 @@ export interface ScrubSummary {
55
55
  readonly mode: ScrubMode;
56
56
  readonly blocked: boolean;
57
57
  readonly findings: readonly ScrubFinding[];
58
+ /** ARC-4 (UC-3 M2 posture a): set to `"media"` on an image/video/audio
59
+ * artifact — bytes the text engine never scanned. Absent on text artifacts. */
60
+ readonly unscreened?: "media";
58
61
  }
59
62
  /** Build the persisted summary from a scrub result. The only producer of the
60
63
  * `artifacts.scrub` value, so the counts-not-values invariant lives in one
@@ -1 +1 @@
1
- {"version":3,"file":"scrub.d.ts","sourceRoot":"","sources":["../../src/artifacts/scrub.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AAEH,OAAO,KAAK,EAAkB,aAAa,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAElF,MAAM,MAAM,SAAS,GAAG,KAAK,GAAG,QAAQ,GAAG,QAAQ,GAAG,OAAO,CAAC;AAE9D,iFAAiF;AACjF,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,IAAI,EAAE,iBAAiB,CAAC;IACjC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,MAAM,iBAAiB,GAAG,KAAK,GAAG,KAAK,GAAG,OAAO,GAAG,OAAO,GAAG,KAAK,CAAC;AAE1E,MAAM,WAAW,WAAW;IAC1B,8EAA8E;IAC9E,QAAQ,CAAC,QAAQ,EAAE,SAAS,YAAY,EAAE,CAAC;IAC3C;iEAC6D;IAC7D,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,oEAAoE;IACpE,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;CAC3B;AAED;;;0EAG0E;AAC1E,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC;IACzB,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,QAAQ,EAAE,SAAS,YAAY,EAAE,CAAC;CAC5C;AAED;;kFAEkF;AAClF,wBAAgB,cAAc,CAAC,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,WAAW,GAAG,YAAY,CAEjF;AAaD,8EAA8E;AAC9E,wBAAgB,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAejD;AA0CD;;;GAGG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,GAAG,WAAW,CA6CvE;AAqBD;;;;;GAKG;AACH,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,aAAa,EAAE,CAIzD;AAED;;;;;;;GAOG;AACH,wBAAgB,cAAc,IAAI,QAAQ,EAAE,CAS3C"}
1
+ {"version":3,"file":"scrub.d.ts","sourceRoot":"","sources":["../../src/artifacts/scrub.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AAEH,OAAO,KAAK,EAAkB,aAAa,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAElF,MAAM,MAAM,SAAS,GAAG,KAAK,GAAG,QAAQ,GAAG,QAAQ,GAAG,OAAO,CAAC;AAE9D,iFAAiF;AACjF,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,IAAI,EAAE,iBAAiB,CAAC;IACjC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,MAAM,iBAAiB,GAAG,KAAK,GAAG,KAAK,GAAG,OAAO,GAAG,OAAO,GAAG,KAAK,CAAC;AAE1E,MAAM,WAAW,WAAW;IAC1B,8EAA8E;IAC9E,QAAQ,CAAC,QAAQ,EAAE,SAAS,YAAY,EAAE,CAAC;IAC3C;iEAC6D;IAC7D,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,oEAAoE;IACpE,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;CAC3B;AAED;;;0EAG0E;AAC1E,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC;IACzB,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,QAAQ,EAAE,SAAS,YAAY,EAAE,CAAC;IAC3C;mFAC+E;IAC/E,QAAQ,CAAC,UAAU,CAAC,EAAE,OAAO,CAAC;CAC/B;AAED;;kFAEkF;AAClF,wBAAgB,cAAc,CAAC,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,WAAW,GAAG,YAAY,CAEjF;AAaD,8EAA8E;AAC9E,wBAAgB,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAejD;AA0CD;;;GAGG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,GAAG,WAAW,CA6CvE;AAqBD;;;;;GAKG;AACH,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,aAAa,EAAE,CAIzD;AAED;;;;;;;GAOG;AACH,wBAAgB,cAAc,IAAI,QAAQ,EAAE,CAS3C"}
@@ -1 +1 @@
1
- {"version":3,"file":"engine.d.ts","sourceRoot":"","sources":["../../src/bakeoff/engine.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAGH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAG1C,OAAO,KAAK,EAAE,aAAa,EAAE,aAAa,EAAE,WAAW,EAAgB,MAAM,gBAAgB,CAAC;AAG9F,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAQ5C,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAEpD,OAAO,KAAK,EAAE,UAAU,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAC7D,OAAO,KAAK,EAAE,eAAe,EAAE,gBAAgB,EAAmC,cAAc,EAAE,MAAM,SAAS,CAAC;AAElH;;+BAE+B;AAC/B,eAAO,MAAM,aAAa,kBAAkB,CAAC;AAE7C;;;gFAGgF;AAChF,MAAM,MAAM,kBAAkB,GAAG,CAAC,KAAK,EAAE;IACvC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,IAAI,EAAE,SAAS,UAAU,EAAE,CAAC;CACtC,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;AAgBpB,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,qEAAqE;IACrE,QAAQ,CAAC,QAAQ,EAAE,SAAS,MAAM,EAAE,CAAC;IACrC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,2EAA2E;IAC3E,QAAQ,CAAC,qBAAqB,EAAE,MAAM,CAAC;IACvC;;wEAEoE;IACpE,QAAQ,CAAC,WAAW,CAAC,EAAE,eAAe,CAAC;IACvC;iEAC6D;IAC7D,QAAQ,CAAC,aAAa,CAAC,EAAE,aAAa,CAAC;IACvC;;wEAEoE;IACpE,QAAQ,CAAC,oBAAoB,CAAC,EAAE,CAAC,KAAK,EAAE,WAAW,KAAK,aAAa,GAAG,IAAI,CAAC;IAC7E;+BAC2B;IAC3B,QAAQ,CAAC,QAAQ,CAAC,EAAE,WAAW,CAAC,MAAM,EAAE;QAAE,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAA;KAAE,CAAC,CAAC;IACrE;;+EAE2E;IAC3E,QAAQ,CAAC,UAAU,CAAC,EAAE,UAAU,CAAC;IACjC;;;;;+EAK2E;IAC3E,QAAQ,CAAC,WAAW,CAAC,EAAE,WAAW,CAAC;CACpC;AAsBD,wBAAsB,UAAU,CAC9B,IAAI,EAAE,OAAO,EACb,QAAQ,EAAE,YAAY,EACtB,GAAG,EAAE,cAAc,EACnB,WAAW,CAAC,EAAE,kBAAkB,GAC/B,OAAO,CAAC,eAAe,CAAC,CAuL1B;AAuCD;iEACiE;AACjE,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,SAAS,CAAC,EAAE,gBAAgB,CAAC;IACtC,mEAAmE;IACnE,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;CACpC;AA8BD;;;;GAIG;AACH,wBAAgB,eAAe,CAC7B,SAAS,EAAE,SAAS,MAAM,EAAE,EAC5B,OAAO,EAAE,SAAS,cAAc,EAAE,EAClC,GAAG,CAAC,EAAE,eAAe,GACpB,eAAe,CAAC,UAAU,CAAC,CA8G7B;AAED,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,OAAO,CAAC,EAAE,OAAO,KAAK,CAAC;CACjC;AAED;;uDAEuD;AACvD,wBAAsB,aAAa,CACjC,QAAQ,EAAE,eAAe,EACzB,MAAM,EAAE,mBAAmB,GAC1B,OAAO,CAAC,IAAI,CAAC,CA6Bf;AAED;;;;4BAI4B;AAC5B,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,SAAS,EAAE,IAAI,GAAG,MAAM,GAAG,SAAS,CAAC;IAC9C,kFAAkF;IAClF,QAAQ,CAAC,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;CAC3C;AAED;;;;;;;GAOG;AACH,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,eAAe,GAAG,aAAa,GAAG,IAAI,CAkBnF;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,eAAe,CAAC,QAAQ,EAAE,eAAe,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CA2CpG"}
1
+ {"version":3,"file":"engine.d.ts","sourceRoot":"","sources":["../../src/bakeoff/engine.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAGH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAG1C,OAAO,KAAK,EAAE,aAAa,EAAE,aAAa,EAAE,WAAW,EAAgB,MAAM,gBAAgB,CAAC;AAI9F,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAQ5C,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAEpD,OAAO,KAAK,EAAE,UAAU,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAC7D,OAAO,KAAK,EAAE,eAAe,EAAE,gBAAgB,EAAmC,cAAc,EAAE,MAAM,SAAS,CAAC;AAElH;;+BAE+B;AAC/B,eAAO,MAAM,aAAa,kBAAkB,CAAC;AAE7C;;;gFAGgF;AAChF,MAAM,MAAM,kBAAkB,GAAG,CAAC,KAAK,EAAE;IACvC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,IAAI,EAAE,SAAS,UAAU,EAAE,CAAC;CACtC,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;AAgBpB,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,qEAAqE;IACrE,QAAQ,CAAC,QAAQ,EAAE,SAAS,MAAM,EAAE,CAAC;IACrC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,2EAA2E;IAC3E,QAAQ,CAAC,qBAAqB,EAAE,MAAM,CAAC;IACvC;;wEAEoE;IACpE,QAAQ,CAAC,WAAW,CAAC,EAAE,eAAe,CAAC;IACvC;iEAC6D;IAC7D,QAAQ,CAAC,aAAa,CAAC,EAAE,aAAa,CAAC;IACvC;;wEAEoE;IACpE,QAAQ,CAAC,oBAAoB,CAAC,EAAE,CAAC,KAAK,EAAE,WAAW,KAAK,aAAa,GAAG,IAAI,CAAC;IAC7E;+BAC2B;IAC3B,QAAQ,CAAC,QAAQ,CAAC,EAAE,WAAW,CAAC,MAAM,EAAE;QAAE,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAA;KAAE,CAAC,CAAC;IACrE;;+EAE2E;IAC3E,QAAQ,CAAC,UAAU,CAAC,EAAE,UAAU,CAAC;IACjC;;;;;+EAK2E;IAC3E,QAAQ,CAAC,WAAW,CAAC,EAAE,WAAW,CAAC;CACpC;AAsBD,wBAAsB,UAAU,CAC9B,IAAI,EAAE,OAAO,EACb,QAAQ,EAAE,YAAY,EACtB,GAAG,EAAE,cAAc,EACnB,WAAW,CAAC,EAAE,kBAAkB,GAC/B,OAAO,CAAC,eAAe,CAAC,CA0L1B;AAuCD;iEACiE;AACjE,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,SAAS,CAAC,EAAE,gBAAgB,CAAC;IACtC,mEAAmE;IACnE,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;CACpC;AA8BD;;;;GAIG;AACH,wBAAgB,eAAe,CAC7B,SAAS,EAAE,SAAS,MAAM,EAAE,EAC5B,OAAO,EAAE,SAAS,cAAc,EAAE,EAClC,GAAG,CAAC,EAAE,eAAe,GACpB,eAAe,CAAC,UAAU,CAAC,CA8G7B;AAED,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,OAAO,CAAC,EAAE,OAAO,KAAK,CAAC;CACjC;AAED;;uDAEuD;AACvD,wBAAsB,aAAa,CACjC,QAAQ,EAAE,eAAe,EACzB,MAAM,EAAE,mBAAmB,GAC1B,OAAO,CAAC,IAAI,CAAC,CA6Bf;AAED;;;;4BAI4B;AAC5B,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,SAAS,EAAE,IAAI,GAAG,MAAM,GAAG,SAAS,CAAC;IAC9C,kFAAkF;IAClF,QAAQ,CAAC,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;CAC3C;AAED;;;;;;;GAOG;AACH,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,eAAe,GAAG,aAAa,GAAG,IAAI,CAkBnF;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,eAAe,CAAC,QAAQ,EAAE,eAAe,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CA2CpG"}
@@ -20,6 +20,7 @@ exports.deriveSwapDirection = deriveSwapDirection;
20
20
  exports.bakeoffEvidence = bakeoffEvidence;
21
21
  const node_crypto_1 = require("node:crypto");
22
22
  const loader_1 = require("../routes/loader");
23
+ const turn_replay_1 = require("./turn-replay");
23
24
  const replay_1 = require("./replay");
24
25
  const stats_1 = require("./stats");
25
26
  const judge_1 = require("./judge");
@@ -109,8 +110,10 @@ async function runBakeoff(deps, captures, req, samplesSink) {
109
110
  ...(req.qualityGate !== undefined ? { qualityGate: req.qualityGate } : {}),
110
111
  });
111
112
  replayN = Math.max(replayN, samples.length);
112
- rawByArm.set(variant, samples);
113
- metrics.push((0, stats_1.aggregateVariantMetrics)(variant, samples));
113
+ // ARC-β ruling M3(c): incomparable cases leave this arm's score + regret, and are reported.
114
+ const split = (0, turn_replay_1.splitIncomparable)(samples);
115
+ rawByArm.set(variant, split.comparable);
116
+ metrics.push(withFallback(withIncomparable((0, stats_1.aggregateVariantMetrics)(variant, split.comparable), split.incomparable), split.comparable, firstCandidateLabel(route, variant)));
114
117
  }
115
118
  // RB-1 reference (ceiling) arm — replay the reference model on the SAME set
116
119
  // under its own envelope (it is dear by definition), then attach each requested
@@ -137,25 +140,26 @@ async function runBakeoff(deps, captures, req, samplesSink) {
137
140
  ...(req.qualityGate !== undefined ? { qualityGate: req.qualityGate } : {}),
138
141
  });
139
142
  const partial = refSamples.length < replayN || refSamples.some((s) => s.failureClass === "budget_exhausted");
140
- refSamplesForMining = refSamples;
143
+ const refSplit = (0, turn_replay_1.splitIncomparable)(refSamples); // ARC-β M3(c): the reference's incomparable cases pair with nothing
144
+ refSamplesForMining = refSplit.comparable;
141
145
  for (let i = 0; i < metrics.length; i += 1) {
142
146
  const raw = rawByArm.get(metrics[i].variant) ?? [];
143
147
  // N5: pairwise standing vs the reference — deterministic where both sides
144
148
  // carry value accuracy, else the opt-in capped judge (fresh per arm, so
145
149
  // maxSamples is per-arm). A REPORTED signal beside regret.
146
150
  const pwJudge = pairwiseJudgeCfg !== undefined ? (0, pairwise_1.makePairwiseJudge)(deps, route, pairwiseJudgeCfg) : null;
147
- const { stats: pairwise, outcomes } = await (0, pairwise_1.scoreArmPairwise)(raw, refSamples, pwJudge);
151
+ const { stats: pairwise, outcomes } = await (0, pairwise_1.scoreArmPairwise)(raw, refSplit.comparable, pwJudge);
148
152
  for (const [captureId, outcome] of outcomes)
149
153
  pairwiseOutcomes.set((0, disagreement_1.pairwiseKey)(metrics[i].variant, captureId), outcome);
150
154
  metrics[i] = {
151
155
  ...metrics[i],
152
- regret: (0, stats_1.pairedRegret)(raw, refSamples),
156
+ regret: (0, stats_1.pairedRegret)(raw, refSplit.comparable),
153
157
  ...(pairwise !== null ? { pairwise } : {}),
154
158
  ...(partial ? { referencePartial: true } : {}),
155
159
  };
156
160
  }
157
161
  metrics.push({
158
- ...(0, stats_1.aggregateVariantMetrics)(exports.REFERENCE_ARM, refSamples),
162
+ ...withIncomparable((0, stats_1.aggregateVariantMetrics)(exports.REFERENCE_ARM, refSplit.comparable), refSplit.incomparable),
159
163
  isReference: true,
160
164
  regret: null,
161
165
  ...(partial ? { referencePartial: true } : {}),
@@ -191,7 +195,7 @@ async function runBakeoff(deps, captures, req, samplesSink) {
191
195
  const keptArms = (0, samples_1.selectArmsToKeep)(req.keepOutputs, incumbentArm, proposal?.variant ?? null, req.variants);
192
196
  const arms = keptArms.map((arm) => ({
193
197
  arm,
194
- armMeta: deriveArmMeta(route, arm),
198
+ armMeta: withFallbackArmMeta(deriveArmMeta(route, arm), metrics.find((m) => m.variant === arm)), // ARC-β R3-1
195
199
  samples: (rawByArm.get(arm) ?? []).map((s) => ({
196
200
  inputRef: s.captureId,
197
201
  inputText: s.inputText ?? "",
@@ -514,3 +518,26 @@ function bakeoffEvidence(artifact, spentUsd) {
514
518
  })),
515
519
  };
516
520
  }
521
+ /** ARC-β M3(c): attach the incomparable counts to an arm's metrics (absent when
522
+ * none — the artifact is then unchanged). */
523
+ function withIncomparable(m, rows) {
524
+ return rows.length > 0 ? { ...m, incomparable: rows } : m;
525
+ }
526
+ /** ARC-β R3-1: the arm's first candidate as `provider/model` (its nominal label). */
527
+ function firstCandidateLabel(route, arm) {
528
+ const variant = arm === "default" ? undefined : route.variants.find((v) => v.name === arm);
529
+ const c = variant?.candidates?.[0] ?? route.candidates[0];
530
+ return c !== undefined ? `${c.provider}/${c.model}` : "unknown";
531
+ }
532
+ /** ARC-β R3-1: when any of an arm's cases was answered by a fallback (its first
533
+ * candidate skipped for the history), attach the per-served-model counts and the
534
+ * honest arm label. Absent ⇒ the metrics (and the artifact) are unchanged. */
535
+ function withFallback(m, samples, first) {
536
+ const rows = (0, turn_replay_1.fallbackServed)(samples);
537
+ return rows.length > 0 ? { ...m, servedByFallback: rows, armLabel: (0, turn_replay_1.fallbackArmLabel)(first, rows, m.samples) } : m;
538
+ }
539
+ /** ARC-β R3-1: the reviewer's arm meta names the honest label, never the
540
+ * first candidate alone, when a fallback served any case. */
541
+ function withFallbackArmMeta(meta, m) {
542
+ return m?.armLabel !== undefined ? { ...meta, model: m.armLabel } : meta;
543
+ }
@@ -26,6 +26,7 @@ import type { KeepOutputsMode } from "./samples";
26
26
  import type { BakeoffArtifact } from "./types";
27
27
  import type { CandidateSpec, ObjectiveMode, RouteBundle } from "../types/route";
28
28
  import type { QualityGate } from "../types/quality";
29
+ import type { TurnReplayCase } from "./turn-replay";
29
30
  /** The narrow store surface the selection reads — kept minimal so this module
30
31
  * depends on four reads, not three whole stores. */
31
32
  export interface EvalCaseSelectionDeps {
@@ -44,6 +45,10 @@ export interface RehydratedCase {
44
45
  * golden to grade against — it scores by conformance only). Carried so the
45
46
  * runner can build the `expected` map from the golden's content. */
46
47
  readonly correctedArtifactId: string | null;
48
+ /** ARC-β: present when the capture is a messages-lane turn — the history up to
49
+ * and including the turn's last user/tool message + the reference reply. The
50
+ * replay answers that final turn only (bakeoff/turn-replay.ts). */
51
+ readonly turn?: TurnReplayCase;
47
52
  }
48
53
  export interface EvalCaseRefusal {
49
54
  readonly caseId: string;
@@ -1 +1 @@
1
- {"version":3,"file":"eval-cases.d.ts","sourceRoot":"","sources":["../../src/bakeoff/eval-cases.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,KAAK,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACrE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AACjE,OAAO,KAAK,EAAE,wBAAwB,EAAE,MAAM,oBAAoB,CAAC;AACnE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,EAAc,KAAK,kBAAkB,EAAE,MAAM,UAAU,CAAC;AAC/D,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAC5C,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AACjD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAC/C,OAAO,KAAK,EAAE,aAAa,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAChF,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAEpD;oDACoD;AACpD,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,mBAAmB,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,WAAW,EAAE,CAAC;IAC9D,QAAQ,CAAC,eAAe,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK;QAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAA;KAAE,GAAG,IAAI,CAAC;IAClG,QAAQ,CAAC,kBAAkB,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,KAAK,MAAM,GAAG,IAAI,CAAC;IAC/E,QAAQ,CAAC,uBAAuB,EAAE,CAAC,WAAW,EAAE,MAAM,KAAK,UAAU,GAAG,IAAI,CAAC;CAC9E;AAED,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,OAAO,EAAE,UAAU,CAAC;IAC7B;;wEAEoE;IACpE,QAAQ,CAAC,mBAAmB,EAAE,MAAM,GAAG,IAAI,CAAC;CAC7C;AAED,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAClC,qEAAqE;IACrE,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,oEAAoE;IACpE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,YAAY,EAAE,SAAS,cAAc,EAAE,CAAC;IACjD,QAAQ,CAAC,QAAQ,EAAE,SAAS,eAAe,EAAE,CAAC;CAC/C;AAED;;;;GAIG;AACH,wBAAgB,sBAAsB,CAAC,IAAI,EAAE,qBAAqB,EAAE,IAAI,EAAE,MAAM,GAAG,iBAAiB,CAgCnG;AAED;;;;;;GAMG;AACH,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,SAAS,cAAc,EAAE,GAAG,YAAY,CAclF;AAED,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,SAAS,EAAE,iBAAiB,CAAC;IACtC;uFACmF;IACnF,QAAQ,CAAC,QAAQ,EAAE,eAAe,GAAG,IAAI,CAAC;CAC3C;AAED;;;;;;GAMG;AACH;;;0DAG0D;AAC1D,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,SAAS,EAAE,IAAI,CAAC,aAAa,EAAE,qBAAqB,GAAG,iBAAiB,CAAC,GAAG,IAAI,CAAC;IAC1F,QAAQ,CAAC,SAAS,EAAE,IAAI,CAAC,wBAAwB,EAAE,oBAAoB,CAAC,CAAC;IACzE,QAAQ,CAAC,QAAQ,EAAE,MAAM,IAAI,CAAC,YAAY,EAAE,kBAAkB,CAAC,CAAC;IAChE;;;;;6DAKyD;IACzD,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,mBAAmB,EAAE,MAAM,KAAK;QAAE,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAA;KAAE,GAAG,IAAI,CAAC;CACvF;AAED,wBAAsB,uBAAuB,CAC3C,IAAI,EAAE,OAAO,EACb,MAAM,EAAE,cAAc,EACtB,GAAG,EAAE;IACH,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,QAAQ,EAAE,SAAS,MAAM,EAAE,CAAC;IACrC,QAAQ,CAAC,qBAAqB,EAAE,MAAM,CAAC;IACvC,QAAQ,CAAC,WAAW,CAAC,EAAE,eAAe,CAAC;IACvC,QAAQ,CAAC,aAAa,CAAC,EAAE,aAAa,CAAC;IACvC,QAAQ,CAAC,oBAAoB,CAAC,EAAE,CAAC,KAAK,EAAE,WAAW,KAAK,aAAa,GAAG,IAAI,CAAC;IAC7E,QAAQ,CAAC,WAAW,CAAC,EAAE,WAAW,CAAC;IACnC;;;sCAGkC;IAClC,QAAQ,CAAC,UAAU,CAAC,EAAE,UAAU,CAAC;CAClC,EACD,WAAW,CAAC,EAAE,kBAAkB,GAC/B,OAAO,CAAC,qBAAqB,CAAC,CA+ChC"}
1
+ {"version":3,"file":"eval-cases.d.ts","sourceRoot":"","sources":["../../src/bakeoff/eval-cases.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,KAAK,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACrE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AACjE,OAAO,KAAK,EAAE,wBAAwB,EAAE,MAAM,oBAAoB,CAAC;AACnE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,EAAc,KAAK,kBAAkB,EAAE,MAAM,UAAU,CAAC;AAC/D,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAC5C,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AACjD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAC/C,OAAO,KAAK,EAAE,aAAa,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAChF,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAEpD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAEpD;oDACoD;AACpD,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,mBAAmB,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,WAAW,EAAE,CAAC;IAC9D,QAAQ,CAAC,eAAe,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK;QAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAA;KAAE,GAAG,IAAI,CAAC;IAClG,QAAQ,CAAC,kBAAkB,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,KAAK,MAAM,GAAG,IAAI,CAAC;IAC/E,QAAQ,CAAC,uBAAuB,EAAE,CAAC,WAAW,EAAE,MAAM,KAAK,UAAU,GAAG,IAAI,CAAC;CAC9E;AAED,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,OAAO,EAAE,UAAU,CAAC;IAC7B;;wEAEoE;IACpE,QAAQ,CAAC,mBAAmB,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5C;;uEAEmE;IACnE,QAAQ,CAAC,IAAI,CAAC,EAAE,cAAc,CAAC;CAChC;AAED,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAClC,qEAAqE;IACrE,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,oEAAoE;IACpE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,YAAY,EAAE,SAAS,cAAc,EAAE,CAAC;IACjD,QAAQ,CAAC,QAAQ,EAAE,SAAS,eAAe,EAAE,CAAC;CAC/C;AAED;;;;GAIG;AACH,wBAAgB,sBAAsB,CAAC,IAAI,EAAE,qBAAqB,EAAE,IAAI,EAAE,MAAM,GAAG,iBAAiB,CAuCnG;AAED;;;;;;GAMG;AACH,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,SAAS,cAAc,EAAE,GAAG,YAAY,CAclF;AAED,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,SAAS,EAAE,iBAAiB,CAAC;IACtC;uFACmF;IACnF,QAAQ,CAAC,QAAQ,EAAE,eAAe,GAAG,IAAI,CAAC;CAC3C;AAED;;;;;;GAMG;AACH;;;0DAG0D;AAC1D,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,SAAS,EAAE,IAAI,CAAC,aAAa,EAAE,qBAAqB,GAAG,iBAAiB,CAAC,GAAG,IAAI,CAAC;IAC1F,QAAQ,CAAC,SAAS,EAAE,IAAI,CAAC,wBAAwB,EAAE,oBAAoB,CAAC,CAAC;IACzE,QAAQ,CAAC,QAAQ,EAAE,MAAM,IAAI,CAAC,YAAY,EAAE,kBAAkB,CAAC,CAAC;IAChE;;;;;6DAKyD;IACzD,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,mBAAmB,EAAE,MAAM,KAAK;QAAE,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAA;KAAE,GAAG,IAAI,CAAC;CACvF;AAED,wBAAsB,uBAAuB,CAC3C,IAAI,EAAE,OAAO,EACb,MAAM,EAAE,cAAc,EACtB,GAAG,EAAE;IACH,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,QAAQ,EAAE,SAAS,MAAM,EAAE,CAAC;IACrC,QAAQ,CAAC,qBAAqB,EAAE,MAAM,CAAC;IACvC,QAAQ,CAAC,WAAW,CAAC,EAAE,eAAe,CAAC;IACvC,QAAQ,CAAC,aAAa,CAAC,EAAE,aAAa,CAAC;IACvC,QAAQ,CAAC,oBAAoB,CAAC,EAAE,CAAC,KAAK,EAAE,WAAW,KAAK,aAAa,GAAG,IAAI,CAAC;IAC7E,QAAQ,CAAC,WAAW,CAAC,EAAE,WAAW,CAAC;IACnC;;;sCAGkC;IAClC,QAAQ,CAAC,UAAU,CAAC,EAAE,UAAU,CAAC;CAClC,EACD,WAAW,CAAC,EAAE,kBAAkB,GAC/B,OAAO,CAAC,qBAAqB,CAAC,CA+ChC"}
@@ -21,6 +21,7 @@ exports.selectEvalCaseCaptures = selectEvalCaseCaptures;
21
21
  exports.evalCaseCaptureStore = evalCaseCaptureStore;
22
22
  exports.runBakeoffFromEvalCases = runBakeoffFromEvalCases;
23
23
  const engine_1 = require("./engine");
24
+ const turn_replay_1 = require("./turn-replay");
24
25
  /**
25
26
  * Partition a task's eval cases into rehydratable (with their captured input) and
26
27
  * refused (named, with a reason). Pure over the injected reads — no network, no
@@ -53,7 +54,15 @@ function selectEvalCaseCaptures(deps, task) {
53
54
  refuse("input not captured locally (route `capture` was off, or the capture was evicted)");
54
55
  continue;
55
56
  }
56
- rehydratable.push({ caseId: c.id, capture, correctedArtifactId: c.correctedArtifactId });
57
+ let turn; // ARC-β: messages captures carry their turn
58
+ try {
59
+ turn = (0, turn_replay_1.turnReplayCase)(capture);
60
+ }
61
+ catch {
62
+ refuse("captured conversation is not a valid messages-lane history (corrupt or hand-edited capture)");
63
+ continue;
64
+ }
65
+ rehydratable.push({ caseId: c.id, capture, correctedArtifactId: c.correctedArtifactId, ...(turn !== null ? { turn } : {}) });
57
66
  }
58
67
  return { task, total: cases.length, rehydratable, refusals };
59
68
  }
@@ -89,6 +89,21 @@ export interface ReplaySampleResult {
89
89
  /** Addendum A3 (keepOutputs only): this arm's output for the review, or
90
90
  * null when the arm produced none (a failure). */
91
91
  readonly outputText?: string | null;
92
+ /** ARC-β ruling M3(c): the served candidate cannot accept this captured
93
+ * history natively (run/messages.ts `historyIncompatibility`). The sample is
94
+ * INCOMPARABLE for that model — excluded from its score and regret, counted
95
+ * and reported, never scored as a failure (`splitIncomparable`). */
96
+ readonly incomparable?: {
97
+ readonly model: string;
98
+ readonly reason: string;
99
+ };
100
+ /** ARC-β R3-1 (messages routes only): the `provider/model` that actually
101
+ * answered this turn — a fallback when the arm's earlier candidates were
102
+ * skipped for the history. Absent on variables routes (unchanged samples). */
103
+ readonly servedBy?: string;
104
+ /** ARC-β R3-1: the run's `history_skipped` tag (`provider/model=code,…`) —
105
+ * present only when candidates were skipped before the ladder. */
106
+ readonly historySkipped?: string;
92
107
  }
93
108
  export declare function replayVariant(deps: RunDeps, captures: CaptureStore, req: ReplayRequest): Promise<readonly ReplaySampleResult[]>;
94
109
  //# sourceMappingURL=replay.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"replay.d.ts","sourceRoot":"","sources":["../../src/bakeoff/replay.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAGH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAGzD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAI1C,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAC5C,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAC3C,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AAExD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAGpD,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B;;uCAEmC;IACnC,QAAQ,CAAC,WAAW,CAAC,EAAE,OAAO,CAAC;IAC/B;;;;kFAI8E;IAC9E,QAAQ,CAAC,uBAAuB,CAAC,EAAE,OAAO,CAAC;IAC3C;;yEAEqE;IACrE,QAAQ,CAAC,QAAQ,CAAC,EAAE,WAAW,CAAC,MAAM,EAAE;QAAE,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAA;KAAE,CAAC,CAAC;IACrE;;sEAEkE;IAClE,QAAQ,CAAC,UAAU,CAAC,EAAE,UAAU,CAAC;IACjC;sFACkF;IAClF,QAAQ,CAAC,KAAK,CAAC,EAAE,WAAW,CAAC;IAC7B;;;sFAGkF;IAClF,QAAQ,CAAC,WAAW,CAAC,EAAE,iBAAiB,CAAC;IACzC;;;;;iFAK6E;IAC7E,QAAQ,CAAC,WAAW,CAAC,EAAE,WAAW,CAAC;CACpC;AAED,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC;IAC3B;wEACoE;IACpE,QAAQ,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC;;;oCAGgC;IAChC,QAAQ,CAAC,aAAa,EAAE,OAAO,GAAG,IAAI,CAAC;IACvC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,mEAAmE;IACnE,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B;yDACqD;IACrD,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAC;IACjC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,oEAAoE;IACpE,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B;sDACkD;IAClD,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACrC;AAED,wBAAsB,aAAa,CACjC,IAAI,EAAE,OAAO,EACb,QAAQ,EAAE,YAAY,EACtB,GAAG,EAAE,aAAa,GACjB,OAAO,CAAC,SAAS,kBAAkB,EAAE,CAAC,CAoJxC"}
1
+ {"version":3,"file":"replay.d.ts","sourceRoot":"","sources":["../../src/bakeoff/replay.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAGH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAGzD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAI1C,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAC5C,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAC3C,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AAExD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAKpD,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B;;uCAEmC;IACnC,QAAQ,CAAC,WAAW,CAAC,EAAE,OAAO,CAAC;IAC/B;;;;kFAI8E;IAC9E,QAAQ,CAAC,uBAAuB,CAAC,EAAE,OAAO,CAAC;IAC3C;;yEAEqE;IACrE,QAAQ,CAAC,QAAQ,CAAC,EAAE,WAAW,CAAC,MAAM,EAAE;QAAE,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAA;KAAE,CAAC,CAAC;IACrE;;sEAEkE;IAClE,QAAQ,CAAC,UAAU,CAAC,EAAE,UAAU,CAAC;IACjC;sFACkF;IAClF,QAAQ,CAAC,KAAK,CAAC,EAAE,WAAW,CAAC;IAC7B;;;sFAGkF;IAClF,QAAQ,CAAC,WAAW,CAAC,EAAE,iBAAiB,CAAC;IACzC;;;;;iFAK6E;IAC7E,QAAQ,CAAC,WAAW,CAAC,EAAE,WAAW,CAAC;CACpC;AAED,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC;IAC3B;wEACoE;IACpE,QAAQ,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC;;;oCAGgC;IAChC,QAAQ,CAAC,aAAa,EAAE,OAAO,GAAG,IAAI,CAAC;IACvC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,mEAAmE;IACnE,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B;yDACqD;IACrD,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAC;IACjC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,oEAAoE;IACpE,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B;sDACkD;IAClD,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC;;;wEAGoE;IACpE,QAAQ,CAAC,YAAY,CAAC,EAAE;QAAE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IAC5E;;kFAE8E;IAC9E,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAC3B;sEACkE;IAClE,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,CAAC;CAClC;AAED,wBAAsB,aAAa,CACjC,IAAI,EAAE,OAAO,EACb,QAAQ,EAAE,YAAY,EACtB,GAAG,EAAE,aAAa,GACjB,OAAO,CAAC,SAAS,kBAAkB,EAAE,CAAC,CAoKxC"}
@@ -26,6 +26,8 @@ const run_1 = require("../run/run");
26
26
  const variant_1 = require("../run/variant");
27
27
  const quality_1 = require("./quality");
28
28
  const quality_gate_1 = require("../run/quality-gate");
29
+ const messages_1 = require("../run/messages");
30
+ const turn_replay_1 = require("./turn-replay");
29
31
  async function replayVariant(deps, captures, req) {
30
32
  const rows = captures.readLast(req.route, req.lastN);
31
33
  const replayId = (0, node_crypto_1.randomUUID)();
@@ -41,6 +43,9 @@ async function replayVariant(deps, captures, req) {
41
43
  const route = deps.routes.get(req.route);
42
44
  const variant = req.variant === "default" ? undefined : route.variants.find((v) => v.name === req.variant);
43
45
  const mode = variant !== undefined && (0, variant_1.isTrialOnlyVariant)(route, variant) ? "trial" : "live";
46
+ // ARC-β turn replay (M15 guard, roadmap §2): a messages route replays real
47
+ // end-user conversations, so it runs ONLY where capture is explicitly on.
48
+ (0, turn_replay_1.assertTurnReplayAllowed)(route, deps.settings ?? null);
44
49
  // Zero production exposure knobs: no capture writes (no replay-of-replay),
45
50
  // and the whole pass shares one hard envelope.
46
51
  const replayDeps = {
@@ -60,6 +65,8 @@ async function replayVariant(deps, captures, req) {
60
65
  let conform = false;
61
66
  let repaired = false;
62
67
  let failureClass = null;
68
+ let incomparable = null;
69
+ let servedBy = null;
63
70
  let outputText = null;
64
71
  let outputValue = undefined;
65
72
  let produced = false;
@@ -77,11 +84,15 @@ async function replayVariant(deps, captures, req) {
77
84
  // route rollups and the savings odometer EXCLUDE trial spend — a
78
85
  // bake-off's real dispatches must never move a live route's numbers.
79
86
  tags: { replay: replayId, replay_capture: sampleTag, variant: req.variant, lane: "trial" },
80
- input: JSON.parse(capture.variablesJson),
87
+ // ARC-β: a messages capture carries the history up to and including the
88
+ // turn's last user/tool message — the candidate answers THAT turn only (the
89
+ // captured output is the reference reply); nothing continues the conversation.
90
+ ...(0, messages_1.parseCaptureInput)(capture.variablesJson),
81
91
  ...(req.variant !== "default" ? { variant: req.variant } : {}),
82
92
  budget: { envelope: envelopeKey },
83
93
  });
84
94
  conform = result.meta.validated;
95
+ servedBy = `${result.meta.provider}/${result.meta.model}`;
85
96
  repaired = result.meta.repairedBy !== null;
86
97
  outputValue = result.output;
87
98
  produced = true;
@@ -95,6 +106,7 @@ async function replayVariant(deps, captures, req) {
95
106
  if (!(err instanceof failures_1.RigFailureError))
96
107
  throw err;
97
108
  failureClass = err.failure.class;
109
+ incomparable = (0, turn_replay_1.incomparableFrom)(err.failure);
98
110
  }
99
111
  const latencyMs = Date.now() - started;
100
112
  // Unified value tier (RB-1 §4.3 + Q1 §4.2, F5 deterministic-first). Only on a
@@ -150,6 +162,9 @@ async function replayVariant(deps, captures, req) {
150
162
  .filter((r) => r.repairedBy !== null)
151
163
  .reduce((a, r) => a + r.costEstimateUsd, 0);
152
164
  const pricingMissing = attemptRows.some((r) => r.pricingMissing);
165
+ // ARC-β R3-1: attribute a messages-route turn to the model that served it,
166
+ // and carry the pre-ladder skip (read from the call's own telemetry tags).
167
+ const historySkipped = route.inputKind === "messages" ? (0, turn_replay_1.skippedTagOf)(attemptRows) : undefined;
153
168
  results.push({
154
169
  captureId: capture.inferenceId,
155
170
  conform,
@@ -161,6 +176,9 @@ async function replayVariant(deps, captures, req) {
161
176
  pricingMissing,
162
177
  latencyMs,
163
178
  failureClass,
179
+ ...(incomparable !== null ? { incomparable } : {}),
180
+ ...(route.inputKind === "messages" && servedBy !== null ? { servedBy } : {}),
181
+ ...(historySkipped !== undefined ? { historySkipped } : {}),
164
182
  ...(retainText
165
183
  ? { inputText: capture.variablesJson, outputText }
166
184
  : {}),
@@ -1 +1 @@
1
- {"version":3,"file":"report.d.ts","sourceRoot":"","sources":["../../src/bakeoff/report.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAKH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAG3C,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AACvD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AACjD,OAAO,KAAK,EAAE,eAAe,EAAkB,MAAM,SAAS,CAAC;AAC/D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AA2CpD;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,QAAQ,EAAE,eAAe,GAAG,MAAM,GAAG,IAAI,CAexE;AAED;;;;GAIG;AACH,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,+EAA+E;IAC/E,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,gFAAgF;IAChF,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAClC,QAAQ,CAAC,WAAW,EAAE,mBAAmB,CAAC;CAC3C;AAED;;;;;;GAMG;AACH,wBAAgB,0BAA0B,CAAC,IAAI,EAAE,SAAS,gBAAgB,EAAE,GAAG,MAAM,EAAE,CAgCtF;AAED;;;;;GAKG;AACH,wBAAgB,yBAAyB,CAAC,QAAQ,EAAE,eAAe,GAAG,MAAM,EAAE,CAiB7E;AAED;;;;;GAKG;AACH,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,eAAe,GAAG,MAAM,EAAE,CAavE;AAED,wBAAgB,WAAW,CAAC,QAAQ,EAAE,eAAe,EAAE,aAAa,GAAE,SAAS,gBAAgB,EAAO,GAAG,IAAI,CA4D5G;AAED;;;;;;GAMG;AACH,wBAAsB,qBAAqB,CACzC,MAAM,EAAE,SAAS,EACjB,IAAI,EAAE,MAAM,EACZ,QAAQ,EAAE,SAAS,MAAM,EAAE,EAC3B,QAAQ,EAAE,MAAM,GAAG,SAAS,EAC5B,IAAI,EAAE,eAAe,GAAG,SAAS,EACjC,MAAM,EAAE,OAAO,EACf,aAAa,CAAC,EAAE,aAAa,GAC5B,OAAO,CAAC,MAAM,CAAC,CA+CjB"}
1
+ {"version":3,"file":"report.d.ts","sourceRoot":"","sources":["../../src/bakeoff/report.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAMH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAG3C,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AACvD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AACjD,OAAO,KAAK,EAAE,eAAe,EAAkB,MAAM,SAAS,CAAC;AAC/D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AA2CpD;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,QAAQ,EAAE,eAAe,GAAG,MAAM,GAAG,IAAI,CAexE;AAED;;;;GAIG;AACH,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,+EAA+E;IAC/E,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,gFAAgF;IAChF,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAClC,QAAQ,CAAC,WAAW,EAAE,mBAAmB,CAAC;CAC3C;AAED;;;;;;GAMG;AACH,wBAAgB,0BAA0B,CAAC,IAAI,EAAE,SAAS,gBAAgB,EAAE,GAAG,MAAM,EAAE,CAgCtF;AAED;;;;;GAKG;AACH,wBAAgB,yBAAyB,CAAC,QAAQ,EAAE,eAAe,GAAG,MAAM,EAAE,CAiB7E;AAED;;;;;GAKG;AACH,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,eAAe,GAAG,MAAM,EAAE,CAavE;AAED,wBAAgB,WAAW,CAAC,QAAQ,EAAE,eAAe,EAAE,aAAa,GAAE,SAAS,gBAAgB,EAAO,GAAG,IAAI,CA8D5G;AAED;;;;;;GAMG;AACH,wBAAsB,qBAAqB,CACzC,MAAM,EAAE,SAAS,EACjB,IAAI,EAAE,MAAM,EACZ,QAAQ,EAAE,SAAS,MAAM,EAAE,EAC3B,QAAQ,EAAE,MAAM,GAAG,SAAS,EAC5B,IAAI,EAAE,eAAe,GAAG,SAAS,EACjC,MAAM,EAAE,OAAO,EACf,aAAa,CAAC,EAAE,aAAa,GAC5B,OAAO,CAAC,MAAM,CAAC,CA+CjB"}