modelrig 0.4.0 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (189) hide show
  1. package/CHANGELOG.md +238 -0
  2. package/README.md +65 -3
  3. package/dist/adapters/anthropic.d.ts.map +1 -1
  4. package/dist/adapters/anthropic.js +36 -7
  5. package/dist/adapters/attempt-meta-map.d.ts +63 -0
  6. package/dist/adapters/attempt-meta-map.d.ts.map +1 -0
  7. package/dist/adapters/attempt-meta-map.js +154 -0
  8. package/dist/adapters/deepseek.d.ts.map +1 -1
  9. package/dist/adapters/deepseek.js +25 -2
  10. package/dist/adapters/gemini.d.ts +39 -0
  11. package/dist/adapters/gemini.d.ts.map +1 -1
  12. package/dist/adapters/gemini.js +82 -4
  13. package/dist/adapters/grok.d.ts.map +1 -1
  14. package/dist/adapters/grok.js +25 -2
  15. package/dist/adapters/openai-compat.d.ts.map +1 -1
  16. package/dist/adapters/openai-compat.js +25 -2
  17. package/dist/adapters/openai-stream.d.ts +2 -0
  18. package/dist/adapters/openai-stream.d.ts.map +1 -1
  19. package/dist/adapters/openai-stream.js +23 -3
  20. package/dist/adapters/openai.d.ts.map +1 -1
  21. package/dist/adapters/openai.js +36 -6
  22. package/dist/artifacts/artifact.d.ts +3 -2
  23. package/dist/artifacts/artifact.d.ts.map +1 -1
  24. package/dist/artifacts/artifact.js +35 -9
  25. package/dist/artifacts/classification.d.ts +27 -0
  26. package/dist/artifacts/classification.d.ts.map +1 -0
  27. package/dist/artifacts/classification.js +38 -0
  28. package/dist/artifacts/index.d.ts +3 -1
  29. package/dist/artifacts/index.d.ts.map +1 -1
  30. package/dist/artifacts/index.js +10 -1
  31. package/dist/artifacts/run-context.d.ts +33 -2
  32. package/dist/artifacts/run-context.d.ts.map +1 -1
  33. package/dist/artifacts/run-context.js +95 -1
  34. package/dist/artifacts/scrub.d.ts +26 -0
  35. package/dist/artifacts/scrub.d.ts.map +1 -1
  36. package/dist/artifacts/scrub.js +53 -0
  37. package/dist/artifacts/step.d.ts +9 -0
  38. package/dist/artifacts/step.d.ts.map +1 -1
  39. package/dist/artifacts/step.js +9 -0
  40. package/dist/bakeoff/engine.d.ts +46 -4
  41. package/dist/bakeoff/engine.d.ts.map +1 -1
  42. package/dist/bakeoff/engine.js +269 -19
  43. package/dist/bakeoff/eval-cases.d.ts +5 -0
  44. package/dist/bakeoff/eval-cases.d.ts.map +1 -1
  45. package/dist/bakeoff/eval-cases.js +3 -0
  46. package/dist/bakeoff/judge.d.ts +39 -0
  47. package/dist/bakeoff/judge.d.ts.map +1 -0
  48. package/dist/bakeoff/judge.js +102 -0
  49. package/dist/bakeoff/quality.d.ts +44 -0
  50. package/dist/bakeoff/quality.d.ts.map +1 -0
  51. package/dist/bakeoff/quality.js +126 -0
  52. package/dist/bakeoff/reference.d.ts +34 -0
  53. package/dist/bakeoff/reference.d.ts.map +1 -0
  54. package/dist/bakeoff/reference.js +68 -0
  55. package/dist/bakeoff/replay.d.ts +25 -0
  56. package/dist/bakeoff/replay.d.ts.map +1 -1
  57. package/dist/bakeoff/replay.js +65 -2
  58. package/dist/bakeoff/report.d.ts +8 -1
  59. package/dist/bakeoff/report.d.ts.map +1 -1
  60. package/dist/bakeoff/report.js +83 -12
  61. package/dist/bakeoff/samples.d.ts +35 -2
  62. package/dist/bakeoff/samples.d.ts.map +1 -1
  63. package/dist/bakeoff/samples.js +67 -6
  64. package/dist/bakeoff/stats.d.ts +17 -1
  65. package/dist/bakeoff/stats.d.ts.map +1 -1
  66. package/dist/bakeoff/stats.js +67 -0
  67. package/dist/bakeoff/types.d.ts +76 -5
  68. package/dist/bakeoff/types.d.ts.map +1 -1
  69. package/dist/bakeoff/types.js +4 -3
  70. package/dist/cli-query.d.ts +14 -0
  71. package/dist/cli-query.d.ts.map +1 -0
  72. package/dist/cli-query.js +59 -0
  73. package/dist/cli.d.ts +2 -1
  74. package/dist/cli.d.ts.map +1 -1
  75. package/dist/cli.js +47 -24
  76. package/dist/config.d.ts +6 -0
  77. package/dist/config.d.ts.map +1 -1
  78. package/dist/config.js +14 -0
  79. package/dist/detect/detector.d.ts +82 -0
  80. package/dist/detect/detector.d.ts.map +1 -0
  81. package/dist/detect/detector.js +23 -0
  82. package/dist/index.d.ts +34 -23
  83. package/dist/index.d.ts.map +1 -1
  84. package/dist/index.js +49 -12
  85. package/dist/query/analytics.d.ts +84 -0
  86. package/dist/query/analytics.d.ts.map +1 -0
  87. package/dist/query/analytics.js +280 -0
  88. package/dist/query/columns.d.ts +59 -0
  89. package/dist/query/columns.d.ts.map +1 -0
  90. package/dist/query/columns.js +89 -0
  91. package/dist/query/grammar.d.ts +74 -7
  92. package/dist/query/grammar.d.ts.map +1 -1
  93. package/dist/query/grammar.js +179 -71
  94. package/dist/query/saved.d.ts +26 -0
  95. package/dist/query/saved.d.ts.map +1 -0
  96. package/dist/query/saved.js +51 -0
  97. package/dist/registry/select.d.ts +59 -0
  98. package/dist/registry/select.d.ts.map +1 -1
  99. package/dist/registry/select.js +66 -0
  100. package/dist/routes/loader-warnings.d.ts +28 -0
  101. package/dist/routes/loader-warnings.d.ts.map +1 -0
  102. package/dist/routes/loader-warnings.js +56 -0
  103. package/dist/routes/loader.d.ts +1 -11
  104. package/dist/routes/loader.d.ts.map +1 -1
  105. package/dist/routes/loader.js +54 -31
  106. package/dist/routes/objective-loader.d.ts +35 -0
  107. package/dist/routes/objective-loader.d.ts.map +1 -0
  108. package/dist/routes/objective-loader.js +286 -0
  109. package/dist/run/accept.d.ts +43 -0
  110. package/dist/run/accept.d.ts.map +1 -0
  111. package/dist/run/accept.js +65 -0
  112. package/dist/run/cost.d.ts +13 -0
  113. package/dist/run/cost.d.ts.map +1 -1
  114. package/dist/run/cost.js +27 -0
  115. package/dist/run/dispatch.d.ts +24 -1
  116. package/dist/run/dispatch.d.ts.map +1 -1
  117. package/dist/run/dispatch.js +19 -0
  118. package/dist/run/ladder-key.d.ts +4 -0
  119. package/dist/run/ladder-key.d.ts.map +1 -1
  120. package/dist/run/ladder-key.js +3 -0
  121. package/dist/run/quality-gate.d.ts +39 -0
  122. package/dist/run/quality-gate.d.ts.map +1 -0
  123. package/dist/run/quality-gate.js +70 -0
  124. package/dist/run/raw-defaults.d.ts +52 -0
  125. package/dist/run/raw-defaults.d.ts.map +1 -0
  126. package/dist/run/raw-defaults.js +123 -0
  127. package/dist/run/raw.d.ts +12 -0
  128. package/dist/run/raw.d.ts.map +1 -1
  129. package/dist/run/raw.js +107 -8
  130. package/dist/run/record-attempt.d.ts.map +1 -1
  131. package/dist/run/record-attempt.js +22 -1
  132. package/dist/run/resolve.d.ts +17 -1
  133. package/dist/run/resolve.d.ts.map +1 -1
  134. package/dist/run/resolve.js +20 -4
  135. package/dist/run/routing-docs.d.ts.map +1 -1
  136. package/dist/run/routing-docs.js +12 -5
  137. package/dist/run/run.d.ts +7 -1
  138. package/dist/run/run.d.ts.map +1 -1
  139. package/dist/run/run.js +54 -69
  140. package/dist/run/variant.d.ts +43 -0
  141. package/dist/run/variant.d.ts.map +1 -0
  142. package/dist/run/variant.js +86 -0
  143. package/dist/savings/emit.d.ts +33 -0
  144. package/dist/savings/emit.d.ts.map +1 -1
  145. package/dist/savings/emit.js +75 -3
  146. package/dist/settings/index.d.ts +1 -1
  147. package/dist/settings/index.d.ts.map +1 -1
  148. package/dist/settings/index.js +2 -1
  149. package/dist/settings/task-settings.d.ts +10 -0
  150. package/dist/settings/task-settings.d.ts.map +1 -1
  151. package/dist/settings/task-settings.js +17 -0
  152. package/dist/telemetry/attempt-meta.d.ts +178 -0
  153. package/dist/telemetry/attempt-meta.d.ts.map +1 -0
  154. package/dist/telemetry/attempt-meta.js +319 -0
  155. package/dist/telemetry/exporter.d.ts.map +1 -1
  156. package/dist/telemetry/exporter.js +6 -0
  157. package/dist/telemetry/ingest-protocol.d.ts +11 -0
  158. package/dist/telemetry/ingest-protocol.d.ts.map +1 -1
  159. package/dist/telemetry/ingest-protocol.js +3 -0
  160. package/dist/telemetry/otel-map.d.ts +10 -0
  161. package/dist/telemetry/otel-map.d.ts.map +1 -1
  162. package/dist/telemetry/otel-map.js +64 -0
  163. package/dist/telemetry/store-migrations.d.ts.map +1 -1
  164. package/dist/telemetry/store-migrations.js +15 -0
  165. package/dist/telemetry/store.d.ts.map +1 -1
  166. package/dist/telemetry/store.js +25 -7
  167. package/dist/types/failures-teaching.d.ts.map +1 -1
  168. package/dist/types/failures-teaching.js +20 -0
  169. package/dist/types/failures.d.ts +1 -1
  170. package/dist/types/failures.d.ts.map +1 -1
  171. package/dist/types/failures.js +1 -0
  172. package/dist/types/provider.d.ts +9 -0
  173. package/dist/types/provider.d.ts.map +1 -1
  174. package/dist/types/quality.d.ts +41 -0
  175. package/dist/types/quality.d.ts.map +1 -0
  176. package/dist/types/quality.js +12 -0
  177. package/dist/types/route.d.ts +120 -0
  178. package/dist/types/route.d.ts.map +1 -1
  179. package/dist/types/telemetry.d.ts +13 -0
  180. package/dist/types/telemetry.d.ts.map +1 -1
  181. package/dist/validate-cli.d.ts.map +1 -1
  182. package/dist/validate-cli.js +10 -0
  183. package/dist/watch/evaluate.d.ts +27 -1
  184. package/dist/watch/evaluate.d.ts.map +1 -1
  185. package/dist/watch/evaluate.js +13 -0
  186. package/dist/watch/proposals.d.ts +28 -0
  187. package/dist/watch/proposals.d.ts.map +1 -1
  188. package/dist/watch/proposals.js +51 -0
  189. package/package.json +1 -1
package/dist/run/raw.js CHANGED
@@ -55,12 +55,38 @@ const grok_1 = require("../adapters/grok");
55
55
  const openai_1 = require("../adapters/openai");
56
56
  const openai_compat_1 = require("../adapters/openai-compat");
57
57
  const cost_1 = require("./cost");
58
+ const raw_defaults_1 = require("./raw-defaults");
58
59
  const resolve_1 = require("./resolve");
59
60
  const raw_support_1 = require("./raw-support");
60
61
  const failures_1 = require("../types/failures");
61
- /** Absent a route timeout, the raw lane uses a generous default — a passthrough
62
- * should not be shorter-fused than the underlying provider SDK. */
63
- const DEFAULT_RAW_TIMEOUT_MS = 120000;
62
+ const run_context_1 = require("../artifacts/run-context");
63
+ const manifest_1 = require("../routes/manifest");
64
+ const attempt_meta_1 = require("../telemetry/attempt-meta");
65
+ /**
66
+ * R1 — resolve the effective raw-lane timeout. An explicit `timeoutMs` always
67
+ * wins; absent, the fallback is TIER-AWARE (`RAW_TIMEOUT_DEFAULT_MS`), because a
68
+ * passthrough should not be shorter-fused than the tier it advertises (a flex
69
+ * grounded prompt needs minutes, not the old 120 s). No pre-dispatch rejection —
70
+ * a short explicit timeout is a choice, and the teaching arrives on the first
71
+ * `timeout` failure (`rawTimeoutFixHint`), which is when the caller is looking.
72
+ */
73
+ function resolveRawTimeoutMs(input) {
74
+ return input.timeoutMs ?? raw_defaults_1.RAW_TIMEOUT_DEFAULT_MS[input.serviceTier ?? "standard"];
75
+ }
76
+ /** R1 — the teaching `fixHint` attached to a `timeout` failure on the raw lane:
77
+ * it names the elapsed budget, the tier, that tier's advertised target, and the
78
+ * two fixes (raise `timeoutMs`, or drop to `standard`). Built from
79
+ * `RAW_TIMEOUT_DEFAULT_MS` so the default it quotes can never drift. */
80
+ function rawTimeoutFixHint(input, tier) {
81
+ const deadlineSec = resolveRawTimeoutMs(input) / 1000;
82
+ const budgetMin = raw_defaults_1.RAW_TIMEOUT_DEFAULT_MS[tier] / 60000;
83
+ const defaultSec = raw_defaults_1.RAW_TIMEOUT_DEFAULT_MS[tier] / 1000;
84
+ // Only flex has a cheaper-latency tier to drop to; at standard/priority the
85
+ // only fix is a longer deadline (never tell a caller to switch to the tier
86
+ // they are already on).
87
+ const tail = tier === "flex" ? ' or use serviceTier: "standard"' : "";
88
+ return `timed out at the ${deadlineSec}s deadline at tier ${tier} — ${tier} targets up to ${budgetMin} min; raise timeoutMs (default ${defaultSec}s)${tail}`;
89
+ }
64
90
  /** Conservative compat-host capability baseline — parity with createRig's
65
91
  * COMPAT_STATIC. Inconsequential for raw calls (outputSchema is always null, so
66
92
  * no JSON shaping fires), kept aligned so a raw and a routed compat call shape
@@ -181,7 +207,8 @@ function buildRawRequest(candidate, input) {
181
207
  // CE-2 F2: forward the DECLARED tier (default "standard"); the meta/row
182
208
  // honesty fix in finalizeRaw reports this same value, never a hardcode.
183
209
  requestedTier: input.serviceTier ?? "standard",
184
- timeoutMs: input.timeoutMs ?? DEFAULT_RAW_TIMEOUT_MS,
210
+ // R1: tier-aware default (RAW_TIMEOUT_DEFAULT_MS); explicit timeoutMs wins.
211
+ timeoutMs: resolveRawTimeoutMs(input),
185
212
  ...(input.maxOutputTokens !== undefined ? { maxOutputTokens: input.maxOutputTokens } : {}),
186
213
  ...(sampling !== undefined ? { sampling } : {}),
187
214
  // CE-2 F1: `{ mode: "native" }` → the adapter's native-grounding directive.
@@ -241,16 +268,39 @@ function recordRawRow(telemetry, p) {
241
268
  callId: p.callId,
242
269
  attemptSeq: 1,
243
270
  candidateIndex: 0,
271
+ meta: p.meta,
244
272
  };
245
273
  telemetry.record(row);
246
274
  }
275
+ /**
276
+ * Read the ambient run context at the ENTRY of a raw call (spec §4) and, if
277
+ * present, resolve the step key `input.stepKey ?? tags.step ?? provider/model#n`
278
+ * (validated) — so `finalizeRaw` records a ground-truth step exactly like the
279
+ * routed lane. Also runs the required-dimension hygiene warning (spec §4),
280
+ * unconditionally, as the routed lane does. Fail-open: no context ⇒ null and no
281
+ * step is ever recorded; the dispatch is byte-identical to pre-G1.
282
+ */
283
+ function prepareRawStep(deps, input) {
284
+ (0, manifest_1.warnMissingRequiredDimensions)(deps.manifest ?? null, input.tags ?? {});
285
+ const runCtx = (0, run_context_1.getActiveRunContext)();
286
+ if (runCtx === null)
287
+ return null;
288
+ const rawModelId = `${input.provider}/${input.model}`;
289
+ const stepKey = (0, run_context_1.resolveStepKey)(input.stepKey, input.tags?.step, () => runCtx.nextStepKey(rawModelId));
290
+ return { runCtx, stepKey, startedAtIso: new Date().toISOString() };
291
+ }
247
292
  /**
248
293
  * Turn an `AdapterResult` into a RunResult, recording exactly one telemetry row.
249
294
  * Shared by the buffered and streaming paths so cost/telemetry are byte-identical.
250
295
  * On a failure result: record the (possibly billed) row, then throw
251
296
  * `RigFailureError` — the server maps it to a status exactly like the slug lane.
297
+ *
298
+ * G1: when a run context was captured at entry (`step`), a ground-truth step is
299
+ * emitted after the telemetry row on BOTH branches (success and failure) —
300
+ * fail-open, so a buffer error there never affects the returned result. No
301
+ * artifact CONTENT is written on the raw lane (spec F3): steps are metadata.
252
302
  */
253
- function finalizeRaw(deps, input, result, latencyMs) {
303
+ function finalizeRaw(deps, input, result, latencyMs, step) {
254
304
  const unpricedRatePerTok = (deps.unpricedRateUsdPerMTok ?? cost_1.DEFAULT_UNPRICED_RATE_USD_PER_MTOK) / 1000000;
255
305
  const rawModelId = `${input.provider}/${input.model}`;
256
306
  // One call_id for this raw invocation — shared by the (single) recorded row and
@@ -260,6 +310,34 @@ function finalizeRaw(deps, input, result, latencyMs) {
260
310
  const requestedTier = input.serviceTier ?? "standard";
261
311
  // lane is SDK-owned and cannot be overridden by a caller-supplied tag.
262
312
  const tags = { ...(input.tags ?? {}), lane: "raw" };
313
+ // WI-1 (spec §2, §6.3 lane parity): the attempt envelope, built by the SAME
314
+ // producer + rendered-request canonicalization the routed lane uses. The raw
315
+ // lane never sends an output schema (unvalidated by design), so schema = null.
316
+ const declaredToolNames = input.tools?.map((t) => t.function.name);
317
+ const rawMeta = (usage) => (0, attempt_meta_1.summarizeAttemptMeta)({
318
+ ...(result.meta !== undefined ? { adapterMeta: result.meta } : {}),
319
+ renderedRequest: (0, attempt_meta_1.renderRequestForHash)({ system: input.systemPrompt, user: input.userPrompt, schema: null }),
320
+ outputText: result.ok ? result.text : null,
321
+ ...(declaredToolNames !== undefined ? { declaredToolNames } : {}),
322
+ provider: input.provider,
323
+ model: input.model,
324
+ ...(usage !== undefined ? { usage } : {}),
325
+ });
326
+ // G1: mirror this raw inference into a ground-truth step (parity with the
327
+ // routed lane's record-attempt). task = tags.step ?? provider/model (spec F1).
328
+ // Fail-open is handled inside emitGroundTruthStep (buffer errors are swallowed).
329
+ const emitStep = (inferenceId, failureClass) => {
330
+ if (step === null)
331
+ return;
332
+ step.runCtx.emitGroundTruthStep({
333
+ stepKey: step.stepKey,
334
+ task: input.tags?.step ?? rawModelId,
335
+ inferenceId,
336
+ startedAt: step.startedAtIso,
337
+ endedAt: new Date().toISOString(),
338
+ failureClass,
339
+ });
340
+ };
263
341
  if (!result.ok) {
264
342
  let costUsd = 0;
265
343
  let pricingMissing = false;
@@ -268,8 +346,9 @@ function finalizeRaw(deps, input, result, latencyMs) {
268
346
  costUsd = settled.costUsd;
269
347
  pricingMissing = settled.pricingMissing;
270
348
  }
349
+ const failId = (0, node_crypto_1.randomUUID)();
271
350
  recordRawRow(deps.telemetry, {
272
- id: (0, node_crypto_1.randomUUID)(),
351
+ id: failId,
273
352
  rawModelId,
274
353
  provider: input.provider,
275
354
  model: input.model,
@@ -281,8 +360,19 @@ function finalizeRaw(deps, input, result, latencyMs) {
281
360
  tags,
282
361
  pricingMissing,
283
362
  callId,
363
+ meta: rawMeta(result.usage),
284
364
  ...(input.cache !== undefined ? { cacheKey: input.cache.key } : {}),
285
365
  });
366
+ emitStep(failId, result.failure.class);
367
+ // R1: a raw-lane timeout is the #245 footgun — attach the tier-aware teaching
368
+ // hint (the customer is looking at THIS error). Any other failure surfaces
369
+ // unchanged. Overrides any adapter-set hint: the tier + budget is the fix.
370
+ if (result.failure.class === "timeout") {
371
+ throw new failures_1.RigFailureError({
372
+ ...result.failure,
373
+ fixHint: rawTimeoutFixHint(input, requestedTier),
374
+ });
375
+ }
286
376
  throw new failures_1.RigFailureError(result.failure);
287
377
  }
288
378
  const { costUsd, pricingMissing } = (0, cost_1.attemptCost)(input.provider, input.model, result.usage, unpricedRatePerTok);
@@ -300,8 +390,10 @@ function finalizeRaw(deps, input, result, latencyMs) {
300
390
  tags,
301
391
  pricingMissing,
302
392
  callId,
393
+ meta: rawMeta(result.usage),
303
394
  ...(input.cache !== undefined ? { cacheKey: input.cache.key } : {}),
304
395
  });
396
+ emitStep(id, null);
305
397
  const toolCalls = result.toolCalls;
306
398
  const isToolTurn = toolCalls !== undefined && toolCalls.length > 0;
307
399
  const meta = {
@@ -332,11 +424,14 @@ async function runRaw(deps, input) {
332
424
  // CE-2 F1/F3: fail closed on an unsupported declared directive BEFORE building
333
425
  // an adapter — no meter, no provider call, no telemetry row.
334
426
  assertRawKnobsSupported(input);
427
+ // G1: bind the ambient run context at ENTRY (ALS boundary) so a raw call inside
428
+ // a run.start records a ground-truth step (spec §4). Null ⇒ uninstrumented.
429
+ const step = prepareRawStep(deps, input);
335
430
  const adapter = buildRawAdapter(input.provider, input.apiKey);
336
431
  const candidate = (0, resolve_1.mintRawCandidate)(input.provider, input.model);
337
432
  const started = Date.now();
338
433
  const result = await adapter.call(buildRawRequest(candidate, input));
339
- return finalizeRaw(deps, input, result, Date.now() - started);
434
+ return finalizeRaw(deps, input, result, Date.now() - started, step);
340
435
  }
341
436
  function streamEndedFailure(input) {
342
437
  return {
@@ -358,6 +453,10 @@ function streamEndedFailure(input) {
358
453
  * the raw lane has no schema and the streamed bytes ARE the answer.
359
454
  */
360
455
  function runRawStream(deps, input) {
456
+ // G1: read the ambient run context HERE — at the synchronous entry, the ALS
457
+ // boundary — NOT inside the async generator below (which defers to drain time,
458
+ // possibly on a different async context, where getActiveRunContext would miss).
459
+ const step = prepareRawStep(deps, input);
361
460
  let settle;
362
461
  let fail;
363
462
  const result = new Promise((res, rej) => {
@@ -394,7 +493,7 @@ function runRawStream(deps, input) {
394
493
  else {
395
494
  adapterResult = await adapter.call(req);
396
495
  }
397
- const runResult = finalizeRaw(deps, input, adapterResult, Date.now() - started);
496
+ const runResult = finalizeRaw(deps, input, adapterResult, Date.now() - started, step);
398
497
  settle(runResult);
399
498
  yield { type: "final", result: runResult };
400
499
  }
@@ -1 +1 @@
1
- {"version":3,"file":"record-attempt.d.ts","sourceRoot":"","sources":["../../src/run/record-attempt.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAGH,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AACnD,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AACjE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAE3C,OAAO,KAAK,EAAE,WAAW,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAC;AACvE,OAAO,KAAK,EAAgB,aAAa,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAGtF,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAEpD,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,SAAS,EAAE,cAAc,CAAC;IACnC,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAC;IAC5B,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAC;IAC7B;;;gDAG4C;IAC5C,QAAQ,CAAC,MAAM,EAAE,eAAe,CAAC;IACjC,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC;IAChC,QAAQ,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC,kFAAkF;IAClF,QAAQ,CAAC,MAAM,EAAE,gBAAgB,GAAG,IAAI,CAAC;IACzC,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC;;qFAEiF;IACjF,QAAQ,CAAC,KAAK,CAAC,EAAE,UAAU,CAAC,OAAO,CAAC,CAAC;IACrC;;;wEAGoE;IACpE,QAAQ,CAAC,kBAAkB,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;CAClD;AAED;;;;;GAKG;AACH,wBAAgB,MAAM,CACpB,OAAO,EAAE,gBAAgB,EACzB,WAAW,EAAE,OAAO,EACpB,cAAc,EAAE,MAAM,GAAG,IAAI,EAC7B,iBAAiB,EAAE,GAAG,CAAC,MAAM,CAAC,GAC7B,aAAa,CAUf;AAED;;yCAEyC;AACzC,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,iBAAiB,GAAG,CAAC,OAAO,EAAE,gBAAgB,KAAK,MAAM,CA+FlG"}
1
+ {"version":3,"file":"record-attempt.d.ts","sourceRoot":"","sources":["../../src/run/record-attempt.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAGH,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AACnD,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AACjE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAE3C,OAAO,KAAK,EAAE,WAAW,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAC;AACvE,OAAO,KAAK,EAAgB,aAAa,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAGtF,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAGpD,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,SAAS,EAAE,cAAc,CAAC;IACnC,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAC;IAC5B,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAC;IAC7B;;;gDAG4C;IAC5C,QAAQ,CAAC,MAAM,EAAE,eAAe,CAAC;IACjC,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC;IAChC,QAAQ,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC,kFAAkF;IAClF,QAAQ,CAAC,MAAM,EAAE,gBAAgB,GAAG,IAAI,CAAC;IACzC,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC;;qFAEiF;IACjF,QAAQ,CAAC,KAAK,CAAC,EAAE,UAAU,CAAC,OAAO,CAAC,CAAC;IACrC;;;wEAGoE;IACpE,QAAQ,CAAC,kBAAkB,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;CAClD;AAED;;;;;GAKG;AACH,wBAAgB,MAAM,CACpB,OAAO,EAAE,gBAAgB,EACzB,WAAW,EAAE,OAAO,EACpB,cAAc,EAAE,MAAM,GAAG,IAAI,EAC7B,iBAAiB,EAAE,GAAG,CAAC,MAAM,CAAC,GAC7B,aAAa,CAUf;AAED;;yCAEyC;AACzC,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,iBAAiB,GAAG,CAAC,OAAO,EAAE,gBAAgB,KAAK,MAAM,CAmHlG"}
@@ -17,6 +17,7 @@ const node_crypto_1 = require("node:crypto");
17
17
  const failures_1 = require("../types/failures");
18
18
  const cache_stamp_1 = require("./cache-stamp");
19
19
  const cache_handle_1 = require("./cache-handle");
20
+ const attempt_meta_1 = require("../telemetry/attempt-meta");
20
21
  /**
21
22
  * Classify a recorded row's ladder rung (R1 review, F1). `primaryDispatched`
22
23
  * carries the candidate indices already given a primary dispatch on this call —
@@ -91,6 +92,23 @@ function createRecordAttempt(deps) {
91
92
  // pre-dispatch terminal failure, then primary-vs-retry by whether this
92
93
  // candidate index was already dispatched on this call.
93
94
  const rung = rungOf(attempt, preDispatch, ladderKey.candidateIndex, primaryDispatched);
95
+ // WI-1 (spec §2): the metadata-safe attempt envelope. The single producer
96
+ // hashes the rendered request/output, computes cost components + context
97
+ // bucket, and enforces the classification rule; it never throws (a failure
98
+ // returns null — the row is byte-identical). Built from the adapter's native
99
+ // meta (metaInputs) plus THIS row's provider/model/usage.
100
+ const meta = (0, attempt_meta_1.summarizeAttemptMeta)({
101
+ ...(attempt.metaInputs ?? {}),
102
+ provider: attempt.candidate.provider,
103
+ model: attempt.candidate.model,
104
+ usage: {
105
+ tokensIn: attempt.tokensIn,
106
+ tokensOut: attempt.tokensOut,
107
+ tokensCached: attempt.tokensCached,
108
+ ...(attempt.tokensCacheWrite !== undefined ? { tokensCacheWrite: attempt.tokensCacheWrite } : {}),
109
+ ...(attempt.tokensCacheWrite1h !== undefined ? { tokensCacheWrite1h: attempt.tokensCacheWrite1h } : {}),
110
+ },
111
+ });
94
112
  const row = {
95
113
  id: (0, node_crypto_1.randomUUID)(),
96
114
  ts: new Date().toISOString(),
@@ -113,7 +131,9 @@ function createRecordAttempt(deps) {
113
131
  ttfbMs: attempt.ttfbMs ?? null,
114
132
  failureClass: attempt.failure?.class ?? null,
115
133
  attemptsJson: JSON.stringify(ledger.snapshot()),
116
- tagsJson: JSON.stringify({ ...effectiveTags, rung }),
134
+ // `rung` is stamped LAST so a caller's extraTags (Q1 quality tags) can
135
+ // never override the recorder-owned rung.
136
+ tagsJson: JSON.stringify({ ...effectiveTags, ...attempt.extraTags, rung }),
117
137
  fingerprintCell: route.fingerprintCell,
118
138
  servedVariant,
119
139
  repairedBy: attempt.repairedBy ?? null,
@@ -121,6 +141,7 @@ function createRecordAttempt(deps) {
121
141
  callId: ladderKey.callId,
122
142
  attemptSeq: ladderKey.attemptSeq,
123
143
  candidateIndex: ladderKey.candidateIndex,
144
+ meta,
124
145
  };
125
146
  telemetry.record(row);
126
147
  // Mirror this inference into a ground-truth step attempt (fail-open — the
@@ -9,6 +9,16 @@ import type { RouteBundle, RunOptions } from "../types/route";
9
9
  import type { CandidateRef, CapabilityFlag, ProviderId } from "../types/provider";
10
10
  /** Static capability stub — replaced by the probed registry in Phase 2. */
11
11
  export type CapabilityLookup = (provider: ProviderId, model: string) => ReadonlySet<CapabilityFlag>;
12
+ /**
13
+ * Resolution mode (routing-console spec §4.4, R3). `live` is every production
14
+ * path (`rig.run`, `runStream`, `runBundle`, the hosted gateway slug lane) and
15
+ * DROPS `trial` candidates before minting — a declared trial candidate is never
16
+ * served live. `trial` is the bake-off replay path ONLY (`replayVariant`), which
17
+ * keeps trial candidates so a customer can measure a new model on captured
18
+ * traffic before promoting it. Default `live` — every existing caller is
19
+ * byte-for-byte unchanged (no trial candidates ⇒ the filter is a no-op anyway).
20
+ */
21
+ export type RunMode = "live" | "trial";
12
22
  export interface CandidateIterator {
13
23
  /** Next eligible candidate, or null when the set is exhausted. */
14
24
  next(): CandidateRef | null;
@@ -17,6 +27,12 @@ export interface CandidateIterator {
17
27
  * the first `next()`; unchanged by a `next()` that returns null. */
18
28
  readonly index: number;
19
29
  }
30
+ /** The live-resolver capability filter. Exported so the declare-first challenger
31
+ * eligibility (`registry/select.ts` `requirementPredicates` +
32
+ * `evaluateChallengerEligibility`) can pin itself in agreement with what would
33
+ * actually serve — an eligibility that drifted from the live filter would offer
34
+ * a challenger the route could never dispatch. */
35
+ export declare function satisfiesRequirements(route: RouteBundle, flags: ReadonlySet<CapabilityFlag>, searchAvailable: boolean): boolean;
20
36
  /**
21
37
  * Request-time candidate filters (G-2) — optional, applied by resolveCandidates
22
38
  * BEFORE the prefer:[cost] sort. Distinct from route.require (capability + ZDR):
@@ -60,7 +76,7 @@ export interface WarmthShadowLog {
60
76
  }[];
61
77
  readonly diverged: boolean;
62
78
  }
63
- export declare function resolveCandidates(route: RouteBundle, capabilities: CapabilityLookup, costUsdPerMTok: (provider: ProviderId, model: string) => number | null, searchAvailable?: boolean, filter?: CandidateFilter, shadowWarmth?: WarmthShadowOptions): CandidateIterator;
79
+ export declare function resolveCandidates(route: RouteBundle, capabilities: CapabilityLookup, costUsdPerMTok: (provider: ProviderId, model: string) => number | null, searchAvailable?: boolean, filter?: CandidateFilter, shadowWarmth?: WarmthShadowOptions, mode?: RunMode): CandidateIterator;
64
80
  /**
65
81
  * Per-run requirement opt-ins (G-3). A caller can demand zero-retention routing
66
82
  * for a single run without editing the route bundle: `zero_retention` is unioned
@@ -1 +1 @@
1
- {"version":3,"file":"resolve.d.ts","sourceRoot":"","sources":["../../src/run/resolve.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAC9D,OAAO,KAAK,EAAE,YAAY,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAElF,2EAA2E;AAC3E,MAAM,MAAM,gBAAgB,GAAG,CAC7B,QAAQ,EAAE,UAAU,EACpB,KAAK,EAAE,MAAM,KACV,WAAW,CAAC,cAAc,CAAC,CAAC;AAEjC,MAAM,WAAW,iBAAiB;IAChC,kEAAkE;IAClE,IAAI,IAAI,YAAY,GAAG,IAAI,CAAC;IAC5B;;wEAEoE;IACpE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;CACxB;AA6CD;;;;;GAKG;AACH,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,kBAAkB,CAAC,EAAE,MAAM,CAAC;IACrC,QAAQ,CAAC,IAAI,CAAC,EAAE,SAAS,UAAU,EAAE,CAAC;IACtC,QAAQ,CAAC,MAAM,CAAC,EAAE,SAAS,UAAU,EAAE,CAAC;CACzC;AAED;;;;;;;;;GASG;AACH,MAAM,WAAW,mBAAmB;IAClC;;oFAEgF;IAChF,QAAQ,CAAC,uBAAuB,EAAE,CAAC,QAAQ,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,KAAK,MAAM,GAAG,IAAI,CAAC;IACzF,gEAAgE;IAChE,QAAQ,CAAC,GAAG,EAAE,CAAC,MAAM,EAAE,eAAe,KAAK,IAAI,CAAC;CACjD;AAED;kFACkF;AAClF,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,SAAS;QAAE,QAAQ,CAAC,QAAQ,EAAE,UAAU,CAAC;QAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IACtF,QAAQ,CAAC,WAAW,EAAE,SAAS;QAAE,QAAQ,CAAC,QAAQ,EAAE,UAAU,CAAC;QAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IAC3F,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC;CAC5B;AAED,wBAAgB,iBAAiB,CAC/B,KAAK,EAAE,WAAW,EAClB,YAAY,EAAE,gBAAgB,EAC9B,cAAc,EAAE,CAAC,QAAQ,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,KAAK,MAAM,GAAG,IAAI,EACtE,eAAe,UAAQ,EACvB,MAAM,CAAC,EAAE,eAAe,EACxB,YAAY,CAAC,EAAE,mBAAmB,GACjC,iBAAiB,CAwFnB;AAED;;;;;;;GAOG;AACH,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,WAAW,EAAE,aAAa,EAAE,OAAO,GAAG,WAAW,CAM5F;AAED;mFACmF;AACnF,wBAAgB,0BAA0B,CAAC,IAAI,EAAE,UAAU,GAAG,eAAe,GAAG,SAAS,CAUxF;AAED;;gDAEgD;AAChD,wBAAgB,aAAa,CAAC,IAAI,EAAE,UAAU,GAAG,MAAM,CAMtD;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,GAAG,YAAY,CAElF;AAED;;;;GAIG;AACH,wBAAgB,wBAAwB,CACtC,KAAK,EAAE,WAAW,EAClB,IAAI,EAAE;IAAE,QAAQ,CAAC,QAAQ,EAAE,UAAU,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;CAAE,GAC9D,YAAY,GAAG,IAAI,CAMrB"}
1
+ {"version":3,"file":"resolve.d.ts","sourceRoot":"","sources":["../../src/run/resolve.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAC9D,OAAO,KAAK,EAAE,YAAY,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAElF,2EAA2E;AAC3E,MAAM,MAAM,gBAAgB,GAAG,CAC7B,QAAQ,EAAE,UAAU,EACpB,KAAK,EAAE,MAAM,KACV,WAAW,CAAC,cAAc,CAAC,CAAC;AAEjC;;;;;;;;GAQG;AACH,MAAM,MAAM,OAAO,GAAG,MAAM,GAAG,OAAO,CAAC;AAEvC,MAAM,WAAW,iBAAiB;IAChC,kEAAkE;IAClE,IAAI,IAAI,YAAY,GAAG,IAAI,CAAC;IAC5B;;wEAEoE;IACpE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;CACxB;AAED;;;;kDAIkD;AAClD,wBAAgB,qBAAqB,CACnC,KAAK,EAAE,WAAW,EAClB,KAAK,EAAE,WAAW,CAAC,cAAc,CAAC,EAClC,eAAe,EAAE,OAAO,GACvB,OAAO,CAqCT;AAED;;;;;GAKG;AACH,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,kBAAkB,CAAC,EAAE,MAAM,CAAC;IACrC,QAAQ,CAAC,IAAI,CAAC,EAAE,SAAS,UAAU,EAAE,CAAC;IACtC,QAAQ,CAAC,MAAM,CAAC,EAAE,SAAS,UAAU,EAAE,CAAC;CACzC;AAED;;;;;;;;;GASG;AACH,MAAM,WAAW,mBAAmB;IAClC;;oFAEgF;IAChF,QAAQ,CAAC,uBAAuB,EAAE,CAAC,QAAQ,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,KAAK,MAAM,GAAG,IAAI,CAAC;IACzF,gEAAgE;IAChE,QAAQ,CAAC,GAAG,EAAE,CAAC,MAAM,EAAE,eAAe,KAAK,IAAI,CAAC;CACjD;AAED;kFACkF;AAClF,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,SAAS;QAAE,QAAQ,CAAC,QAAQ,EAAE,UAAU,CAAC;QAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IACtF,QAAQ,CAAC,WAAW,EAAE,SAAS;QAAE,QAAQ,CAAC,QAAQ,EAAE,UAAU,CAAC;QAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IAC3F,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC;CAC5B;AAED,wBAAgB,iBAAiB,CAC/B,KAAK,EAAE,WAAW,EAClB,YAAY,EAAE,gBAAgB,EAC9B,cAAc,EAAE,CAAC,QAAQ,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,KAAK,MAAM,GAAG,IAAI,EACtE,eAAe,UAAQ,EACvB,MAAM,CAAC,EAAE,eAAe,EACxB,YAAY,CAAC,EAAE,mBAAmB,EAClC,IAAI,GAAE,OAAgB,GACrB,iBAAiB,CAkGnB;AAED;;;;;;;GAOG;AACH,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,WAAW,EAAE,aAAa,EAAE,OAAO,GAAG,WAAW,CAM5F;AAED;mFACmF;AACnF,wBAAgB,0BAA0B,CAAC,IAAI,EAAE,UAAU,GAAG,eAAe,GAAG,SAAS,CAUxF;AAED;;gDAEgD;AAChD,wBAAgB,aAAa,CAAC,IAAI,EAAE,UAAU,GAAG,MAAM,CAMtD;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,GAAG,YAAY,CAElF;AAED;;;;GAIG;AACH,wBAAgB,wBAAwB,CACtC,KAAK,EAAE,WAAW,EAClB,IAAI,EAAE;IAAE,QAAQ,CAAC,QAAQ,EAAE,UAAU,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;CAAE,GAC9D,YAAY,GAAG,IAAI,CAMrB"}
@@ -7,12 +7,18 @@
7
7
  * retryable failure the run loop asks for the next candidate.
8
8
  */
9
9
  Object.defineProperty(exports, "__esModule", { value: true });
10
+ exports.satisfiesRequirements = satisfiesRequirements;
10
11
  exports.resolveCandidates = resolveCandidates;
11
12
  exports.applyRunRequirements = applyRunRequirements;
12
13
  exports.candidateFilterFromOptions = candidateFilterFromOptions;
13
14
  exports.filterSummary = filterSummary;
14
15
  exports.mintRawCandidate = mintRawCandidate;
15
16
  exports.resolveDeclaredCandidate = resolveDeclaredCandidate;
17
+ /** The live-resolver capability filter. Exported so the declare-first challenger
18
+ * eligibility (`registry/select.ts` `requirementPredicates` +
19
+ * `evaluateChallengerEligibility`) can pin itself in agreement with what would
20
+ * actually serve — an eligibility that drifted from the live filter would offer
21
+ * a challenger the route could never dispatch. */
16
22
  function satisfiesRequirements(route, flags, searchAvailable) {
17
23
  for (const requirement of route.require) {
18
24
  switch (requirement) {
@@ -56,11 +62,21 @@ function satisfiesRequirements(route, flags, searchAvailable) {
56
62
  }
57
63
  return true;
58
64
  }
59
- function resolveCandidates(route, capabilities, costUsdPerMTok, searchAvailable = false, filter, shadowWarmth) {
60
- // Filter order (spec §4.1): provider only → provider ignore → capability+ZDR
61
- // → maxPrice → prefer:[cost]. When `filter` is absent every step is a no-op,
62
- // so the path is byte-for-byte the pre-G-2 behavior (back-compat).
65
+ function resolveCandidates(route, capabilities, costUsdPerMTok, searchAvailable = false, filter, shadowWarmth, mode = "live") {
66
+ // Filter order (spec §4.1): trial-drop → provider only → provider ignore →
67
+ // capability+ZDR → maxPrice → prefer:[cost]. When `filter` is absent every
68
+ // step is a no-op, so the path is byte-for-byte the pre-G-2 behavior
69
+ // (back-compat).
63
70
  let pool = route.candidates;
71
+ // Declare-first (spec §4.4): a `trial` candidate is declared but NEVER served
72
+ // live — drop it first, exactly as a `providers.ignore` entry is dropped, on
73
+ // every live path. Only bake-off replay (`mode: "trial"`) keeps it. The
74
+ // candidate-set invariant is untouched: `route.candidates` (the declared set
75
+ // the run loop's runtime guard checks against) still includes trial entries;
76
+ // they are simply never minted into a CandidateRef on a live path.
77
+ if (mode === "live") {
78
+ pool = pool.filter((c) => c.trial !== true);
79
+ }
64
80
  if (filter?.only && filter.only.length > 0) {
65
81
  const only = new Set(filter.only);
66
82
  pool = pool.filter((c) => only.has(c.provider));
@@ -1 +1 @@
1
- {"version":3,"file":"routing-docs.d.ts","sourceRoot":"","sources":["../../src/run/routing-docs.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAItD,MAAM,MAAM,cAAc,GAAG,WAAW,GAAG,UAAU,GAAG,QAAQ,GAAG,WAAW,CAAC;AAE/E,MAAM,WAAW,gBAAgB;IAC/B,sEAAsE;IACtE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,6DAA6D;IAC7D,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC;IAC/B,yEAAyE;IACzE,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;CAC7B;AAED,MAAM,WAAW,aAAa;IAC5B,6EAA6E;IAC7E,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,MAAM,EAAE,cAAc,CAAC;IAChC,kBAAkB;IAClB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,uDAAuD;IACvD,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,kDAAkD;IAClD,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,QAAQ,CAAC,EAAE,gBAAgB,GAAG;QAAE,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAA;KAAE,CAAC;CAC3E;AAED,yCAAyC;AACzC,eAAO,MAAM,qBAAqB,EAAE,QAAQ,CAAC,MAAM,CAAC,cAAc,EAAE,MAAM,CAAC,CAK1E,CAAC;AAEF;;;;;;GAMG;AACH,eAAO,MAAM,gBAAgB,EAAE,SAAS,aAAa,EA0FpD,CAAC;AAwBF;;;;;;;;GAQG;AACH,wBAAgB,qBAAqB,CAAC,SAAS,CAAC,EAAE,WAAW,CAAC,MAAM,CAAC,GAAG,MAAM,CAoB7E;AAED;;;;+DAI+D;AAC/D,wBAAgB,iBAAiB,IAAI,SAAS,YAAY,EAAE,CAE3D;AA0BD;;;;;GAKG;AACH,wBAAgB,oBAAoB,IAAI,MAAM,CAW7C;AAED,gFAAgF;AAChF,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,YAAY,GAAG,MAAM,CAE5D"}
1
+ {"version":3,"file":"routing-docs.d.ts","sourceRoot":"","sources":["../../src/run/routing-docs.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAItD,MAAM,MAAM,cAAc,GAAG,WAAW,GAAG,UAAU,GAAG,QAAQ,GAAG,WAAW,CAAC;AAE/E,MAAM,WAAW,gBAAgB;IAC/B,sEAAsE;IACtE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,6DAA6D;IAC7D,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC;IAC/B,yEAAyE;IACzE,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;CAC7B;AAED,MAAM,WAAW,aAAa;IAC5B,6EAA6E;IAC7E,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,MAAM,EAAE,cAAc,CAAC;IAChC,kBAAkB;IAClB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,uDAAuD;IACvD,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,kDAAkD;IAClD,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,QAAQ,CAAC,EAAE,gBAAgB,GAAG;QAAE,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAA;KAAE,CAAC;CAC3E;AAED,yCAAyC;AACzC,eAAO,MAAM,qBAAqB,EAAE,QAAQ,CAAC,MAAM,CAAC,cAAc,EAAE,MAAM,CAAC,CAK1E,CAAC;AAEF;;;;;;GAMG;AACH,eAAO,MAAM,gBAAgB,EAAE,SAAS,aAAa,EAiGpD,CAAC;AAwBF;;;;;;;;GAQG;AACH,wBAAgB,qBAAqB,CAAC,SAAS,CAAC,EAAE,WAAW,CAAC,MAAM,CAAC,GAAG,MAAM,CAoB7E;AAED;;;;+DAI+D;AAC/D,wBAAgB,iBAAiB,IAAI,SAAS,YAAY,EAAE,CAE3D;AA0BD;;;;;GAKG;AACH,wBAAgB,oBAAoB,IAAI,MAAM,CAW7C;AAED,gFAAgF;AAChF,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,YAAY,GAAG,MAAM,CAE5D"}
@@ -38,16 +38,23 @@ exports.ROUTING_BOUNDARY = [
38
38
  {
39
39
  id: "B1",
40
40
  status: "not_built",
41
- capability: "Quality / value-accuracy gate in the ladder",
42
- statement: "Fall-through is triggered by structure and infrastructure only; no builder-defined predicate or judge can reject a schema-valid output at run time — value accuracy exists offline, in bake-offs.",
43
- tracking: "customer request on file (2026-08-21); not scheduled",
44
- tripwire: { file: "src/types/route.ts", mustNotMatch: /qualityGate|"judged"/ },
41
+ capability: "Route-declared judge (hosted lane)",
42
+ statement: "The SDK-lane `qualityGate` predicate (it rejects a schema-valid output as `quality_rejected`) ships; a route-DECLARED judge that runs on the hosted lane — a `judge:` block in the bundle with a rubric — is not built, and the bake-off-scoped `bakeoff.judge` (offline scoring) (which only measures value accuracy during a bake-off) is a separate SDK feature that never serves the hosted lane.",
43
+ tracking: "Route-declared judge — customer request on file (2026-08-21); follows the SDK predicate",
44
+ // The SDK predicate flipped this entry: adding `qualityGate` to route.ts would
45
+ // fire the old /qualityGate|"judged"/ tripwire, so it watches for the
46
+ // HOSTED declared-judge shape — a `judge?: JudgePolicy` field or a judge block
47
+ // carrying a `rubric` (spec §3). RB-1 added `bakeoff.judge?: { model,
48
+ // maxSamples }` (no rubric, nested in BakeoffConfig) — deliberately NOT the
49
+ // hosted judge, so the regex requires the rubric/JudgePolicy shape to avoid a
50
+ // false positive. `judgedBy:` and prose mentions of "judge" still do not match.
51
+ tripwire: { file: "src/types/route.ts", mustNotMatch: /\bjudge\??\s*:\s*JudgePolicy\b|\brubric\s*:/ },
45
52
  },
46
53
  {
47
54
  id: "B2",
48
55
  status: "not_built",
49
56
  capability: "Structure on the raw lane",
50
- statement: '`runRaw` never validates or repairs; `responseFormat: "json_object"` shapes MIME, not schema, so `meta.validated` stays false (ruled schema-free in CE-2 F4b; re-requested #223 R2).',
57
+ statement: '`runRaw` never validates or repairs; `responseFormat: "json_object"` shapes MIME, not schema, so `meta.validated` stays false — structure on the raw lane is ruled out by design, and the path is to graduate that step to a route.',
51
58
  tracking: "ruled schema-free on the raw lane; re-request on file",
52
59
  tripwire: { file: "src/run/raw.ts", mustNotMatch: /readonly outputSchema\??:/ },
53
60
  },
package/dist/run/run.d.ts CHANGED
@@ -12,7 +12,7 @@ import type { RigManifest } from "../routes/manifest";
12
12
  import type { RunOptions, RunResult } from "../types/route";
13
13
  import type { TelemetryStore } from "../types/telemetry";
14
14
  import type { CaptureStore } from "../telemetry/capture";
15
- import type { CapabilityLookup } from "./resolve";
15
+ import type { CapabilityLookup, RunMode } from "./resolve";
16
16
  import type { SearchProvider } from "./grounding";
17
17
  import type { StreamDeltaSink } from "./dispatch";
18
18
  import type { TaskSettingsResolver } from "../settings/task-settings";
@@ -50,6 +50,12 @@ export interface RunDeps {
50
50
  * content before a capture row is written. null/omitted = local-only (the
51
51
  * YAML flag governs, no per-task scrub policy). */
52
52
  readonly settings?: TaskSettingsResolver | null;
53
+ /** Resolution mode (routing-console spec §4.4, R3) — `live` (default) on every
54
+ * production surface, `trial` ONLY on the bake-off replay path (`replayVariant`
55
+ * sets it). `trial` keeps declared `trial` candidates and permits selecting a
56
+ * trial variant; `live` drops trial candidates before minting and rejects a
57
+ * trial variant. Absent ⇒ live ⇒ byte-identical to the pre-R3 loop. */
58
+ readonly mode?: RunMode;
53
59
  }
54
60
  export declare function run(deps: RunDeps, routeName: string, opts: RunOptions): Promise<RunResult>;
55
61
  //# sourceMappingURL=run.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../src/run/run.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAMrD,OAAO,KAAK,EAAE,eAAe,EAAgB,MAAM,mBAAmB,CAAC;AAGvE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAEtD,OAAO,KAAK,EAAsC,UAAU,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAChG,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACzD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAQlD,OAAO,KAAK,EAAE,cAAc,EAAoC,MAAM,aAAa,CAAC;AAUpF,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAKlD,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,2BAA2B,CAAC;AAEtE,MAAM,WAAW,OAAO;IACtB,QAAQ,CAAC,MAAM,EAAE,YAAY,CAAC;IAC9B,QAAQ,CAAC,QAAQ,EAAE,WAAW,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;IACxD,QAAQ,CAAC,SAAS,EAAE,cAAc,CAAC;IACnC,QAAQ,CAAC,YAAY,EAAE,gBAAgB,CAAC;IACxC,+EAA+E;IAC/E,QAAQ,CAAC,iBAAiB,CAAC,EAAE,MAAM,CAAC;IACpC;;gDAE4C;IAC5C,QAAQ,CAAC,sBAAsB,CAAC,EAAE,MAAM,CAAC;IACzC,0EAA0E;IAC1E,QAAQ,CAAC,QAAQ,CAAC,EAAE,WAAW,GAAG,IAAI,CAAC;IACvC;2EACuE;IACvE,QAAQ,CAAC,QAAQ,CAAC,EAAE,YAAY,GAAG,IAAI,CAAC;IACxC;;0BAEsB;IACtB,QAAQ,CAAC,MAAM,CAAC,EAAE,cAAc,GAAG,IAAI,CAAC;IACxC,qDAAqD;IACrD,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAC/C;;;mFAG+E;IAC/E,QAAQ,CAAC,aAAa,CAAC,EAAE,eAAe,CAAC;IACzC;;;;;uDAKmD;IACnD,QAAQ,CAAC,QAAQ,CAAC,EAAE,oBAAoB,GAAG,IAAI,CAAC;CACjD;AAqDD,wBAAsB,GAAG,CAAC,IAAI,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC,CAmjBhG"}
1
+ {"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../src/run/run.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAMrD,OAAO,KAAK,EAAE,eAAe,EAAgB,MAAM,mBAAmB,CAAC;AAGvE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAEtD,OAAO,KAAK,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAC5D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACzD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,KAAK,EAAE,gBAAgB,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAS3D,OAAO,KAAK,EAAE,cAAc,EAAoC,MAAM,aAAa,CAAC;AAWpF,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAOlD,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,2BAA2B,CAAC;AAEtE,MAAM,WAAW,OAAO;IACtB,QAAQ,CAAC,MAAM,EAAE,YAAY,CAAC;IAC9B,QAAQ,CAAC,QAAQ,EAAE,WAAW,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;IACxD,QAAQ,CAAC,SAAS,EAAE,cAAc,CAAC;IACnC,QAAQ,CAAC,YAAY,EAAE,gBAAgB,CAAC;IACxC,+EAA+E;IAC/E,QAAQ,CAAC,iBAAiB,CAAC,EAAE,MAAM,CAAC;IACpC;;gDAE4C;IAC5C,QAAQ,CAAC,sBAAsB,CAAC,EAAE,MAAM,CAAC;IACzC,0EAA0E;IAC1E,QAAQ,CAAC,QAAQ,CAAC,EAAE,WAAW,GAAG,IAAI,CAAC;IACvC;2EACuE;IACvE,QAAQ,CAAC,QAAQ,CAAC,EAAE,YAAY,GAAG,IAAI,CAAC;IACxC;;0BAEsB;IACtB,QAAQ,CAAC,MAAM,CAAC,EAAE,cAAc,GAAG,IAAI,CAAC;IACxC,qDAAqD;IACrD,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAC/C;;;mFAG+E;IAC/E,QAAQ,CAAC,aAAa,CAAC,EAAE,eAAe,CAAC;IACzC;;;;;uDAKmD;IACnD,QAAQ,CAAC,QAAQ,CAAC,EAAE,oBAAoB,GAAG,IAAI,CAAC;IAChD;;;;2EAIuE;IACvE,QAAQ,CAAC,IAAI,CAAC,EAAE,OAAO,CAAC;CACzB;AAsBD,wBAAsB,GAAG,CAAC,IAAI,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC,CAqkBhG"}