modelrig 0.11.0 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +82 -0
- package/dist/adapters/anthropic.d.ts +4 -4
- package/dist/adapters/anthropic.d.ts.map +1 -1
- package/dist/adapters/anthropic.js +8 -1
- package/dist/adapters/deepseek.d.ts +2 -4
- package/dist/adapters/deepseek.d.ts.map +1 -1
- package/dist/adapters/deepseek.js +9 -5
- package/dist/adapters/gemini.d.ts +4 -1
- package/dist/adapters/gemini.d.ts.map +1 -1
- package/dist/adapters/gemini.js +16 -1
- package/dist/adapters/grok.d.ts +2 -4
- package/dist/adapters/grok.d.ts.map +1 -1
- package/dist/adapters/grok.js +8 -4
- package/dist/adapters/history-mapping.d.ts +126 -0
- package/dist/adapters/history-mapping.d.ts.map +1 -0
- package/dist/adapters/history-mapping.js +230 -0
- package/dist/adapters/openai-compat.d.ts +2 -4
- package/dist/adapters/openai-compat.d.ts.map +1 -1
- package/dist/adapters/openai-compat.js +9 -5
- package/dist/adapters/openai.d.ts +2 -4
- package/dist/adapters/openai.d.ts.map +1 -1
- package/dist/adapters/openai.js +8 -4
- package/dist/adapters/responses-api.d.ts.map +1 -1
- package/dist/adapters/responses-api.js +9 -4
- package/dist/artifacts/artifact.d.ts.map +1 -1
- package/dist/artifacts/artifact.js +79 -0
- package/dist/artifacts/conversation.d.ts +128 -0
- package/dist/artifacts/conversation.d.ts.map +1 -0
- package/dist/artifacts/conversation.js +172 -0
- package/dist/artifacts/index.d.ts +1 -1
- package/dist/artifacts/index.d.ts.map +1 -1
- package/dist/artifacts/media.d.ts +46 -0
- package/dist/artifacts/media.d.ts.map +1 -0
- package/dist/artifacts/media.js +88 -0
- package/dist/artifacts/run-context.d.ts +61 -2
- package/dist/artifacts/run-context.d.ts.map +1 -1
- package/dist/artifacts/run-context.js +183 -12
- package/dist/artifacts/scrub.d.ts +3 -0
- package/dist/artifacts/scrub.d.ts.map +1 -1
- package/dist/bakeoff/engine.d.ts.map +1 -1
- package/dist/bakeoff/engine.js +34 -7
- package/dist/bakeoff/eval-cases.d.ts +5 -0
- package/dist/bakeoff/eval-cases.d.ts.map +1 -1
- package/dist/bakeoff/eval-cases.js +10 -1
- package/dist/bakeoff/replay.d.ts +15 -0
- package/dist/bakeoff/replay.d.ts.map +1 -1
- package/dist/bakeoff/replay.js +19 -1
- package/dist/bakeoff/report.d.ts.map +1 -1
- package/dist/bakeoff/report.js +9 -2
- package/dist/bakeoff/turn-replay.d.ts +93 -0
- package/dist/bakeoff/turn-replay.d.ts.map +1 -0
- package/dist/bakeoff/turn-replay.js +150 -0
- package/dist/bakeoff/types.d.ts +12 -0
- package/dist/bakeoff/types.d.ts.map +1 -1
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +1 -0
- package/dist/config.d.ts +18 -0
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +23 -3
- package/dist/doctor-preflight.d.ts.map +1 -1
- package/dist/doctor-preflight.js +6 -1
- package/dist/feedback/grade.d.ts +7 -2
- package/dist/feedback/grade.d.ts.map +1 -1
- package/dist/feedback/grade.js +14 -0
- package/dist/feedback/outcome.d.ts +103 -0
- package/dist/feedback/outcome.d.ts.map +1 -0
- package/dist/feedback/outcome.js +175 -0
- package/dist/gateway/history-bridge.d.ts +32 -0
- package/dist/gateway/history-bridge.d.ts.map +1 -0
- package/dist/gateway/history-bridge.js +64 -0
- package/dist/gateway/serve-client.d.ts.map +1 -1
- package/dist/gateway/serve-client.js +6 -0
- package/dist/index.d.ts +44 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +110 -9
- package/dist/init/init-cli.d.ts.map +1 -1
- package/dist/init/init-cli.js +21 -4
- package/dist/init/observe-cli.js +1 -1
- package/dist/init/observe.d.ts +7 -1
- package/dist/init/observe.d.ts.map +1 -1
- package/dist/init/observe.js +74 -22
- package/dist/init/starter.d.ts +36 -0
- package/dist/init/starter.d.ts.map +1 -0
- package/dist/init/starter.js +85 -0
- package/dist/registry/build.d.ts +4 -0
- package/dist/registry/build.d.ts.map +1 -1
- package/dist/registry/generation-models.d.ts +68 -0
- package/dist/registry/generation-models.d.ts.map +1 -0
- package/dist/registry/generation-models.js +168 -0
- package/dist/registry/model-kind.d.ts +34 -0
- package/dist/registry/model-kind.d.ts.map +1 -0
- package/dist/registry/model-kind.js +40 -0
- package/dist/registry/pricing-shape.d.ts +29 -1
- package/dist/registry/pricing-shape.d.ts.map +1 -1
- package/dist/registry/pricing-shape.js +9 -0
- package/dist/registry/pricing.d.ts +26 -0
- package/dist/registry/pricing.d.ts.map +1 -1
- package/dist/registry/pricing.js +29 -0
- package/dist/registry/rows.d.ts.map +1 -1
- package/dist/registry/rows.js +3 -1
- package/dist/registry/unit-pricing.d.ts +32 -0
- package/dist/registry/unit-pricing.d.ts.map +1 -0
- package/dist/registry/unit-pricing.js +62 -0
- package/dist/routes/input-loader.d.ts +38 -0
- package/dist/routes/input-loader.d.ts.map +1 -0
- package/dist/routes/input-loader.js +80 -0
- package/dist/routes/loader.d.ts.map +1 -1
- package/dist/routes/loader.js +3 -0
- package/dist/routes/resolved-bundle.d.ts +2 -0
- package/dist/routes/resolved-bundle.d.ts.map +1 -1
- package/dist/routes/resolved-bundle.js +3 -0
- package/dist/run/cost.d.ts +28 -0
- package/dist/run/cost.d.ts.map +1 -1
- package/dist/run/cost.js +74 -0
- package/dist/run/dispatch.d.ts +4 -1
- package/dist/run/dispatch.d.ts.map +1 -1
- package/dist/run/dispatch.js +3 -0
- package/dist/run/limits.d.ts +88 -0
- package/dist/run/limits.d.ts.map +1 -0
- package/dist/run/limits.js +155 -0
- package/dist/run/messages.d.ts +199 -0
- package/dist/run/messages.d.ts.map +1 -0
- package/dist/run/messages.js +430 -0
- package/dist/run/raw-finalize.d.ts +5 -2
- package/dist/run/raw-finalize.d.ts.map +1 -1
- package/dist/run/raw-finalize.js +8 -5
- package/dist/run/raw-request.d.ts.map +1 -1
- package/dist/run/raw-request.js +2 -1
- package/dist/run/raw.d.ts +10 -4
- package/dist/run/raw.d.ts.map +1 -1
- package/dist/run/raw.js +11 -3
- package/dist/run/rawonly.d.ts +29 -5
- package/dist/run/rawonly.d.ts.map +1 -1
- package/dist/run/rawonly.js +54 -19
- package/dist/run/record-attempt.d.ts.map +1 -1
- package/dist/run/record-attempt.js +16 -6
- package/dist/run/record-call.d.ts +13 -2
- package/dist/run/record-call.d.ts.map +1 -1
- package/dist/run/record-call.js +50 -7
- package/dist/run/run.d.ts.map +1 -1
- package/dist/run/run.js +12 -8
- package/dist/telemetry/attempt-meta.d.ts +49 -1
- package/dist/telemetry/attempt-meta.d.ts.map +1 -1
- package/dist/telemetry/attempt-meta.js +45 -1
- package/dist/telemetry/exporter.d.ts.map +1 -1
- package/dist/telemetry/exporter.js +2 -1
- package/dist/telemetry/grade-mappers.d.ts +1 -1
- package/dist/telemetry/grade-mappers.d.ts.map +1 -1
- package/dist/telemetry/grade-mappers.js +7 -1
- package/dist/telemetry/grade-task.d.ts +49 -0
- package/dist/telemetry/grade-task.d.ts.map +1 -0
- package/dist/telemetry/grade-task.js +85 -0
- package/dist/telemetry/grades-sql.d.ts +16 -1
- package/dist/telemetry/grades-sql.d.ts.map +1 -1
- package/dist/telemetry/grades-sql.js +64 -4
- package/dist/telemetry/ingest-protocol.d.ts +24 -1
- package/dist/telemetry/ingest-protocol.d.ts.map +1 -1
- package/dist/telemetry/ingest-protocol.js +72 -43
- package/dist/telemetry/otel-content.d.ts +31 -0
- package/dist/telemetry/otel-content.d.ts.map +1 -0
- package/dist/telemetry/otel-content.js +106 -0
- package/dist/telemetry/otel-hierarchy.d.ts +57 -0
- package/dist/telemetry/otel-hierarchy.d.ts.map +1 -0
- package/dist/telemetry/otel-hierarchy.js +155 -0
- package/dist/telemetry/otel-normalize.d.ts +15 -1
- package/dist/telemetry/otel-normalize.d.ts.map +1 -1
- package/dist/telemetry/otel-normalize.js +24 -39
- package/dist/telemetry/sinks.d.ts +4 -1
- package/dist/telemetry/sinks.d.ts.map +1 -1
- package/dist/telemetry/sinks.js +53 -34
- package/dist/telemetry/store-migrations.d.ts.map +1 -1
- package/dist/telemetry/store-migrations.js +8 -0
- package/dist/telemetry/store.d.ts +9 -0
- package/dist/telemetry/store.d.ts.map +1 -1
- package/dist/telemetry/store.js +10 -0
- package/dist/telemetry/uniform-rows.d.ts +43 -0
- package/dist/telemetry/uniform-rows.d.ts.map +1 -0
- package/dist/telemetry/uniform-rows.js +139 -0
- package/dist/types/failures-teaching.d.ts.map +1 -1
- package/dist/types/failures-teaching.js +11 -0
- package/dist/types/failures.d.ts +1 -1
- package/dist/types/failures.d.ts.map +1 -1
- package/dist/types/failures.js +2 -0
- package/dist/types/provider.d.ts +12 -0
- package/dist/types/provider.d.ts.map +1 -1
- package/dist/types/route.d.ts +13 -0
- package/dist/types/route.d.ts.map +1 -1
- package/dist/types/telemetry.d.ts +14 -1
- package/dist/types/telemetry.d.ts.map +1 -1
- package/docs/agent-skill/SKILL.md +122 -12
- package/docs/how-it-fits.md +4 -4
- package/docs/instrumentation.md +148 -44
- package/docs/llms.txt +143 -13
- package/docs/mcp-oracle.md +15 -4
- package/docs/quickstart.md +129 -16
- package/package.json +4 -2
- package/registry/registry.json +456 -0
- package/templates/creative-approval.json +81 -0
- package/templates/support-conversation.json +82 -0
|
@@ -21,9 +21,12 @@
|
|
|
21
21
|
* v1 wire, zero new rows, zero behaviour change (spec §1.4).
|
|
22
22
|
*/
|
|
23
23
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
24
|
+
exports.OPEN_CONVERSATION_IDLE_MS = exports.MAX_OPEN_CONVERSATIONS = void 0;
|
|
24
25
|
exports.resetRunMetaWarningsForTest = resetRunMetaWarningsForTest;
|
|
25
26
|
exports.withRunIdTag = withRunIdTag;
|
|
26
27
|
exports.getActiveRunContext = getActiveRunContext;
|
|
28
|
+
exports.runLimitRefusal = runLimitRefusal;
|
|
29
|
+
exports.enforceRunLimits = enforceRunLimits;
|
|
27
30
|
exports.isValidStepKey = isValidStepKey;
|
|
28
31
|
exports.resolveStepKey = resolveStepKey;
|
|
29
32
|
exports.resetStepKeyWarningsForTest = resetStepKeyWarningsForTest;
|
|
@@ -33,6 +36,9 @@ const node_crypto_1 = require("node:crypto");
|
|
|
33
36
|
const otel_map_1 = require("../telemetry/otel-map");
|
|
34
37
|
const attempt_meta_1 = require("../telemetry/attempt-meta");
|
|
35
38
|
const attempt_meta_2 = require("../telemetry/attempt-meta");
|
|
39
|
+
const conversation_1 = require("./conversation");
|
|
40
|
+
const limits_1 = require("../run/limits");
|
|
41
|
+
const failures_1 = require("../types/failures");
|
|
36
42
|
/** Once-per-process latch for the run-envelope sanitize warning (Ruling A,
|
|
37
43
|
* PR #262). Mirrors {@link warnedStepKeys}: a dropped field is named ONCE, so a
|
|
38
44
|
* mislabelled `actor` teaches without spamming the run loop. */
|
|
@@ -82,9 +88,13 @@ const storage = new node_async_hooks_1.AsyncLocalStorage();
|
|
|
82
88
|
* (what the `/projects` tree joins on) or, without one, the run's id. Never
|
|
83
89
|
* overrides a caller-supplied `run_id`; no context ⇒ tags unchanged. */
|
|
84
90
|
function withRunIdTag(tags, ctx) {
|
|
85
|
-
if (ctx === null
|
|
91
|
+
if (ctx === null)
|
|
86
92
|
return tags;
|
|
87
|
-
|
|
93
|
+
// ARC-1: a conversation's user/subject tags sit UNDER the caller's (caller wins).
|
|
94
|
+
const withDefaults = ctx.defaultTags !== undefined ? { ...ctx.defaultTags, ...tags } : tags;
|
|
95
|
+
if (typeof tags.run_id === "string" && tags.run_id !== "")
|
|
96
|
+
return withDefaults;
|
|
97
|
+
return { ...withDefaults, run_id: ctx.episodeKey ?? ctx.runId };
|
|
88
98
|
}
|
|
89
99
|
/** The active run context, or null when none is set (namespace disabled, or no
|
|
90
100
|
* run.start on this async path). Read by the run loop's auto-step seam. */
|
|
@@ -92,6 +102,46 @@ function getActiveRunContext() {
|
|
|
92
102
|
const ctx = storage.getStore();
|
|
93
103
|
return ctx !== undefined && !ctx.ended ? ctx : null;
|
|
94
104
|
}
|
|
105
|
+
/** ARC-5: the limits recorded on a stored run's RunMeta (snake → camel), or null. */
|
|
106
|
+
function limitsFromMeta(meta) {
|
|
107
|
+
const l = meta?.limits;
|
|
108
|
+
if (l === undefined)
|
|
109
|
+
return null;
|
|
110
|
+
return (0, limits_1.validateRunLimits)({
|
|
111
|
+
...(l.max_usd !== undefined ? { maxUsd: l.max_usd } : {}),
|
|
112
|
+
...(l.max_calls !== undefined ? { maxCalls: l.max_calls } : {}),
|
|
113
|
+
...(l.max_wall_ms !== undefined ? { maxWallMs: l.max_wall_ms } : {}),
|
|
114
|
+
...(l.max_repeat_tool_call !== undefined ? { maxRepeatToolCall: l.max_repeat_tool_call } : {}),
|
|
115
|
+
});
|
|
116
|
+
}
|
|
117
|
+
/**
|
|
118
|
+
* ARC-5 pre-dispatch check — the ONE call `run()` and `runRaw()` make before a
|
|
119
|
+
* provider dispatch. Reads the ambient run (INCLUDING one already stopped at a
|
|
120
|
+
* limit, so it keeps refusing on that path). No run, or a run with no limits ⇒
|
|
121
|
+
* null, zero work. On a fresh breach it ends the run `failed` with
|
|
122
|
+
* `failure_reason: limit:<which>` and returns the typed error to throw.
|
|
123
|
+
* `ctx` overrides the ambient lookup (the raw stream captures it at entry).
|
|
124
|
+
*/
|
|
125
|
+
function runLimitRefusal(ctx) {
|
|
126
|
+
const state = (ctx === undefined ? storage.getStore() : (ctx ?? undefined));
|
|
127
|
+
if (state === undefined || state.limits === null || state.limits === undefined)
|
|
128
|
+
return null;
|
|
129
|
+
if (state.limitStop !== undefined)
|
|
130
|
+
return new limits_1.RunLimitExceeded(state.limitStop, state.runId);
|
|
131
|
+
if (state.ended)
|
|
132
|
+
return null;
|
|
133
|
+
const breach = (0, limits_1.checkRunLimits)(state.limitState, state.limits, Date.now());
|
|
134
|
+
if (breach === null)
|
|
135
|
+
return null;
|
|
136
|
+
state.stopForLimit(breach);
|
|
137
|
+
return new limits_1.RunLimitExceeded(breach, state.runId);
|
|
138
|
+
}
|
|
139
|
+
/** Throwing form of {@link runLimitRefusal} — `run()` / `runRaw()` call this. */
|
|
140
|
+
function enforceRunLimits(ctx) {
|
|
141
|
+
const refusal = runLimitRefusal(ctx);
|
|
142
|
+
if (refusal !== null)
|
|
143
|
+
throw refusal;
|
|
144
|
+
}
|
|
95
145
|
/** Valid ground-truth step key (G1 spec §3): a caller-supplied `stepKey` (either
|
|
96
146
|
* lane) or a `tags.step` used as one. Auto-minted keys (`label#seq`) may contain
|
|
97
147
|
* a `/` from a route slug or `provider/model` label and are NOT validated — the
|
|
@@ -128,10 +178,19 @@ function warnOnce(scope, err) {
|
|
|
128
178
|
console.warn(`[modelrig artifacts] ${scope} buffer write failed (dropped, run unaffected): ` +
|
|
129
179
|
`${err instanceof Error ? err.message : String(err)}`);
|
|
130
180
|
}
|
|
181
|
+
/** ARC-1 (fix round 1, M4): the in-process open-conversation registry's bounds.
|
|
182
|
+
* An entry idle longer than the TTL (the server's default idle-close window) or
|
|
183
|
+
* past the LRU cap is EVICTED — not ended: the run stays `running` in the local
|
|
184
|
+
* store, so a later `rig.conversation(id)` re-enters it through `findOpenRun`. */
|
|
185
|
+
exports.MAX_OPEN_CONVERSATIONS = 10000;
|
|
186
|
+
exports.OPEN_CONVERSATION_IDLE_MS = 30 * 60000;
|
|
131
187
|
function createRunNamespace(deps) {
|
|
132
188
|
const { buffer, enabled } = deps;
|
|
189
|
+
const maxOpen = deps.maxOpenConversations ?? exports.MAX_OPEN_CONVERSATIONS;
|
|
190
|
+
const idleMs = deps.conversationIdleMs ?? exports.OPEN_CONVERSATION_IDLE_MS;
|
|
191
|
+
const clock = deps.now ?? Date.now;
|
|
133
192
|
const inertHandle = { runId: "", end() { } };
|
|
134
|
-
function buildContext(startRow, traceId, zeroRetention) {
|
|
193
|
+
function buildContext(startRow, traceId, zeroRetention, conversation, limits = null) {
|
|
135
194
|
const runId = startRow.id;
|
|
136
195
|
const episodeKey = startRow.episodeKey;
|
|
137
196
|
const attemptByStep = new Map();
|
|
@@ -140,7 +199,7 @@ function createRunNamespace(deps) {
|
|
|
140
199
|
// The ambient step key (G1): the last step minted or emitted in this run,
|
|
141
200
|
// which an artifact.save without an explicit stepKey attaches to.
|
|
142
201
|
let lastStepKey = null;
|
|
143
|
-
|
|
202
|
+
const state = {
|
|
144
203
|
runId,
|
|
145
204
|
episodeKey,
|
|
146
205
|
traceId,
|
|
@@ -148,6 +207,15 @@ function createRunNamespace(deps) {
|
|
|
148
207
|
ended: false,
|
|
149
208
|
buffer,
|
|
150
209
|
startRow,
|
|
210
|
+
limits,
|
|
211
|
+
limitState: (0, limits_1.initialLimitState)(Date.parse(startRow.startedAt) || Date.now()),
|
|
212
|
+
stopForLimit(breach) {
|
|
213
|
+
state.limitStop = breach;
|
|
214
|
+
endContext(state, "failed", { failureReason: (0, limits_1.limitFailureReason)(breach.which) });
|
|
215
|
+
},
|
|
216
|
+
...(conversation !== undefined
|
|
217
|
+
? { turns: conversation.turns, conversationKey: conversation.key, ...(Object.keys(conversation.tags).length > 0 ? { defaultTags: conversation.tags } : {}) }
|
|
218
|
+
: {}),
|
|
151
219
|
noteLoad(artifactId) {
|
|
152
220
|
loaded.add(artifactId);
|
|
153
221
|
},
|
|
@@ -157,6 +225,12 @@ function createRunNamespace(deps) {
|
|
|
157
225
|
return ids;
|
|
158
226
|
},
|
|
159
227
|
nextStepKey(label) {
|
|
228
|
+
// ARC-1: inside a conversation the auto key is the turn key `turn:<n>.<k>`
|
|
229
|
+
// (an explicit caller stepKey never reaches here — resolveStepKey).
|
|
230
|
+
if (conversation !== undefined) {
|
|
231
|
+
lastStepKey = conversation.turns.mint();
|
|
232
|
+
return lastStepKey;
|
|
233
|
+
}
|
|
160
234
|
stepSeq += 1;
|
|
161
235
|
lastStepKey = `${label}#${stepSeq}`;
|
|
162
236
|
return lastStepKey;
|
|
@@ -165,9 +239,20 @@ function createRunNamespace(deps) {
|
|
|
165
239
|
return lastStepKey;
|
|
166
240
|
},
|
|
167
241
|
emitGroundTruthStep(input) {
|
|
242
|
+
// M4: activity keeps a conversation fresh in the in-process registry.
|
|
243
|
+
if (state.conversationKey !== undefined && openConversations.has(state.conversationKey))
|
|
244
|
+
touch(state.conversationKey, state);
|
|
168
245
|
// A caller-supplied key on the raw lane never minted through nextStepKey;
|
|
169
246
|
// record it as the ambient step so artifacts saved around this call join it.
|
|
170
247
|
lastStepKey = input.stepKey;
|
|
248
|
+
// ARC-5: fold this attempt into the limit tally (pure; no-op without limits).
|
|
249
|
+
if (state.limits !== null) {
|
|
250
|
+
state.limitState = (0, limits_1.accumulate)(state.limitState, {
|
|
251
|
+
costUsd: input.costUsd ?? 0,
|
|
252
|
+
notDispatched: input.failureClass !== null && failures_1.TERMINAL_CLASSES.has(input.failureClass),
|
|
253
|
+
...(input.toolCalls !== undefined ? { toolCalls: input.toolCalls } : {}),
|
|
254
|
+
});
|
|
255
|
+
}
|
|
171
256
|
const attemptNo = (attemptByStep.get(input.stepKey) ?? 0) + 1;
|
|
172
257
|
attemptByStep.set(input.stepKey, attemptNo);
|
|
173
258
|
try {
|
|
@@ -190,10 +275,67 @@ function createRunNamespace(deps) {
|
|
|
190
275
|
}
|
|
191
276
|
},
|
|
192
277
|
};
|
|
278
|
+
return state;
|
|
279
|
+
}
|
|
280
|
+
function seededCounter(stepKeys) {
|
|
281
|
+
const { turn, call } = (0, conversation_1.turnStateOf)(stepKeys);
|
|
282
|
+
return new conversation_1.TurnCounter(turn, call);
|
|
283
|
+
}
|
|
284
|
+
/** ARC-1: the open conversations of THIS namespace, by `agent \0 id`, in LRU
|
|
285
|
+
* order (a touch re-inserts), each with its last-touch time (M4). */
|
|
286
|
+
const openConversations = new Map();
|
|
287
|
+
function touch(key, ctx) {
|
|
288
|
+
openConversations.delete(key);
|
|
289
|
+
openConversations.set(key, { ctx, touchedAt: clock() });
|
|
290
|
+
}
|
|
291
|
+
/** Evict idle-past-TTL entries and anything past the cap (oldest first). */
|
|
292
|
+
function evict() {
|
|
293
|
+
const t = clock();
|
|
294
|
+
for (const [key, entry] of openConversations) {
|
|
295
|
+
if (openConversations.size > maxOpen || t - entry.touchedAt >= idleMs || entry.ctx.ended)
|
|
296
|
+
openConversations.delete(key);
|
|
297
|
+
else
|
|
298
|
+
break; // LRU order: the rest are newer
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
/** Test seam: the registry's live size. */
|
|
302
|
+
function openConversationCount() {
|
|
303
|
+
return openConversations.size;
|
|
304
|
+
}
|
|
305
|
+
/** Find (in-process, then local store) or start a conversation's run. */
|
|
306
|
+
function openOrResume(id, opts) {
|
|
307
|
+
const key = `${opts.agent}\u0000${id}`;
|
|
308
|
+
evict();
|
|
309
|
+
const live = openConversations.get(key)?.ctx;
|
|
310
|
+
if (live !== undefined && !live.ended) {
|
|
311
|
+
touch(key, live);
|
|
312
|
+
return { ctx: live, resumed: true };
|
|
313
|
+
}
|
|
314
|
+
const tags = (0, conversation_1.conversationTags)(opts);
|
|
315
|
+
let stored = null;
|
|
316
|
+
try {
|
|
317
|
+
stored = buffer.findOpenRun?.(opts.agent, id) ?? null;
|
|
318
|
+
}
|
|
319
|
+
catch (err) {
|
|
320
|
+
warnOnce("conversation lookup", err); // fail-open: start a fresh run
|
|
321
|
+
}
|
|
322
|
+
const ctx = stored !== null
|
|
323
|
+
? buildContext(stored.row, null, false, { turns: seededCounter(stored.stepKeys), key, tags }, limitsFromMeta(stored.row.meta) ?? (0, limits_1.validateRunLimits)(opts.limits))
|
|
324
|
+
: newRun({
|
|
325
|
+
pipeline: opts.agent,
|
|
326
|
+
episodeKey: id,
|
|
327
|
+
...(opts.environment !== undefined ? { environment: opts.environment } : {}),
|
|
328
|
+
meta: { v: 1, trigger: "conversation" },
|
|
329
|
+
...(opts.limits !== undefined ? { limits: opts.limits } : {}),
|
|
330
|
+
}, { turns: new conversation_1.TurnCounter(0), key, tags });
|
|
331
|
+
touch(key, ctx);
|
|
332
|
+
evict();
|
|
333
|
+
return { ctx, resumed: stored !== null };
|
|
193
334
|
}
|
|
194
335
|
/** Build the context + record the `running` row. Shared by start() and
|
|
195
336
|
* scope() so the two entry points agree on identity + buffering. */
|
|
196
|
-
function newRun(opts) {
|
|
337
|
+
function newRun(opts, conversation) {
|
|
338
|
+
const limits = (0, limits_1.validateRunLimits)(opts.limits);
|
|
197
339
|
const startRow = {
|
|
198
340
|
id: (0, node_crypto_1.randomUUID)(),
|
|
199
341
|
pipeline: opts.pipeline,
|
|
@@ -205,12 +347,12 @@ function createRunNamespace(deps) {
|
|
|
205
347
|
status: "running",
|
|
206
348
|
// WI-1 (spec §3): the run envelope, git-sha auto-filled. endContext spreads
|
|
207
349
|
// ...ctx.startRow, so the terminal projection re-exports the same meta.
|
|
208
|
-
meta: buildRunMeta(opts.meta),
|
|
350
|
+
meta: buildRunMeta(limits !== null ? { v: 1, ...(opts.meta ?? {}), limits: (0, limits_1.limitsForMeta)(limits) } : opts.meta),
|
|
209
351
|
};
|
|
210
352
|
const traceId = opts.traceparent !== undefined
|
|
211
353
|
? ((0, otel_map_1.parseTraceparent)(opts.traceparent)?.traceId ?? null)
|
|
212
354
|
: null;
|
|
213
|
-
const ctx = buildContext(startRow, traceId, opts.zeroRetention === true);
|
|
355
|
+
const ctx = buildContext(startRow, traceId, opts.zeroRetention === true, conversation, limits);
|
|
214
356
|
try {
|
|
215
357
|
buffer.recordRun(startRow);
|
|
216
358
|
}
|
|
@@ -227,13 +369,20 @@ function createRunNamespace(deps) {
|
|
|
227
369
|
storage.enterWith(ctx);
|
|
228
370
|
return {
|
|
229
371
|
runId: ctx.runId,
|
|
230
|
-
end: (status) => endContext(ctx, status),
|
|
372
|
+
end: (status, opts) => endContext(ctx, status, opts),
|
|
231
373
|
};
|
|
232
374
|
},
|
|
233
|
-
end(status) {
|
|
375
|
+
end(status, opts) {
|
|
234
376
|
const ctx = storage.getStore();
|
|
235
377
|
if (ctx !== undefined)
|
|
236
|
-
endContext(ctx, status);
|
|
378
|
+
endContext(ctx, status, opts);
|
|
379
|
+
},
|
|
380
|
+
setFailureReason(reason) {
|
|
381
|
+
if (!enabled)
|
|
382
|
+
return;
|
|
383
|
+
const ctx = storage.getStore();
|
|
384
|
+
if (ctx !== undefined && !ctx.ended)
|
|
385
|
+
ctx.failureReason = reason;
|
|
237
386
|
},
|
|
238
387
|
async scope(opts, fn) {
|
|
239
388
|
if (!enabled)
|
|
@@ -254,11 +403,32 @@ function createRunNamespace(deps) {
|
|
|
254
403
|
}
|
|
255
404
|
});
|
|
256
405
|
},
|
|
406
|
+
openConversationCount,
|
|
407
|
+
conversation(id, opts) {
|
|
408
|
+
return (0, conversation_1.createConversation)({
|
|
409
|
+
enabled,
|
|
410
|
+
openOrResume,
|
|
411
|
+
runIdOf: (ctx) => ctx.runId,
|
|
412
|
+
counterOf: (ctx) => ctx.turns,
|
|
413
|
+
enter: (ctx) => storage.enterWith(ctx),
|
|
414
|
+
runIn: (ctx, fn) => storage.run(ctx, fn),
|
|
415
|
+
end: (ctx, status, endOpts) => endContext(ctx, status, endOpts),
|
|
416
|
+
}, id, opts);
|
|
417
|
+
},
|
|
257
418
|
};
|
|
258
|
-
function endContext(ctx, status) {
|
|
419
|
+
function endContext(ctx, status, opts) {
|
|
259
420
|
if (ctx.ended)
|
|
260
421
|
return;
|
|
261
422
|
ctx.ended = true;
|
|
423
|
+
if (ctx.conversationKey !== undefined)
|
|
424
|
+
openConversations.delete(ctx.conversationKey);
|
|
425
|
+
const terminal = status ?? "succeeded";
|
|
426
|
+
const reason = opts?.failureReason ?? ctx.failureReason;
|
|
427
|
+
// R9: a failed run carries its reason in runs.meta (sanitized like every
|
|
428
|
+
// envelope field — a free-text reason is dropped, never smuggled).
|
|
429
|
+
const meta = terminal === "failed" && reason !== undefined
|
|
430
|
+
? buildRunMeta({ ...(ctx.startRow.meta ?? { v: 1 }), failure_reason: reason })
|
|
431
|
+
: ctx.startRow.meta;
|
|
262
432
|
// Update the projection in place: same identity as the start row, now with
|
|
263
433
|
// an ended_at and a terminal status. recordRun bumps export_seq so this
|
|
264
434
|
// terminal state re-exports past the `:runs` watermark. Emitting the full
|
|
@@ -267,8 +437,9 @@ function createRunNamespace(deps) {
|
|
|
267
437
|
try {
|
|
268
438
|
buffer.recordRun({
|
|
269
439
|
...ctx.startRow,
|
|
440
|
+
meta,
|
|
270
441
|
endedAt: new Date().toISOString(),
|
|
271
|
-
status:
|
|
442
|
+
status: terminal,
|
|
272
443
|
});
|
|
273
444
|
}
|
|
274
445
|
catch (err) {
|
|
@@ -55,6 +55,9 @@ export interface ScrubSummary {
|
|
|
55
55
|
readonly mode: ScrubMode;
|
|
56
56
|
readonly blocked: boolean;
|
|
57
57
|
readonly findings: readonly ScrubFinding[];
|
|
58
|
+
/** ARC-4 (UC-3 M2 posture a): set to `"media"` on an image/video/audio
|
|
59
|
+
* artifact — bytes the text engine never scanned. Absent on text artifacts. */
|
|
60
|
+
readonly unscreened?: "media";
|
|
58
61
|
}
|
|
59
62
|
/** Build the persisted summary from a scrub result. The only producer of the
|
|
60
63
|
* `artifacts.scrub` value, so the counts-not-values invariant lives in one
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"scrub.d.ts","sourceRoot":"","sources":["../../src/artifacts/scrub.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AAEH,OAAO,KAAK,EAAkB,aAAa,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAElF,MAAM,MAAM,SAAS,GAAG,KAAK,GAAG,QAAQ,GAAG,QAAQ,GAAG,OAAO,CAAC;AAE9D,iFAAiF;AACjF,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,IAAI,EAAE,iBAAiB,CAAC;IACjC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,MAAM,iBAAiB,GAAG,KAAK,GAAG,KAAK,GAAG,OAAO,GAAG,OAAO,GAAG,KAAK,CAAC;AAE1E,MAAM,WAAW,WAAW;IAC1B,8EAA8E;IAC9E,QAAQ,CAAC,QAAQ,EAAE,SAAS,YAAY,EAAE,CAAC;IAC3C;iEAC6D;IAC7D,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,oEAAoE;IACpE,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;CAC3B;AAED;;;0EAG0E;AAC1E,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC;IACzB,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,QAAQ,EAAE,SAAS,YAAY,EAAE,CAAC;
|
|
1
|
+
{"version":3,"file":"scrub.d.ts","sourceRoot":"","sources":["../../src/artifacts/scrub.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AAEH,OAAO,KAAK,EAAkB,aAAa,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAElF,MAAM,MAAM,SAAS,GAAG,KAAK,GAAG,QAAQ,GAAG,QAAQ,GAAG,OAAO,CAAC;AAE9D,iFAAiF;AACjF,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,IAAI,EAAE,iBAAiB,CAAC;IACjC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,MAAM,iBAAiB,GAAG,KAAK,GAAG,KAAK,GAAG,OAAO,GAAG,OAAO,GAAG,KAAK,CAAC;AAE1E,MAAM,WAAW,WAAW;IAC1B,8EAA8E;IAC9E,QAAQ,CAAC,QAAQ,EAAE,SAAS,YAAY,EAAE,CAAC;IAC3C;iEAC6D;IAC7D,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,oEAAoE;IACpE,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;CAC3B;AAED;;;0EAG0E;AAC1E,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC;IACzB,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,QAAQ,EAAE,SAAS,YAAY,EAAE,CAAC;IAC3C;mFAC+E;IAC/E,QAAQ,CAAC,UAAU,CAAC,EAAE,OAAO,CAAC;CAC/B;AAED;;kFAEkF;AAClF,wBAAgB,cAAc,CAAC,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,WAAW,GAAG,YAAY,CAEjF;AAaD,8EAA8E;AAC9E,wBAAgB,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAejD;AA0CD;;;GAGG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,GAAG,WAAW,CA6CvE;AAqBD;;;;;GAKG;AACH,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,aAAa,EAAE,CAIzD;AAED;;;;;;;GAOG;AACH,wBAAgB,cAAc,IAAI,QAAQ,EAAE,CAS3C"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"engine.d.ts","sourceRoot":"","sources":["../../src/bakeoff/engine.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAGH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAG1C,OAAO,KAAK,EAAE,aAAa,EAAE,aAAa,EAAE,WAAW,EAAgB,MAAM,gBAAgB,CAAC;
|
|
1
|
+
{"version":3,"file":"engine.d.ts","sourceRoot":"","sources":["../../src/bakeoff/engine.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAGH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAG1C,OAAO,KAAK,EAAE,aAAa,EAAE,aAAa,EAAE,WAAW,EAAgB,MAAM,gBAAgB,CAAC;AAI9F,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAQ5C,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAEpD,OAAO,KAAK,EAAE,UAAU,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAC7D,OAAO,KAAK,EAAE,eAAe,EAAE,gBAAgB,EAAmC,cAAc,EAAE,MAAM,SAAS,CAAC;AAElH;;+BAE+B;AAC/B,eAAO,MAAM,aAAa,kBAAkB,CAAC;AAE7C;;;gFAGgF;AAChF,MAAM,MAAM,kBAAkB,GAAG,CAAC,KAAK,EAAE;IACvC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,IAAI,EAAE,SAAS,UAAU,EAAE,CAAC;CACtC,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;AAgBpB,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,qEAAqE;IACrE,QAAQ,CAAC,QAAQ,EAAE,SAAS,MAAM,EAAE,CAAC;IACrC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,2EAA2E;IAC3E,QAAQ,CAAC,qBAAqB,EAAE,MAAM,CAAC;IACvC;;wEAEoE;IACpE,QAAQ,CAAC,WAAW,CAAC,EAAE,eAAe,CAAC;IACvC;iEAC6D;IAC7D,QAAQ,CAAC,aAAa,CAAC,EAAE,aAAa,CAAC;IACvC;;wEAEoE;IACpE,QAAQ,CAAC,oBAAoB,CAAC,EAAE,CAAC,KAAK,EAAE,WAAW,KAAK,aAAa,GAAG,IAAI,CAAC;IAC7E;+BAC2B;IAC3B,QAAQ,CAAC,QAAQ,CAAC,EAAE,WAAW,CAAC,MAAM,EAAE;QAAE,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAA;KAAE,CAAC,CAAC;IACrE;;+EAE2E;IAC3E,QAAQ,CAAC,UAAU,CAAC,EAAE,UAAU,CAAC;IACjC;;;;;+EAK2E;IAC3E,QAAQ,CAAC,WAAW,CAAC,EAAE,WAAW,CAAC;CACpC;AAsBD,wBAAsB,UAAU,CAC9B,IAAI,EAAE,OAAO,EACb,QAAQ,EAAE,YAAY,EACtB,GAAG,EAAE,cAAc,EACnB,WAAW,CAAC,EAAE,kBAAkB,GAC/B,OAAO,CAAC,eAAe,CAAC,CA0L1B;AAuCD;iEACiE;AACjE,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,SAAS,CAAC,EAAE,gBAAgB,CAAC;IACtC,mEAAmE;IACnE,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;CACpC;AA8BD;;;;GAIG;AACH,wBAAgB,eAAe,CAC7B,SAAS,EAAE,SAAS,MAAM,EAAE,EAC5B,OAAO,EAAE,SAAS,cAAc,EAAE,EAClC,GAAG,CAAC,EAAE,eAAe,GACpB,eAAe,CAAC,UAAU,CAAC,CA8G7B;AAED,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,OAAO,CAAC,EAAE,OAAO,KAAK,CAAC;CACjC;AAED;;uDAEuD;AACvD,wBAAsB,aAAa,CACjC,QAAQ,EAAE,eAAe,EACzB,MAAM,EAAE,mBAAmB,GAC1B,OAAO,CAAC,IAAI,CAAC,CA6Bf;AAED;;;;4BAI4B;AAC5B,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,SAAS,EAAE,IAAI,GAAG,MAAM,GAAG,SAAS,CAAC;IAC9C,kFAAkF;IAClF,QAAQ,CAAC,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;CAC3C;AAED;;;;;;;GAOG;AACH,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,eAAe,GAAG,aAAa,GAAG,IAAI,CAkBnF;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,eAAe,CAAC,QAAQ,EAAE,eAAe,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CA2CpG"}
|
package/dist/bakeoff/engine.js
CHANGED
|
@@ -20,6 +20,7 @@ exports.deriveSwapDirection = deriveSwapDirection;
|
|
|
20
20
|
exports.bakeoffEvidence = bakeoffEvidence;
|
|
21
21
|
const node_crypto_1 = require("node:crypto");
|
|
22
22
|
const loader_1 = require("../routes/loader");
|
|
23
|
+
const turn_replay_1 = require("./turn-replay");
|
|
23
24
|
const replay_1 = require("./replay");
|
|
24
25
|
const stats_1 = require("./stats");
|
|
25
26
|
const judge_1 = require("./judge");
|
|
@@ -109,8 +110,10 @@ async function runBakeoff(deps, captures, req, samplesSink) {
|
|
|
109
110
|
...(req.qualityGate !== undefined ? { qualityGate: req.qualityGate } : {}),
|
|
110
111
|
});
|
|
111
112
|
replayN = Math.max(replayN, samples.length);
|
|
112
|
-
|
|
113
|
-
|
|
113
|
+
// ARC-β ruling M3(c): incomparable cases leave this arm's score + regret, and are reported.
|
|
114
|
+
const split = (0, turn_replay_1.splitIncomparable)(samples);
|
|
115
|
+
rawByArm.set(variant, split.comparable);
|
|
116
|
+
metrics.push(withFallback(withIncomparable((0, stats_1.aggregateVariantMetrics)(variant, split.comparable), split.incomparable), split.comparable, firstCandidateLabel(route, variant)));
|
|
114
117
|
}
|
|
115
118
|
// RB-1 reference (ceiling) arm — replay the reference model on the SAME set
|
|
116
119
|
// under its own envelope (it is dear by definition), then attach each requested
|
|
@@ -137,25 +140,26 @@ async function runBakeoff(deps, captures, req, samplesSink) {
|
|
|
137
140
|
...(req.qualityGate !== undefined ? { qualityGate: req.qualityGate } : {}),
|
|
138
141
|
});
|
|
139
142
|
const partial = refSamples.length < replayN || refSamples.some((s) => s.failureClass === "budget_exhausted");
|
|
140
|
-
|
|
143
|
+
const refSplit = (0, turn_replay_1.splitIncomparable)(refSamples); // ARC-β M3(c): the reference's incomparable cases pair with nothing
|
|
144
|
+
refSamplesForMining = refSplit.comparable;
|
|
141
145
|
for (let i = 0; i < metrics.length; i += 1) {
|
|
142
146
|
const raw = rawByArm.get(metrics[i].variant) ?? [];
|
|
143
147
|
// N5: pairwise standing vs the reference — deterministic where both sides
|
|
144
148
|
// carry value accuracy, else the opt-in capped judge (fresh per arm, so
|
|
145
149
|
// maxSamples is per-arm). A REPORTED signal beside regret.
|
|
146
150
|
const pwJudge = pairwiseJudgeCfg !== undefined ? (0, pairwise_1.makePairwiseJudge)(deps, route, pairwiseJudgeCfg) : null;
|
|
147
|
-
const { stats: pairwise, outcomes } = await (0, pairwise_1.scoreArmPairwise)(raw,
|
|
151
|
+
const { stats: pairwise, outcomes } = await (0, pairwise_1.scoreArmPairwise)(raw, refSplit.comparable, pwJudge);
|
|
148
152
|
for (const [captureId, outcome] of outcomes)
|
|
149
153
|
pairwiseOutcomes.set((0, disagreement_1.pairwiseKey)(metrics[i].variant, captureId), outcome);
|
|
150
154
|
metrics[i] = {
|
|
151
155
|
...metrics[i],
|
|
152
|
-
regret: (0, stats_1.pairedRegret)(raw,
|
|
156
|
+
regret: (0, stats_1.pairedRegret)(raw, refSplit.comparable),
|
|
153
157
|
...(pairwise !== null ? { pairwise } : {}),
|
|
154
158
|
...(partial ? { referencePartial: true } : {}),
|
|
155
159
|
};
|
|
156
160
|
}
|
|
157
161
|
metrics.push({
|
|
158
|
-
...(0, stats_1.aggregateVariantMetrics)(exports.REFERENCE_ARM,
|
|
162
|
+
...withIncomparable((0, stats_1.aggregateVariantMetrics)(exports.REFERENCE_ARM, refSplit.comparable), refSplit.incomparable),
|
|
159
163
|
isReference: true,
|
|
160
164
|
regret: null,
|
|
161
165
|
...(partial ? { referencePartial: true } : {}),
|
|
@@ -191,7 +195,7 @@ async function runBakeoff(deps, captures, req, samplesSink) {
|
|
|
191
195
|
const keptArms = (0, samples_1.selectArmsToKeep)(req.keepOutputs, incumbentArm, proposal?.variant ?? null, req.variants);
|
|
192
196
|
const arms = keptArms.map((arm) => ({
|
|
193
197
|
arm,
|
|
194
|
-
armMeta: deriveArmMeta(route, arm),
|
|
198
|
+
armMeta: withFallbackArmMeta(deriveArmMeta(route, arm), metrics.find((m) => m.variant === arm)), // ARC-β R3-1
|
|
195
199
|
samples: (rawByArm.get(arm) ?? []).map((s) => ({
|
|
196
200
|
inputRef: s.captureId,
|
|
197
201
|
inputText: s.inputText ?? "",
|
|
@@ -514,3 +518,26 @@ function bakeoffEvidence(artifact, spentUsd) {
|
|
|
514
518
|
})),
|
|
515
519
|
};
|
|
516
520
|
}
|
|
521
|
+
/** ARC-β M3(c): attach the incomparable counts to an arm's metrics (absent when
|
|
522
|
+
* none — the artifact is then unchanged). */
|
|
523
|
+
function withIncomparable(m, rows) {
|
|
524
|
+
return rows.length > 0 ? { ...m, incomparable: rows } : m;
|
|
525
|
+
}
|
|
526
|
+
/** ARC-β R3-1: the arm's first candidate as `provider/model` (its nominal label). */
|
|
527
|
+
function firstCandidateLabel(route, arm) {
|
|
528
|
+
const variant = arm === "default" ? undefined : route.variants.find((v) => v.name === arm);
|
|
529
|
+
const c = variant?.candidates?.[0] ?? route.candidates[0];
|
|
530
|
+
return c !== undefined ? `${c.provider}/${c.model}` : "unknown";
|
|
531
|
+
}
|
|
532
|
+
/** ARC-β R3-1: when any of an arm's cases was answered by a fallback (its first
|
|
533
|
+
* candidate skipped for the history), attach the per-served-model counts and the
|
|
534
|
+
* honest arm label. Absent ⇒ the metrics (and the artifact) are unchanged. */
|
|
535
|
+
function withFallback(m, samples, first) {
|
|
536
|
+
const rows = (0, turn_replay_1.fallbackServed)(samples);
|
|
537
|
+
return rows.length > 0 ? { ...m, servedByFallback: rows, armLabel: (0, turn_replay_1.fallbackArmLabel)(first, rows, m.samples) } : m;
|
|
538
|
+
}
|
|
539
|
+
/** ARC-β R3-1: the reviewer's arm meta names the honest label, never the
|
|
540
|
+
* first candidate alone, when a fallback served any case. */
|
|
541
|
+
function withFallbackArmMeta(meta, m) {
|
|
542
|
+
return m?.armLabel !== undefined ? { ...meta, model: m.armLabel } : meta;
|
|
543
|
+
}
|
|
@@ -26,6 +26,7 @@ import type { KeepOutputsMode } from "./samples";
|
|
|
26
26
|
import type { BakeoffArtifact } from "./types";
|
|
27
27
|
import type { CandidateSpec, ObjectiveMode, RouteBundle } from "../types/route";
|
|
28
28
|
import type { QualityGate } from "../types/quality";
|
|
29
|
+
import type { TurnReplayCase } from "./turn-replay";
|
|
29
30
|
/** The narrow store surface the selection reads — kept minimal so this module
|
|
30
31
|
* depends on four reads, not three whole stores. */
|
|
31
32
|
export interface EvalCaseSelectionDeps {
|
|
@@ -44,6 +45,10 @@ export interface RehydratedCase {
|
|
|
44
45
|
* golden to grade against — it scores by conformance only). Carried so the
|
|
45
46
|
* runner can build the `expected` map from the golden's content. */
|
|
46
47
|
readonly correctedArtifactId: string | null;
|
|
48
|
+
/** ARC-β: present when the capture is a messages-lane turn — the history up to
|
|
49
|
+
* and including the turn's last user/tool message + the reference reply. The
|
|
50
|
+
* replay answers that final turn only (bakeoff/turn-replay.ts). */
|
|
51
|
+
readonly turn?: TurnReplayCase;
|
|
47
52
|
}
|
|
48
53
|
export interface EvalCaseRefusal {
|
|
49
54
|
readonly caseId: string;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"eval-cases.d.ts","sourceRoot":"","sources":["../../src/bakeoff/eval-cases.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,KAAK,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACrE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AACjE,OAAO,KAAK,EAAE,wBAAwB,EAAE,MAAM,oBAAoB,CAAC;AACnE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,EAAc,KAAK,kBAAkB,EAAE,MAAM,UAAU,CAAC;AAC/D,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAC5C,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AACjD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAC/C,OAAO,KAAK,EAAE,aAAa,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAChF,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAEpD;oDACoD;AACpD,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,mBAAmB,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,WAAW,EAAE,CAAC;IAC9D,QAAQ,CAAC,eAAe,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK;QAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAA;KAAE,GAAG,IAAI,CAAC;IAClG,QAAQ,CAAC,kBAAkB,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,KAAK,MAAM,GAAG,IAAI,CAAC;IAC/E,QAAQ,CAAC,uBAAuB,EAAE,CAAC,WAAW,EAAE,MAAM,KAAK,UAAU,GAAG,IAAI,CAAC;CAC9E;AAED,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,OAAO,EAAE,UAAU,CAAC;IAC7B;;wEAEoE;IACpE,QAAQ,CAAC,mBAAmB,EAAE,MAAM,GAAG,IAAI,CAAC;
|
|
1
|
+
{"version":3,"file":"eval-cases.d.ts","sourceRoot":"","sources":["../../src/bakeoff/eval-cases.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,KAAK,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACrE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AACjE,OAAO,KAAK,EAAE,wBAAwB,EAAE,MAAM,oBAAoB,CAAC;AACnE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,EAAc,KAAK,kBAAkB,EAAE,MAAM,UAAU,CAAC;AAC/D,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAC5C,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AACjD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAC/C,OAAO,KAAK,EAAE,aAAa,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAChF,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAEpD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAEpD;oDACoD;AACpD,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,mBAAmB,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,WAAW,EAAE,CAAC;IAC9D,QAAQ,CAAC,eAAe,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK;QAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAA;KAAE,GAAG,IAAI,CAAC;IAClG,QAAQ,CAAC,kBAAkB,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,KAAK,MAAM,GAAG,IAAI,CAAC;IAC/E,QAAQ,CAAC,uBAAuB,EAAE,CAAC,WAAW,EAAE,MAAM,KAAK,UAAU,GAAG,IAAI,CAAC;CAC9E;AAED,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,OAAO,EAAE,UAAU,CAAC;IAC7B;;wEAEoE;IACpE,QAAQ,CAAC,mBAAmB,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5C;;uEAEmE;IACnE,QAAQ,CAAC,IAAI,CAAC,EAAE,cAAc,CAAC;CAChC;AAED,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAClC,qEAAqE;IACrE,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,oEAAoE;IACpE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,YAAY,EAAE,SAAS,cAAc,EAAE,CAAC;IACjD,QAAQ,CAAC,QAAQ,EAAE,SAAS,eAAe,EAAE,CAAC;CAC/C;AAED;;;;GAIG;AACH,wBAAgB,sBAAsB,CAAC,IAAI,EAAE,qBAAqB,EAAE,IAAI,EAAE,MAAM,GAAG,iBAAiB,CAuCnG;AAED;;;;;;GAMG;AACH,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,SAAS,cAAc,EAAE,GAAG,YAAY,CAclF;AAED,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,SAAS,EAAE,iBAAiB,CAAC;IACtC;uFACmF;IACnF,QAAQ,CAAC,QAAQ,EAAE,eAAe,GAAG,IAAI,CAAC;CAC3C;AAED;;;;;;GAMG;AACH;;;0DAG0D;AAC1D,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,SAAS,EAAE,IAAI,CAAC,aAAa,EAAE,qBAAqB,GAAG,iBAAiB,CAAC,GAAG,IAAI,CAAC;IAC1F,QAAQ,CAAC,SAAS,EAAE,IAAI,CAAC,wBAAwB,EAAE,oBAAoB,CAAC,CAAC;IACzE,QAAQ,CAAC,QAAQ,EAAE,MAAM,IAAI,CAAC,YAAY,EAAE,kBAAkB,CAAC,CAAC;IAChE;;;;;6DAKyD;IACzD,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,mBAAmB,EAAE,MAAM,KAAK;QAAE,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAA;KAAE,GAAG,IAAI,CAAC;CACvF;AAED,wBAAsB,uBAAuB,CAC3C,IAAI,EAAE,OAAO,EACb,MAAM,EAAE,cAAc,EACtB,GAAG,EAAE;IACH,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,QAAQ,EAAE,SAAS,MAAM,EAAE,CAAC;IACrC,QAAQ,CAAC,qBAAqB,EAAE,MAAM,CAAC;IACvC,QAAQ,CAAC,WAAW,CAAC,EAAE,eAAe,CAAC;IACvC,QAAQ,CAAC,aAAa,CAAC,EAAE,aAAa,CAAC;IACvC,QAAQ,CAAC,oBAAoB,CAAC,EAAE,CAAC,KAAK,EAAE,WAAW,KAAK,aAAa,GAAG,IAAI,CAAC;IAC7E,QAAQ,CAAC,WAAW,CAAC,EAAE,WAAW,CAAC;IACnC;;;sCAGkC;IAClC,QAAQ,CAAC,UAAU,CAAC,EAAE,UAAU,CAAC;CAClC,EACD,WAAW,CAAC,EAAE,kBAAkB,GAC/B,OAAO,CAAC,qBAAqB,CAAC,CA+ChC"}
|
|
@@ -21,6 +21,7 @@ exports.selectEvalCaseCaptures = selectEvalCaseCaptures;
|
|
|
21
21
|
exports.evalCaseCaptureStore = evalCaseCaptureStore;
|
|
22
22
|
exports.runBakeoffFromEvalCases = runBakeoffFromEvalCases;
|
|
23
23
|
const engine_1 = require("./engine");
|
|
24
|
+
const turn_replay_1 = require("./turn-replay");
|
|
24
25
|
/**
|
|
25
26
|
* Partition a task's eval cases into rehydratable (with their captured input) and
|
|
26
27
|
* refused (named, with a reason). Pure over the injected reads — no network, no
|
|
@@ -53,7 +54,15 @@ function selectEvalCaseCaptures(deps, task) {
|
|
|
53
54
|
refuse("input not captured locally (route `capture` was off, or the capture was evicted)");
|
|
54
55
|
continue;
|
|
55
56
|
}
|
|
56
|
-
|
|
57
|
+
let turn; // ARC-β: messages captures carry their turn
|
|
58
|
+
try {
|
|
59
|
+
turn = (0, turn_replay_1.turnReplayCase)(capture);
|
|
60
|
+
}
|
|
61
|
+
catch {
|
|
62
|
+
refuse("captured conversation is not a valid messages-lane history (corrupt or hand-edited capture)");
|
|
63
|
+
continue;
|
|
64
|
+
}
|
|
65
|
+
rehydratable.push({ caseId: c.id, capture, correctedArtifactId: c.correctedArtifactId, ...(turn !== null ? { turn } : {}) });
|
|
57
66
|
}
|
|
58
67
|
return { task, total: cases.length, rehydratable, refusals };
|
|
59
68
|
}
|
package/dist/bakeoff/replay.d.ts
CHANGED
|
@@ -89,6 +89,21 @@ export interface ReplaySampleResult {
|
|
|
89
89
|
/** Addendum A3 (keepOutputs only): this arm's output for the review, or
|
|
90
90
|
* null when the arm produced none (a failure). */
|
|
91
91
|
readonly outputText?: string | null;
|
|
92
|
+
/** ARC-β ruling M3(c): the served candidate cannot accept this captured
|
|
93
|
+
* history natively (run/messages.ts `historyIncompatibility`). The sample is
|
|
94
|
+
* INCOMPARABLE for that model — excluded from its score and regret, counted
|
|
95
|
+
* and reported, never scored as a failure (`splitIncomparable`). */
|
|
96
|
+
readonly incomparable?: {
|
|
97
|
+
readonly model: string;
|
|
98
|
+
readonly reason: string;
|
|
99
|
+
};
|
|
100
|
+
/** ARC-β R3-1 (messages routes only): the `provider/model` that actually
|
|
101
|
+
* answered this turn — a fallback when the arm's earlier candidates were
|
|
102
|
+
* skipped for the history. Absent on variables routes (unchanged samples). */
|
|
103
|
+
readonly servedBy?: string;
|
|
104
|
+
/** ARC-β R3-1: the run's `history_skipped` tag (`provider/model=code,…`) —
|
|
105
|
+
* present only when candidates were skipped before the ladder. */
|
|
106
|
+
readonly historySkipped?: string;
|
|
92
107
|
}
|
|
93
108
|
export declare function replayVariant(deps: RunDeps, captures: CaptureStore, req: ReplayRequest): Promise<readonly ReplaySampleResult[]>;
|
|
94
109
|
//# sourceMappingURL=replay.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"replay.d.ts","sourceRoot":"","sources":["../../src/bakeoff/replay.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAGH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAGzD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAI1C,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAC5C,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAC3C,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AAExD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;
|
|
1
|
+
{"version":3,"file":"replay.d.ts","sourceRoot":"","sources":["../../src/bakeoff/replay.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAGH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAGzD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAI1C,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAC5C,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAC3C,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AAExD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAKpD,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B;;uCAEmC;IACnC,QAAQ,CAAC,WAAW,CAAC,EAAE,OAAO,CAAC;IAC/B;;;;kFAI8E;IAC9E,QAAQ,CAAC,uBAAuB,CAAC,EAAE,OAAO,CAAC;IAC3C;;yEAEqE;IACrE,QAAQ,CAAC,QAAQ,CAAC,EAAE,WAAW,CAAC,MAAM,EAAE;QAAE,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAA;KAAE,CAAC,CAAC;IACrE;;sEAEkE;IAClE,QAAQ,CAAC,UAAU,CAAC,EAAE,UAAU,CAAC;IACjC;sFACkF;IAClF,QAAQ,CAAC,KAAK,CAAC,EAAE,WAAW,CAAC;IAC7B;;;sFAGkF;IAClF,QAAQ,CAAC,WAAW,CAAC,EAAE,iBAAiB,CAAC;IACzC;;;;;iFAK6E;IAC7E,QAAQ,CAAC,WAAW,CAAC,EAAE,WAAW,CAAC;CACpC;AAED,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC;IAC3B;wEACoE;IACpE,QAAQ,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC;;;oCAGgC;IAChC,QAAQ,CAAC,aAAa,EAAE,OAAO,GAAG,IAAI,CAAC;IACvC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,mEAAmE;IACnE,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B;yDACqD;IACrD,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAC;IACjC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,oEAAoE;IACpE,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B;sDACkD;IAClD,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC;;;wEAGoE;IACpE,QAAQ,CAAC,YAAY,CAAC,EAAE;QAAE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IAC5E;;kFAE8E;IAC9E,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAC3B;sEACkE;IAClE,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,CAAC;CAClC;AAED,wBAAsB,aAAa,CACjC,IAAI,EAAE,OAAO,EACb,QAAQ,EAAE,YAAY,EACtB,GAAG,EAAE,aAAa,GACjB,OAAO,CAAC,SAAS,kBAAkB,EAAE,CAAC,CAoKxC"}
|
package/dist/bakeoff/replay.js
CHANGED
|
@@ -26,6 +26,8 @@ const run_1 = require("../run/run");
|
|
|
26
26
|
const variant_1 = require("../run/variant");
|
|
27
27
|
const quality_1 = require("./quality");
|
|
28
28
|
const quality_gate_1 = require("../run/quality-gate");
|
|
29
|
+
const messages_1 = require("../run/messages");
|
|
30
|
+
const turn_replay_1 = require("./turn-replay");
|
|
29
31
|
async function replayVariant(deps, captures, req) {
|
|
30
32
|
const rows = captures.readLast(req.route, req.lastN);
|
|
31
33
|
const replayId = (0, node_crypto_1.randomUUID)();
|
|
@@ -41,6 +43,9 @@ async function replayVariant(deps, captures, req) {
|
|
|
41
43
|
const route = deps.routes.get(req.route);
|
|
42
44
|
const variant = req.variant === "default" ? undefined : route.variants.find((v) => v.name === req.variant);
|
|
43
45
|
const mode = variant !== undefined && (0, variant_1.isTrialOnlyVariant)(route, variant) ? "trial" : "live";
|
|
46
|
+
// ARC-β turn replay (M15 guard, roadmap §2): a messages route replays real
|
|
47
|
+
// end-user conversations, so it runs ONLY where capture is explicitly on.
|
|
48
|
+
(0, turn_replay_1.assertTurnReplayAllowed)(route, deps.settings ?? null);
|
|
44
49
|
// Zero production exposure knobs: no capture writes (no replay-of-replay),
|
|
45
50
|
// and the whole pass shares one hard envelope.
|
|
46
51
|
const replayDeps = {
|
|
@@ -60,6 +65,8 @@ async function replayVariant(deps, captures, req) {
|
|
|
60
65
|
let conform = false;
|
|
61
66
|
let repaired = false;
|
|
62
67
|
let failureClass = null;
|
|
68
|
+
let incomparable = null;
|
|
69
|
+
let servedBy = null;
|
|
63
70
|
let outputText = null;
|
|
64
71
|
let outputValue = undefined;
|
|
65
72
|
let produced = false;
|
|
@@ -77,11 +84,15 @@ async function replayVariant(deps, captures, req) {
|
|
|
77
84
|
// route rollups and the savings odometer EXCLUDE trial spend — a
|
|
78
85
|
// bake-off's real dispatches must never move a live route's numbers.
|
|
79
86
|
tags: { replay: replayId, replay_capture: sampleTag, variant: req.variant, lane: "trial" },
|
|
80
|
-
|
|
87
|
+
// ARC-β: a messages capture carries the history up to and including the
|
|
88
|
+
// turn's last user/tool message — the candidate answers THAT turn only (the
|
|
89
|
+
// captured output is the reference reply); nothing continues the conversation.
|
|
90
|
+
...(0, messages_1.parseCaptureInput)(capture.variablesJson),
|
|
81
91
|
...(req.variant !== "default" ? { variant: req.variant } : {}),
|
|
82
92
|
budget: { envelope: envelopeKey },
|
|
83
93
|
});
|
|
84
94
|
conform = result.meta.validated;
|
|
95
|
+
servedBy = `${result.meta.provider}/${result.meta.model}`;
|
|
85
96
|
repaired = result.meta.repairedBy !== null;
|
|
86
97
|
outputValue = result.output;
|
|
87
98
|
produced = true;
|
|
@@ -95,6 +106,7 @@ async function replayVariant(deps, captures, req) {
|
|
|
95
106
|
if (!(err instanceof failures_1.RigFailureError))
|
|
96
107
|
throw err;
|
|
97
108
|
failureClass = err.failure.class;
|
|
109
|
+
incomparable = (0, turn_replay_1.incomparableFrom)(err.failure);
|
|
98
110
|
}
|
|
99
111
|
const latencyMs = Date.now() - started;
|
|
100
112
|
// Unified value tier (RB-1 §4.3 + Q1 §4.2, F5 deterministic-first). Only on a
|
|
@@ -150,6 +162,9 @@ async function replayVariant(deps, captures, req) {
|
|
|
150
162
|
.filter((r) => r.repairedBy !== null)
|
|
151
163
|
.reduce((a, r) => a + r.costEstimateUsd, 0);
|
|
152
164
|
const pricingMissing = attemptRows.some((r) => r.pricingMissing);
|
|
165
|
+
// ARC-β R3-1: attribute a messages-route turn to the model that served it,
|
|
166
|
+
// and carry the pre-ladder skip (read from the call's own telemetry tags).
|
|
167
|
+
const historySkipped = route.inputKind === "messages" ? (0, turn_replay_1.skippedTagOf)(attemptRows) : undefined;
|
|
153
168
|
results.push({
|
|
154
169
|
captureId: capture.inferenceId,
|
|
155
170
|
conform,
|
|
@@ -161,6 +176,9 @@ async function replayVariant(deps, captures, req) {
|
|
|
161
176
|
pricingMissing,
|
|
162
177
|
latencyMs,
|
|
163
178
|
failureClass,
|
|
179
|
+
...(incomparable !== null ? { incomparable } : {}),
|
|
180
|
+
...(route.inputKind === "messages" && servedBy !== null ? { servedBy } : {}),
|
|
181
|
+
...(historySkipped !== undefined ? { historySkipped } : {}),
|
|
164
182
|
...(retainText
|
|
165
183
|
? { inputText: capture.variablesJson, outputText }
|
|
166
184
|
: {}),
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"report.d.ts","sourceRoot":"","sources":["../../src/bakeoff/report.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;
|
|
1
|
+
{"version":3,"file":"report.d.ts","sourceRoot":"","sources":["../../src/bakeoff/report.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAMH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAG3C,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AACvD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AACjD,OAAO,KAAK,EAAE,eAAe,EAAkB,MAAM,SAAS,CAAC;AAC/D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AA2CpD;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,QAAQ,EAAE,eAAe,GAAG,MAAM,GAAG,IAAI,CAexE;AAED;;;;GAIG;AACH,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,+EAA+E;IAC/E,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,gFAAgF;IAChF,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAClC,QAAQ,CAAC,WAAW,EAAE,mBAAmB,CAAC;CAC3C;AAED;;;;;;GAMG;AACH,wBAAgB,0BAA0B,CAAC,IAAI,EAAE,SAAS,gBAAgB,EAAE,GAAG,MAAM,EAAE,CAgCtF;AAED;;;;;GAKG;AACH,wBAAgB,yBAAyB,CAAC,QAAQ,EAAE,eAAe,GAAG,MAAM,EAAE,CAiB7E;AAED;;;;;GAKG;AACH,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,eAAe,GAAG,MAAM,EAAE,CAavE;AAED,wBAAgB,WAAW,CAAC,QAAQ,EAAE,eAAe,EAAE,aAAa,GAAE,SAAS,gBAAgB,EAAO,GAAG,IAAI,CA8D5G;AAED;;;;;;GAMG;AACH,wBAAsB,qBAAqB,CACzC,MAAM,EAAE,SAAS,EACjB,IAAI,EAAE,MAAM,EACZ,QAAQ,EAAE,SAAS,MAAM,EAAE,EAC3B,QAAQ,EAAE,MAAM,GAAG,SAAS,EAC5B,IAAI,EAAE,eAAe,GAAG,SAAS,EACjC,MAAM,EAAE,OAAO,EACf,aAAa,CAAC,EAAE,aAAa,GAC5B,OAAO,CAAC,MAAM,CAAC,CA+CjB"}
|