@axlsdk/eval 0.25.3 → 0.27.0

package/dist/cli.js

@@ -14,7 +14,7 @@ import {
   runEval,
   scorerCounts,
   serializeRequestRecords
-} from "./chunk-F4MJ26YK.js";
+} from "./chunk-3XLPDV3O.js";
 
 // src/cli.ts
 import { readdirSync, statSync } from "fs";

package/dist/index.cjs

@@ -42,11 +42,15 @@ __export(index_exports, {
   evaluateScorerTolerance: () => evaluateScorerTolerance,
   isBudgetStopped: () => isBudgetStopped,
   llmScorer: () => llmScorer,
+  loadModelCalls: () => loadModelCalls,
+  modelCallDataset: () => modelCallDataset,
   normalizeScorerResult: () => normalizeScorerResult,
   pairedBootstrapCI: () => pairedBootstrapCI,
   parseRequestRecords: () => parseRequestRecords,
   readAccounting: () => readAccounting,
+  recordedModelCall: () => recordedModelCall,
   refusedWork: () => refusedWork,
+  replayModelCall: () => replayModelCall,
   rescore: () => rescore,
   resolveCaptureLimits: () => resolveCaptureLimits,
   runEval: () => runEval,
@@ -150,6 +154,401 @@ function dataset(config) {
   };
 }
 
+// src/model-calls.ts
+var import_promises2 = require("fs/promises");
+var import_zod = require("zod");
+var import_axl = require("@axlsdk/axl");
+function invalid(location, message) {
+  throw new import_axl.AxlError("INVALID_MODEL_CALL_SOURCE", `${location}: ${message}`);
+}
+function versionOf(value) {
+  return value !== null && typeof value === "object" ? Object.getOwnPropertyDescriptor(value, "v")?.value : void 0;
+}
+function parseJson(value, location) {
+  try {
+    return JSON.parse(value);
+  } catch {
+    return invalid(location, "invalid JSON");
+  }
+}
+function arrayLength(source, location) {
+  let array;
+  let descriptor;
+  try {
+    array = Array.isArray(source);
+    if (array) descriptor = Object.getOwnPropertyDescriptor(source, "length");
+  } catch {
+    return invalid(location, "array could not be inspected");
+  }
+  if (!array || !descriptor || !("value" in descriptor)) invalid(location, "expected an array");
+  const length = descriptor.value;
+  if (typeof length !== "number" || !Number.isSafeInteger(length) || length < 0)
+    invalid(location, "invalid array length");
+  return length;
+}
+function arraySlot(source, index, location) {
+  let descriptor;
+  try {
+    descriptor = Object.getOwnPropertyDescriptor(source, String(index));
+  } catch {
+    return invalid(location, "array entry could not be inspected");
+  }
+  if (!descriptor || !("value" in descriptor))
+    invalid(location, "array entry is missing or is an accessor");
+  return descriptor.value;
+}
+function* locateArray(source, at) {
+  const length = arrayLength(source, "source");
+  for (let index = 0; index < length; index++) {
+    const location = at(index);
+    yield { value: arraySlot(source, index, location), position: index, location };
+  }
+}
+async function loadModelCalls(source) {
+  let located;
+  if (typeof source === "string") {
+    const content = (await (0, import_promises2.readFile)(source, "utf8")).replace(/^\uFEFF/, "");
+    if (content.trimStart().startsWith("[")) {
+      const parsed = parseJson(content, `file ${source}`);
+      if (!Array.isArray(parsed)) invalid(`file ${source}`, "expected a JSON array");
+      located = locateArray(parsed, (index) => `entry ${index + 1} of ${source}`);
+    } else {
+      const rows = content.split(/\r?\n/);
+      if (content.endsWith("\n")) rows.pop();
+      located = rows.flatMap((row, index) => {
+        const location = `line ${index + 1} of ${source}`;
+        if (row.trim() === "") return [];
+        return [{ value: parseJson(row, location), position: index, location }];
+      });
+    }
+  } else {
+    located = locateArray(source, (index) => `item ${index + 1}`);
+  }
+  const journal = [];
+  const folded = [];
+  const firstJournal = /* @__PURE__ */ new Map();
+  let format;
+  for (const entry of located) {
+    let version;
+    try {
+      version = versionOf(entry.value);
+    } catch {
+      invalid(entry.location, "record version could not be inspected");
+    }
+    if (version === 1 || version === 2) {
+      if (format !== void 0 && format !== version)
+        invalid(entry.location, "one source cannot mix v1 journal lines with v2 records");
+      format = version;
+    }
+    if (version === 1) {
+      const parsed = import_axl.capturedJournalLineSchema.safeParse(entry.value);
+      if (!parsed.success) invalid(entry.location, parsed.error.message);
+      journal.push({ ...entry, value: parsed.data, operationId: parsed.data.operationId });
+      if (!firstJournal.has(parsed.data.operationId))
+        firstJournal.set(parsed.data.operationId, entry);
+    } else if (version === 2) {
+      const parsed = import_axl.foldedOperationRecordSchema.safeParse(entry.value);
+      if (!parsed.success) invalid(entry.location, parsed.error.message);
+      folded.push({ record: parsed.data, location: entry.location, position: entry.position });
+    } else {
+      invalid(entry.location, "unsupported capture version");
+    }
+  }
+  if (journal.length) {
+    let records;
+    try {
+      records = (0, import_axl.foldJournal)(journal.map((entry) => entry.value));
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      const line = /\bat line (\d+)\b/.exec(message) ?? /^Invalid capture journal line (\d+):/.exec(message);
+      const indexed = line ? journal[Number(line[1]) - 1] : void 0;
+      const matched = indexed ?? [...journal].reverse().find((entry) => {
+        const marker = `capture operation ${entry.operationId}`;
+        const at = message.indexOf(marker);
+        if (at < 0) return false;
+        const suffix = message.slice(at + marker.length);
+        return suffix === "" || suffix.startsWith(":") || suffix.startsWith(" at line");
+      });
+      invalid(matched?.location ?? journal[0].location, message);
+    }
+    for (const record of records) {
+      const first = firstJournal.get(record.id);
+      folded.push({ record, location: first.location, position: first.position });
+    }
+  }
+  folded.sort((a, b) => a.position - b.position);
+  const seen = /* @__PURE__ */ new Map();
+  for (const item of folded) {
+    const previous = seen.get(item.record.id);
+    if (previous)
+      invalid(item.location, `duplicate operation id ${item.record.id} (first at ${previous})`);
+    seen.set(item.record.id, item.location);
+  }
+  return folded.map((item) => item.record);
+}
+var modelCallInputSchema = import_zod.z.object({ recorded: import_axl.modelCallRecordSchema }).transform(
+  ({ recorded }) => ({
+    ...recorded.request ? { request: recorded.request } : {},
+    ids: recorded.ids,
+    attributes: recorded.attributes,
+    recorded
+  })
+);
+function modelCallDataset(records, options) {
+  const items = [];
+  const length = arrayLength(records, "records");
+  for (let index = 0; index < length; index++) {
+    const parsed = import_axl.foldedOperationRecordSchema.safeParse(
+      arraySlot(records, index, `record ${index + 1}`)
+    );
+    if (!parsed.success) invalid(`record ${index + 1}`, parsed.error.message);
+    const record = parsed.data;
+    if (options?.filter && !options.filter(record)) continue;
+    if (record.kind === "external")
+      throw new import_axl.AxlError(
+        "MODEL_CALL_EXPECTED",
+        `record ${index + 1} (${record.id}) is external; supply a filter to exclude it`
+      );
+    const input = {
+      ...record.request ? { request: record.request } : {},
+      ids: record.ids,
+      attributes: record.attributes,
+      recorded: record
+    };
+    items.push({
+      input,
+      ...options?.annotate ? { annotations: options.annotate(record, index) } : {}
+    });
+  }
+  return dataset({
+    name: "model-calls",
+    schema: modelCallInputSchema,
+    annotations: import_zod.z.custom(),
+    items
+  });
+}
+function readInput(input) {
+  const candidate = input !== null && typeof input === "object" ? Object.getOwnPropertyDescriptor(input, "recorded")?.value : void 0;
+  if (candidate !== null && typeof candidate === "object" && Object.getOwnPropertyDescriptor(candidate, "kind")?.value === "external")
+    throw new import_axl.AxlError(
+      "MODEL_CALL_EXPECTED",
+      "external operations cannot be scored or replayed as model calls"
+    );
+  const parsed = import_axl.modelCallRecordSchema.safeParse(candidate);
+  if (!parsed.success)
+    throw new import_axl.AxlError(
+      "INVALID_MODEL_CALL_INPUT",
+      `expected a validated model-call input: ${parsed.error.message}`
+    );
+  return parsed.data;
+}
+function modelCallOutput(response, record) {
+  const output = {
+    content: response.content,
+    ...response.thinking ? { thinking: response.thinking } : {},
+    ...response.tool_calls?.length ? { tool_calls: response.tool_calls } : {},
+    ...response.finishReason !== void 0 ? { finishReason: response.finishReason } : {},
+    ...response.rawFinishReason !== void 0 ? { rawFinishReason: response.rawFinishReason } : {},
+    ...response.responseId !== void 0 ? { responseId: response.responseId } : {},
+    ...response.responseModel !== void 0 ? { responseModel: response.responseModel } : {}
+  };
+  const format = record.request?.responseFormat?.type;
+  if (format !== "json_object" && format !== "json_schema") return output;
+  if (output.content === "" && output.tool_calls) return output;
+  try {
+    return { ...output, json: JSON.parse(output.content) };
+  } catch {
+    throw new import_axl.AxlError(
+      "INVALID_MODEL_CALL_JSON",
+      `model call ${record.id} did not return valid JSON`
+    );
+  }
+}
+function providerOutput(response, record) {
+  return modelCallOutput(
+    {
+      content: response.content,
+      ...response.thinking_content !== void 0 ? { thinking: response.thinking_content } : {},
+      ...response.tool_calls !== void 0 ? { tool_calls: response.tool_calls } : {},
+      ...response.finishReason !== void 0 ? { finishReason: response.finishReason } : {},
+      ...response.rawFinishReason !== void 0 ? { rawFinishReason: response.rawFinishReason } : {},
+      ...response.responseId !== void 0 ? { responseId: response.responseId } : {},
+      ...response.responseModel !== void 0 ? { responseModel: response.responseModel } : {}
+    },
+    record
+  );
+}
+function recordedModelCall() {
+  return async (input) => {
+    const record = readInput(input);
+    if (record.captured.redacted)
+      throw new import_axl.AxlError("RECORDED_OUTPUT_UNAVAILABLE", `model call ${record.id} was redacted`);
+    if (record.outcome.status !== "ok" || !("response" in record.outcome))
+      throw new import_axl.AxlError(
+        "RECORDED_OUTPUT_UNAVAILABLE",
+        `model call ${record.id} has no complete successful output (${record.outcome.status})`
+      );
+    return { output: modelCallOutput(record.outcome.response, record) };
+  };
+}
+function replayRequest(record) {
+  const request = record.request;
+  if (!request || !record.captured.replayable || record.captured.continuityCaptured !== true || record.captured.redacted || record.captured.truncated || record.captured.omitted.length > 0 || (request.providerOptionKeys?.length ?? 0) > 0)
+    throw new import_axl.AxlError(
+      "MODEL_CALL_NOT_REPLAYABLE",
+      `model call ${record.id} does not contain an exact replayable request`
+    );
+  return request;
+}
+function chatOptions(request, model) {
+  if (!request.options.model)
+    throw new import_axl.AxlError("INVALID_MODEL_CALL_INPUT", "captured request model is empty");
+  const effort = request.options.effort;
+  if (effort !== void 0 && !["none", "low", "medium", "high", "xhigh", "max"].includes(effort))
+    throw new import_axl.AxlError("INVALID_MODEL_CALL_INPUT", `unsupported captured effort ${effort}`);
+  const toolChoice = request.options.toolChoice;
+  if (toolChoice !== void 0 && toolChoice !== "auto" && toolChoice !== "none" && toolChoice !== "required" && !(toolChoice !== null && typeof toolChoice === "object" && !Array.isArray(toolChoice) && toolChoice.type === "function" && Object.keys(toolChoice).every((key) => key === "type" || key === "function") && toolChoice.function !== null && typeof toolChoice.function === "object" && !Array.isArray(toolChoice.function) && typeof toolChoice.function.name === "string" && Object.keys(toolChoice.function).every((key) => key === "name")))
+    throw new import_axl.AxlError("INVALID_MODEL_CALL_INPUT", "unsupported captured tool choice");
+  return {
+    model,
+    ...request.options.temperature !== void 0 ? { temperature: request.options.temperature } : {},
+    ...request.options.maxTokens !== void 0 ? { maxTokens: request.options.maxTokens } : {},
+    ...request.options.thinkingBudget !== void 0 ? { thinkingBudget: request.options.thinkingBudget } : {},
+    ...request.options.includeThoughts !== void 0 ? { includeThoughts: request.options.includeThoughts } : {},
+    ...request.options.stop !== void 0 ? { stop: [...request.options.stop] } : {},
+    ...request.options.promptCache !== void 0 ? { promptCache: request.options.promptCache } : {},
+    ...effort !== void 0 ? { effort } : {},
+    ...toolChoice !== void 0 ? { toolChoice } : {},
+    ...request.tools ? {
+      tools: request.tools.map((tool) => ({
+        type: "function",
+        function: {
+          name: tool.name,
+          description: tool.description ?? "",
+          parameters: tool.parameters,
+          ...tool.strict !== void 0 ? { strict: tool.strict } : {}
+        }
+      }))
+    } : {},
+    ...request.responseFormat ? { responseFormat: request.responseFormat } : {}
+  };
+}
+function chatMessages(request) {
+  return request.messages.map((message) => {
+    if (!["system", "user", "assistant", "tool"].includes(message.role))
+      throw new import_axl.AxlError("MODEL_CALL_NOT_REPLAYABLE", "captured message cannot be reconstructed");
+    let content;
+    if (message.input) {
+      const text = message.content ?? "";
+      let offset = 0;
+      const parts = [];
+      for (const part of message.input.parts) {
+        if (part.type !== "text" || offset + part.characters > text.length)
+          throw new import_axl.AxlError(
+            "MODEL_CALL_NOT_REPLAYABLE",
+            "captured content parts cannot be reconstructed"
+          );
+        parts.push({ type: "text", text: text.slice(offset, offset + part.characters) });
+        offset += part.characters;
+      }
+      if (offset !== text.length)
+        throw new import_axl.AxlError(
+          "MODEL_CALL_NOT_REPLAYABLE",
+          "captured text lengths do not match content"
+        );
+      content = parts;
+    } else if (message.content !== null) {
+      content = message.content;
+    } else {
+      throw new import_axl.AxlError("MODEL_CALL_NOT_REPLAYABLE", "captured message content is absent");
+    }
+    return {
+      role: message.role,
+      content,
+      ...message.name !== void 0 ? { name: message.name } : {},
+      ...message.agent !== void 0 ? { agent: message.agent } : {},
+      ...message.origin !== void 0 ? { origin: message.origin } : {},
+      ...message.tool_calls !== void 0 ? { tool_calls: message.tool_calls } : {},
+      ...message.tool_call_id !== void 0 ? { tool_call_id: message.tool_call_id } : {},
+      ...message.providerMetadata !== void 0 ? { providerMetadata: message.providerMetadata } : {}
+    };
+  });
+}
+function replayModelCall(options = {}) {
+  return async (input, runtime) => {
+    const record = readInput(input);
+    const request = replayRequest(record);
+    const messages = chatMessages(request);
+    const capturedOptions = chatOptions(request, request.options.model);
+    if (!runtime)
+      throw new import_axl.AxlError("MODEL_CALL_RUNTIME_REQUIRED", "replay requires an AxlRuntime");
+    const uri = options.model ?? `${record.provider}:${request.options.model}`;
+    if (!uri.includes(":"))
+      throw new import_axl.AxlError("MODEL_CALL_MODEL_UNRESOLVED", "model override must use provider:model");
+    let resolved;
+    try {
+      resolved = runtime.resolveProvider(uri);
+    } catch {
+      throw new import_axl.AxlError(
+        "MODEL_CALL_MODEL_UNRESOLVED",
+        `cannot resolve ${uri}; supply an explicit provider:model override`
+      );
+    }
+    if (!options.model && resolved.provider.name !== record.provider)
+      throw new import_axl.AxlError(
+        "MODEL_CALL_MODEL_UNRESOLVED",
+        `resolved ${uri} to a different provider; supply an explicit override`
+      );
+    const callOptions = {
+      ...capturedOptions,
+      model: options.model ? resolved.model : request.options.model
+    };
+    if (record.kind === "chat") {
+      const response = await resolved.provider.chat(messages, callOptions);
+      return { output: providerOutput(response, record) };
+    }
+    let content = "";
+    let thinking = "";
+    const calls = /* @__PURE__ */ new Map();
+    let done;
+    for await (const chunk of resolved.provider.stream(messages, callOptions)) {
+      if (chunk.type === "text_delta") content += chunk.content;
+      else if (chunk.type === "thinking_delta") thinking += chunk.content;
+      else if (chunk.type === "tool_call_delta") {
+        const call = calls.get(chunk.id) ?? { id: chunk.id, name: "", arguments: "" };
+        if (chunk.name) call.name = chunk.name;
+        if (chunk.arguments) call.arguments += chunk.arguments;
+        calls.set(chunk.id, call);
+      } else if (chunk.type === "done") {
+        done = chunk;
+        break;
+      }
+    }
+    if (!done)
+      throw new import_axl.AxlError("MODEL_CALL_STREAM_INCOMPLETE", "replayed stream ended without done");
+    return {
+      output: providerOutput(
+        {
+          content,
+          ...thinking ? { thinking_content: thinking } : {},
+          ...calls.size ? {
+            tool_calls: [...calls.values()].map((call) => ({
+              id: call.id,
+              type: "function",
+              function: { name: call.name, arguments: call.arguments }
+            }))
+          } : {},
+          ...done.finishReason !== void 0 ? { finishReason: done.finishReason } : {},
+          ...done.rawFinishReason !== void 0 ? { rawFinishReason: done.rawFinishReason } : {},
+          ...done.responseId !== void 0 ? { responseId: done.responseId } : {},
+          ...done.responseModel !== void 0 ? { responseModel: done.responseModel } : {}
+        },
+        record
+      )
+    };
+  };
+}
+
 // src/scorer.ts
 function normalizeScorerResult(result) {
   return typeof result === "number" ? { score: result } : result;
@@ -174,16 +573,16 @@ function scorer(config) {
 }
 
 // src/llm-scorer.ts
-var import_zod = require("zod");
-var import_axl = require("@axlsdk/axl");
+var import_zod2 = require("zod");
+var import_axl2 = require("@axlsdk/axl");
 function isZodIssueLike(value) {
   if (!value || typeof value !== "object") return false;
   const issue = value;
   return Array.isArray(issue.path) && issue.path.every((segment) => typeof segment === "string" || typeof segment === "number") && typeof issue.message === "string";
 }
 function llmScorer(config) {
-  const schema = config.schema ?? import_zod.z.object({ score: import_zod.z.number().min(0).max(1), reasoning: import_zod.z.string() });
-  const schemaJson = JSON.stringify((0, import_axl.zodToJsonSchema)(schema), null, 2);
+  const schema = config.schema ?? import_zod2.z.object({ score: import_zod2.z.number().min(0).max(1), reasoning: import_zod2.z.string() });
+  const schemaJson = JSON.stringify((0, import_axl2.zodToJsonSchema)(schema), null, 2);
   return {
     name: config.name,
     description: config.description,
@@ -232,7 +631,7 @@ ${JSON.stringify(annotations, null, 2)}` : "",
       const responseCost = response.cost;
       let validated;
       try {
-        const parsed = JSON.parse((0, import_axl.extractJson)(response.content));
+        const parsed = JSON.parse((0, import_axl2.extractJson)(response.content));
         validated = schema.parse(parsed);
       } catch (err) {
         attachScorerErrorCost(err, responseCost);
@@ -263,7 +662,7 @@ function defineEval(config) {
 }
 
 // src/runner.ts
-var import_axl3 = require("@axlsdk/axl");
+var import_axl4 = require("@axlsdk/axl");
 
 // src/utils.ts
 function computeStats(scores) {
@@ -333,7 +732,7 @@ async function mapWithConcurrency(items, concurrency, task) {
 }
 
 // src/accounting.ts
-var import_axl2 = require("@axlsdk/axl");
+var import_axl3 = require("@axlsdk/axl");
 var USAGE_KEYS = [
   "inputTokens",
   "outputTokens",
@@ -395,14 +794,14 @@ async function trackScope(runtime, fn, options) {
 function parseBudget(budget) {
   const match = typeof budget === "string" ? budget.match(/^\$?(\d+(?:\.\d+)?)$/) : null;
   if (!match) {
-    throw new import_axl2.AxlError(
+    throw new import_axl3.AxlError(
       "INVALID_BUDGET",
       `Invalid budget "${String(budget)}": expected a non-negative USD amount like "$1" or "0.50".`
     );
   }
   const limit = Number(match[1]);
   if (!Number.isFinite(limit)) {
-    throw new import_axl2.AxlError("INVALID_BUDGET", `Invalid budget "${budget}": not a finite amount.`);
+    throw new import_axl3.AxlError("INVALID_BUDGET", `Invalid budget "${budget}": not a finite amount.`);
   }
   return limit;
 }
@@ -410,7 +809,7 @@ function usableCost(value) {
   return typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : 0;
 }
 function isAdmissionDenied(err) {
-  return (0, import_axl2.isAdmissionDeniedError)(err);
+  return (0, import_axl3.isAdmissionDeniedError)(err);
 }
 function readAccounting(result) {
   if (result?.accounting) return result.accounting;
@@ -750,7 +1149,7 @@ async function stageCapture(runtime, ownerId, option) {
   const staged = await runtime.stageDiagnosticArtifact({ kind: "eval", id: ownerId });
   return {
     artifactId: staged.artifactId,
-    channel: new import_axl3.RequestCaptureChannel({
+    channel: new import_axl4.RequestCaptureChannel({
       sink: staged.sink,
       ...limits,
       // Redaction is the RUNTIME's policy, not the caller's: a run cannot opt
@@ -818,7 +1217,7 @@ function attachOperationRefs(items, channel) {
 }
 var MAX_CAUSE_DEPTH = 8;
 function isProviderErrorLike(value) {
-  if (value instanceof import_axl3.ProviderError) return true;
+  if (value instanceof import_axl4.ProviderError) return true;
   if (typeof value !== "object" || value === null) return false;
   const candidate = value;
   return candidate.code === "PROVIDER_ERROR" && candidate.name === "ProviderError";
@@ -826,7 +1225,7 @@ function isProviderErrorLike(value) {
 function isTimeoutErrorLike(value) {
   if (typeof value !== "object" || value === null) return false;
   const candidate = value;
-  return candidate.name === "TimeoutError" && (value instanceof import_axl3.TimeoutError || candidate.code === "TIMEOUT");
+  return candidate.name === "TimeoutError" && (value instanceof import_axl4.TimeoutError || candidate.code === "TIMEOUT");
 }
 function describeTimeoutFailure(error) {
   const breakdown = error.breakdown;
@@ -872,12 +1271,12 @@ async function runEval(config, executeWorkflow, runtime, options) {
   const id = (0, import_node_crypto.randomUUID)();
   const itemErrorRateLimit = config.failOnItemErrorRate ?? DEFAULT_ITEM_ERROR_RATE_LIMIT;
   if (!isErrorRateLimit(itemErrorRateLimit)) {
-    throw new import_axl3.AxlError(
+    throw new import_axl4.AxlError(
       "INVALID_ITEM_ERROR_RATE",
       `Invalid failOnItemErrorRate (${String(itemErrorRateLimit)}): expected a number in [0, 1]; 1 disables the gate.`
     );
   }
-  const admission = config.budget != null ? new import_axl3.AdmissionController({ limit: parseBudget(config.budget) }) : void 0;
+  const admission = config.budget != null ? new import_axl4.AdmissionController({ limit: parseBudget(config.budget) }) : void 0;
   const capture = await stageCapture(runtime, id, options?.captureRequests);
   let items;
   try {
@@ -1672,7 +2071,7 @@ function evaluateItemErrorRateGate(baseline, candidate, maxItemErrorRate = DEFAU
 }
 
 // src/rescore.ts
-var import_axl4 = require("@axlsdk/axl");
+var import_axl5 = require("@axlsdk/axl");
 var import_node_crypto2 = require("crypto");
 var COPY_SHARE_OF_RUN_BOUND = 0.75;
 function describeFailure(error) {
@@ -1686,7 +2085,7 @@ async function beginCapture(source, ownerId, runtime, options) {
   if (!limits) return {};
   const open = (artifactId, sink, carriedBytes) => ({
     artifactId,
-    channel: new import_axl4.RequestCaptureChannel({
+    channel: new import_axl5.RequestCaptureChannel({
       sink,
       ...limits,
       carriedBytes,
@@ -1769,7 +2168,7 @@ async function rescore(result, scorers, runtime, options) {
   const rescoredId = (0, import_node_crypto2.randomUUID)();
   const concurrency = options?.concurrency ?? 5;
   const scorerConcurrency = options?.scorerConcurrency ?? 5;
-  const admission = options?.budget != null ? new import_axl4.AdmissionController({ limit: parseBudget(options.budget) }) : void 0;
+  const admission = options?.budget != null ? new import_axl5.AdmissionController({ limit: parseBudget(options.budget) }) : void 0;
   let closedBy;
   const noteClosure = (source) => {
     if (closedBy === void 0 && admission?.closed) closedBy = source;
@@ -2045,11 +2444,15 @@ function aggregateRuns(runs) {
   evaluateScorerTolerance,
   isBudgetStopped,
   llmScorer,
+  loadModelCalls,
+  modelCallDataset,
   normalizeScorerResult,
   pairedBootstrapCI,
   parseRequestRecords,
   readAccounting,
+  recordedModelCall,
   refusedWork,
+  replayModelCall,
   rescore,
   resolveCaptureLimits,
   runEval,