@axlsdk/eval 0.25.2 → 0.26.0

package/dist/index.d.cts

@@ -1,5 +1,5 @@
 import { z } from 'zod';
-import { Provider, Effort, ArtifactManifest, CapturedRequestRecord, Accounting, AxlEvent, AxlRuntime } from '@axlsdk/axl';
+import { FoldedOperationRecord, ModelCallRecord, ToolCallMessage, FinishReason, EvalExecuteWorkflow, Provider, Effort, ArtifactManifest, CapturedRequestRecord, Accounting, AxlEvent, AxlRuntime } from '@axlsdk/axl';
 
 type DatasetItem<TInput, TAnnotations = undefined> = {
     input: TInput;
@@ -60,6 +60,46 @@ type Dataset<TInput = unknown, TAnnotations = unknown> = {
 };
 declare function dataset<TInput extends z.ZodType, TAnnotations extends z.ZodType>(config: DatasetConfig<TInput, TAnnotations>): Dataset<z.infer<TInput>, z.infer<TAnnotations>>;
 
+type ModelCallInput = {
+    request?: ModelCallRecord['request'];
+    ids: ModelCallRecord['ids'];
+    attributes: ModelCallRecord['attributes'];
+    recorded: ModelCallRecord;
+};
+type ModelCallDatasetOptions<TAnnotations> = {
+    /** Runs after validation, including on external operations. */
+    filter?: (record: FoldedOperationRecord) => boolean;
+    /** Attach ground truth for scorers without changing their signature. */
+    annotate?: (record: ModelCallRecord, index: number) => TAnnotations;
+};
+type ReplayModelCallOptions = {
+    model?: string;
+};
+/**
+ * The `output` both executors return: one model response, field-named like a
+ * recorded `ok` outcome's `response` so a scorer can compare a replay with
+ * `input.recorded` directly. Tool calls are part of the answer, not dropped.
+ */
+type ModelCallOutput = {
+    content: string;
+    /** Parsed `content`, present when the request asked for JSON and the model answered in text. */
+    json?: unknown;
+    thinking?: string;
+    tool_calls?: ToolCallMessage[];
+    finishReason?: FinishReason;
+    rawFinishReason?: string;
+    responseId?: string;
+    responseModel?: string;
+};
+/** Read v2 records or v1 phase journals without dropping external operations. */
+declare function loadModelCalls(source: string | readonly unknown[]): Promise<FoldedOperationRecord[]>;
+/** Create an ordinary Dataset for runEval/defineEval; external operations require an explicit filter. */
+declare function modelCallDataset<TAnnotations = undefined>(records: readonly FoldedOperationRecord[], options?: ModelCallDatasetOptions<TAnnotations>): Dataset<ModelCallInput, TAnnotations>;
+/** Rescore captured output (a {@link ModelCallOutput}) without calling a provider or returning historical cost. */
+declare function recordedModelCall(): EvalExecuteWorkflow;
+/** Replay one captured provider call into a {@link ModelCallOutput}; returned tool calls are not executed. */
+declare function replayModelCall(options?: ReplayModelCallOptions): EvalExecuteWorkflow;
+
 /** Result from a scorer that includes metadata beyond the numeric score. */
 type ScorerResult = {
     score: number;
@@ -203,7 +243,7 @@ type RequestRecord = CapturedRequestRecord;
  */
 type OperationRef = {
     operationId: string;
-    kind: 'chat' | 'stream';
+    kind: 'chat' | 'stream' | 'external';
     turn?: number;
     attempt?: number;
     status: 'recorded' | 'start_only' | 'truncated' | 'omitted';
@@ -1288,4 +1328,4 @@ declare function isBudgetStopped(summary: {
  */
 declare function aggregateAccounting(inputs: readonly Accounting[]): Accounting;
 
-export { type BootstrapCIResult, type CaptureRequestsOption, DEFAULT_COPY_MAX_BYTES, DEFAULT_SIDECAR_MAX_BYTES, type Dataset, type DatasetConfig, type DatasetItem, type DegradedScorer, type DiagnosticManifest, type EvalAccounting, type EvalBudgetStatus, type EvalCompareOptions, type EvalComparison, type EvalComparisonPartial, type EvalConfig, type EvalCoverage, type EvalImprovement, type EvalItem, type EvalItemFailure, type EvalItemOutcome, type EvalProgressEvent, type EvalRegression, type EvalResult, type EvalSummary, type ExtraKeyPolicy, type ItemErrorRate, type ItemModelTiming, type LlmScorerConfig, type ModelTimingStats, type MultiRunSummary, type OperationRef, type RequestRecord, type RescoreOptions, type RunEvalOptions, type Scorer, type ScorerApplies, type ScorerConfig, type ScorerContext, type ScorerDetail, type ScorerFn, type ScorerOutcome, type ScorerResult, type ScorerToleranceVerdict, type SidecarValidation, aggregateAccounting, aggregateRuns, dataset, defineEval, evalCompare, evaluateItemErrorRateGate, evaluateScorerErrorRateGate, evaluateScorerTolerance, isBudgetStopped, llmScorer, normalizeScorerResult, pairedBootstrapCI, parseRequestRecords, readAccounting, refusedWork, rescore, resolveCaptureLimits, runEval, scorer, serializeRequestRecords, toDiagnosticManifest, unavailableManifest, validateRequestSidecar };
+export { type BootstrapCIResult, type CaptureRequestsOption, DEFAULT_COPY_MAX_BYTES, DEFAULT_SIDECAR_MAX_BYTES, type Dataset, type DatasetConfig, type DatasetItem, type DegradedScorer, type DiagnosticManifest, type EvalAccounting, type EvalBudgetStatus, type EvalCompareOptions, type EvalComparison, type EvalComparisonPartial, type EvalConfig, type EvalCoverage, type EvalImprovement, type EvalItem, type EvalItemFailure, type EvalItemOutcome, type EvalProgressEvent, type EvalRegression, type EvalResult, type EvalSummary, type ExtraKeyPolicy, type ItemErrorRate, type ItemModelTiming, type LlmScorerConfig, type ModelCallDatasetOptions, type ModelCallInput, type ModelCallOutput, type ModelTimingStats, type MultiRunSummary, type OperationRef, type ReplayModelCallOptions, type RequestRecord, type RescoreOptions, type RunEvalOptions, type Scorer, type ScorerApplies, type ScorerConfig, type ScorerContext, type ScorerDetail, type ScorerFn, type ScorerOutcome, type ScorerResult, type ScorerToleranceVerdict, type SidecarValidation, aggregateAccounting, aggregateRuns, dataset, defineEval, evalCompare, evaluateItemErrorRateGate, evaluateScorerErrorRateGate, evaluateScorerTolerance, isBudgetStopped, llmScorer, loadModelCalls, modelCallDataset, normalizeScorerResult, pairedBootstrapCI, parseRequestRecords, readAccounting, recordedModelCall, refusedWork, replayModelCall, rescore, resolveCaptureLimits, runEval, scorer, serializeRequestRecords, toDiagnosticManifest, unavailableManifest, validateRequestSidecar };

package/dist/index.d.ts

@@ -1,5 +1,5 @@
 import { z } from 'zod';
-import { Provider, Effort, ArtifactManifest, CapturedRequestRecord, Accounting, AxlEvent, AxlRuntime } from '@axlsdk/axl';
+import { FoldedOperationRecord, ModelCallRecord, ToolCallMessage, FinishReason, EvalExecuteWorkflow, Provider, Effort, ArtifactManifest, CapturedRequestRecord, Accounting, AxlEvent, AxlRuntime } from '@axlsdk/axl';
 
 type DatasetItem<TInput, TAnnotations = undefined> = {
     input: TInput;
@@ -60,6 +60,46 @@ type Dataset<TInput = unknown, TAnnotations = unknown> = {
 };
 declare function dataset<TInput extends z.ZodType, TAnnotations extends z.ZodType>(config: DatasetConfig<TInput, TAnnotations>): Dataset<z.infer<TInput>, z.infer<TAnnotations>>;
 
+type ModelCallInput = {
+    request?: ModelCallRecord['request'];
+    ids: ModelCallRecord['ids'];
+    attributes: ModelCallRecord['attributes'];
+    recorded: ModelCallRecord;
+};
+type ModelCallDatasetOptions<TAnnotations> = {
+    /** Runs after validation, including on external operations. */
+    filter?: (record: FoldedOperationRecord) => boolean;
+    /** Attach ground truth for scorers without changing their signature. */
+    annotate?: (record: ModelCallRecord, index: number) => TAnnotations;
+};
+type ReplayModelCallOptions = {
+    model?: string;
+};
+/**
+ * The `output` both executors return: one model response, field-named like a
+ * recorded `ok` outcome's `response` so a scorer can compare a replay with
+ * `input.recorded` directly. Tool calls are part of the answer, not dropped.
+ */
+type ModelCallOutput = {
+    content: string;
+    /** Parsed `content`, present when the request asked for JSON and the model answered in text. */
+    json?: unknown;
+    thinking?: string;
+    tool_calls?: ToolCallMessage[];
+    finishReason?: FinishReason;
+    rawFinishReason?: string;
+    responseId?: string;
+    responseModel?: string;
+};
+/** Read v2 records or v1 phase journals without dropping external operations. */
+declare function loadModelCalls(source: string | readonly unknown[]): Promise<FoldedOperationRecord[]>;
+/** Create an ordinary Dataset for runEval/defineEval; external operations require an explicit filter. */
+declare function modelCallDataset<TAnnotations = undefined>(records: readonly FoldedOperationRecord[], options?: ModelCallDatasetOptions<TAnnotations>): Dataset<ModelCallInput, TAnnotations>;
+/** Rescore captured output (a {@link ModelCallOutput}) without calling a provider or returning historical cost. */
+declare function recordedModelCall(): EvalExecuteWorkflow;
+/** Replay one captured provider call into a {@link ModelCallOutput}; returned tool calls are not executed. */
+declare function replayModelCall(options?: ReplayModelCallOptions): EvalExecuteWorkflow;
+
 /** Result from a scorer that includes metadata beyond the numeric score. */
 type ScorerResult = {
     score: number;
@@ -203,7 +243,7 @@ type RequestRecord = CapturedRequestRecord;
  */
 type OperationRef = {
     operationId: string;
-    kind: 'chat' | 'stream';
+    kind: 'chat' | 'stream' | 'external';
     turn?: number;
     attempt?: number;
     status: 'recorded' | 'start_only' | 'truncated' | 'omitted';
@@ -1288,4 +1328,4 @@ declare function isBudgetStopped(summary: {
  */
 declare function aggregateAccounting(inputs: readonly Accounting[]): Accounting;
 
-export { type BootstrapCIResult, type CaptureRequestsOption, DEFAULT_COPY_MAX_BYTES, DEFAULT_SIDECAR_MAX_BYTES, type Dataset, type DatasetConfig, type DatasetItem, type DegradedScorer, type DiagnosticManifest, type EvalAccounting, type EvalBudgetStatus, type EvalCompareOptions, type EvalComparison, type EvalComparisonPartial, type EvalConfig, type EvalCoverage, type EvalImprovement, type EvalItem, type EvalItemFailure, type EvalItemOutcome, type EvalProgressEvent, type EvalRegression, type EvalResult, type EvalSummary, type ExtraKeyPolicy, type ItemErrorRate, type ItemModelTiming, type LlmScorerConfig, type ModelTimingStats, type MultiRunSummary, type OperationRef, type RequestRecord, type RescoreOptions, type RunEvalOptions, type Scorer, type ScorerApplies, type ScorerConfig, type ScorerContext, type ScorerDetail, type ScorerFn, type ScorerOutcome, type ScorerResult, type ScorerToleranceVerdict, type SidecarValidation, aggregateAccounting, aggregateRuns, dataset, defineEval, evalCompare, evaluateItemErrorRateGate, evaluateScorerErrorRateGate, evaluateScorerTolerance, isBudgetStopped, llmScorer, normalizeScorerResult, pairedBootstrapCI, parseRequestRecords, readAccounting, refusedWork, rescore, resolveCaptureLimits, runEval, scorer, serializeRequestRecords, toDiagnosticManifest, unavailableManifest, validateRequestSidecar };
+export { type BootstrapCIResult, type CaptureRequestsOption, DEFAULT_COPY_MAX_BYTES, DEFAULT_SIDECAR_MAX_BYTES, type Dataset, type DatasetConfig, type DatasetItem, type DegradedScorer, type DiagnosticManifest, type EvalAccounting, type EvalBudgetStatus, type EvalCompareOptions, type EvalComparison, type EvalComparisonPartial, type EvalConfig, type EvalCoverage, type EvalImprovement, type EvalItem, type EvalItemFailure, type EvalItemOutcome, type EvalProgressEvent, type EvalRegression, type EvalResult, type EvalSummary, type ExtraKeyPolicy, type ItemErrorRate, type ItemModelTiming, type LlmScorerConfig, type ModelCallDatasetOptions, type ModelCallInput, type ModelCallOutput, type ModelTimingStats, type MultiRunSummary, type OperationRef, type ReplayModelCallOptions, type RequestRecord, type RescoreOptions, type RunEvalOptions, type Scorer, type ScorerApplies, type ScorerConfig, type ScorerContext, type ScorerDetail, type ScorerFn, type ScorerOutcome, type ScorerResult, type ScorerToleranceVerdict, type SidecarValidation, aggregateAccounting, aggregateRuns, dataset, defineEval, evalCompare, evaluateItemErrorRateGate, evaluateScorerErrorRateGate, evaluateScorerTolerance, isBudgetStopped, llmScorer, loadModelCalls, modelCallDataset, normalizeScorerResult, pairedBootstrapCI, parseRequestRecords, readAccounting, recordedModelCall, refusedWork, replayModelCall, rescore, resolveCaptureLimits, runEval, scorer, serializeRequestRecords, toDiagnosticManifest, unavailableManifest, validateRequestSidecar };

package/dist/index.js

@@ -22,7 +22,7 @@ import {
   toDiagnosticManifest,
   unavailableManifest,
   validateRequestSidecar
-} from "./chunk-F4MJ26YK.js";
+} from "./chunk-3XLPDV3O.js";
 
 // src/dataset.ts
 import { readFile } from "fs/promises";
@@ -116,8 +116,409 @@ function dataset(config) {
   };
 }
 
-// src/llm-scorer.ts
+// src/model-calls.ts
+import { readFile as readFile2 } from "fs/promises";
 import { z } from "zod";
+import {
+  AxlError,
+  capturedJournalLineSchema,
+  foldJournal,
+  foldedOperationRecordSchema,
+  modelCallRecordSchema
+} from "@axlsdk/axl";
+function invalid(location, message) {
+  throw new AxlError("INVALID_MODEL_CALL_SOURCE", `${location}: ${message}`);
+}
+function versionOf(value) {
+  return value !== null && typeof value === "object" ? Object.getOwnPropertyDescriptor(value, "v")?.value : void 0;
+}
+function parseJson(value, location) {
+  try {
+    return JSON.parse(value);
+  } catch {
+    return invalid(location, "invalid JSON");
+  }
+}
+function arrayLength(source, location) {
+  let array;
+  let descriptor;
+  try {
+    array = Array.isArray(source);
+    if (array) descriptor = Object.getOwnPropertyDescriptor(source, "length");
+  } catch {
+    return invalid(location, "array could not be inspected");
+  }
+  if (!array || !descriptor || !("value" in descriptor)) invalid(location, "expected an array");
+  const length = descriptor.value;
+  if (typeof length !== "number" || !Number.isSafeInteger(length) || length < 0)
+    invalid(location, "invalid array length");
+  return length;
+}
+function arraySlot(source, index, location) {
+  let descriptor;
+  try {
+    descriptor = Object.getOwnPropertyDescriptor(source, String(index));
+  } catch {
+    return invalid(location, "array entry could not be inspected");
+  }
+  if (!descriptor || !("value" in descriptor))
+    invalid(location, "array entry is missing or is an accessor");
+  return descriptor.value;
+}
+function* locateArray(source, at) {
+  const length = arrayLength(source, "source");
+  for (let index = 0; index < length; index++) {
+    const location = at(index);
+    yield { value: arraySlot(source, index, location), position: index, location };
+  }
+}
+async function loadModelCalls(source) {
+  let located;
+  if (typeof source === "string") {
+    const content = (await readFile2(source, "utf8")).replace(/^\uFEFF/, "");
+    if (content.trimStart().startsWith("[")) {
+      const parsed = parseJson(content, `file ${source}`);
+      if (!Array.isArray(parsed)) invalid(`file ${source}`, "expected a JSON array");
+      located = locateArray(parsed, (index) => `entry ${index + 1} of ${source}`);
+    } else {
+      const rows = content.split(/\r?\n/);
+      if (content.endsWith("\n")) rows.pop();
+      located = rows.flatMap((row, index) => {
+        const location = `line ${index + 1} of ${source}`;
+        if (row.trim() === "") return [];
+        return [{ value: parseJson(row, location), position: index, location }];
+      });
+    }
+  } else {
+    located = locateArray(source, (index) => `item ${index + 1}`);
+  }
+  const journal = [];
+  const folded = [];
+  const firstJournal = /* @__PURE__ */ new Map();
+  let format;
+  for (const entry of located) {
+    let version;
+    try {
+      version = versionOf(entry.value);
+    } catch {
+      invalid(entry.location, "record version could not be inspected");
+    }
+    if (version === 1 || version === 2) {
+      if (format !== void 0 && format !== version)
+        invalid(entry.location, "one source cannot mix v1 journal lines with v2 records");
+      format = version;
+    }
+    if (version === 1) {
+      const parsed = capturedJournalLineSchema.safeParse(entry.value);
+      if (!parsed.success) invalid(entry.location, parsed.error.message);
+      journal.push({ ...entry, value: parsed.data, operationId: parsed.data.operationId });
+      if (!firstJournal.has(parsed.data.operationId))
+        firstJournal.set(parsed.data.operationId, entry);
+    } else if (version === 2) {
+      const parsed = foldedOperationRecordSchema.safeParse(entry.value);
+      if (!parsed.success) invalid(entry.location, parsed.error.message);
+      folded.push({ record: parsed.data, location: entry.location, position: entry.position });
+    } else {
+      invalid(entry.location, "unsupported capture version");
+    }
+  }
+  if (journal.length) {
+    let records;
+    try {
+      records = foldJournal(journal.map((entry) => entry.value));
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      const line = /\bat line (\d+)\b/.exec(message) ?? /^Invalid capture journal line (\d+):/.exec(message);
+      const indexed = line ? journal[Number(line[1]) - 1] : void 0;
+      const matched = indexed ?? [...journal].reverse().find((entry) => {
+        const marker = `capture operation ${entry.operationId}`;
+        const at = message.indexOf(marker);
+        if (at < 0) return false;
+        const suffix = message.slice(at + marker.length);
+        return suffix === "" || suffix.startsWith(":") || suffix.startsWith(" at line");
+      });
+      invalid(matched?.location ?? journal[0].location, message);
+    }
+    for (const record of records) {
+      const first = firstJournal.get(record.id);
+      folded.push({ record, location: first.location, position: first.position });
+    }
+  }
+  folded.sort((a, b) => a.position - b.position);
+  const seen = /* @__PURE__ */ new Map();
+  for (const item of folded) {
+    const previous = seen.get(item.record.id);
+    if (previous)
+      invalid(item.location, `duplicate operation id ${item.record.id} (first at ${previous})`);
+    seen.set(item.record.id, item.location);
+  }
+  return folded.map((item) => item.record);
+}
+var modelCallInputSchema = z.object({ recorded: modelCallRecordSchema }).transform(
+  ({ recorded }) => ({
+    ...recorded.request ? { request: recorded.request } : {},
+    ids: recorded.ids,
+    attributes: recorded.attributes,
+    recorded
+  })
+);
+function modelCallDataset(records, options) {
+  const items = [];
+  const length = arrayLength(records, "records");
+  for (let index = 0; index < length; index++) {
+    const parsed = foldedOperationRecordSchema.safeParse(
+      arraySlot(records, index, `record ${index + 1}`)
+    );
+    if (!parsed.success) invalid(`record ${index + 1}`, parsed.error.message);
+    const record = parsed.data;
+    if (options?.filter && !options.filter(record)) continue;
+    if (record.kind === "external")
+      throw new AxlError(
+        "MODEL_CALL_EXPECTED",
+        `record ${index + 1} (${record.id}) is external; supply a filter to exclude it`
+      );
+    const input = {
+      ...record.request ? { request: record.request } : {},
+      ids: record.ids,
+      attributes: record.attributes,
+      recorded: record
+    };
+    items.push({
+      input,
+      ...options?.annotate ? { annotations: options.annotate(record, index) } : {}
+    });
+  }
+  return dataset({
+    name: "model-calls",
+    schema: modelCallInputSchema,
+    annotations: z.custom(),
+    items
+  });
+}
+function readInput(input) {
+  const candidate = input !== null && typeof input === "object" ? Object.getOwnPropertyDescriptor(input, "recorded")?.value : void 0;
+  if (candidate !== null && typeof candidate === "object" && Object.getOwnPropertyDescriptor(candidate, "kind")?.value === "external")
+    throw new AxlError(
+      "MODEL_CALL_EXPECTED",
+      "external operations cannot be scored or replayed as model calls"
+    );
+  const parsed = modelCallRecordSchema.safeParse(candidate);
+  if (!parsed.success)
+    throw new AxlError(
+      "INVALID_MODEL_CALL_INPUT",
+      `expected a validated model-call input: ${parsed.error.message}`
+    );
+  return parsed.data;
+}
+function modelCallOutput(response, record) {
+  const output = {
+    content: response.content,
+    ...response.thinking ? { thinking: response.thinking } : {},
+    ...response.tool_calls?.length ? { tool_calls: response.tool_calls } : {},
+    ...response.finishReason !== void 0 ? { finishReason: response.finishReason } : {},
+    ...response.rawFinishReason !== void 0 ? { rawFinishReason: response.rawFinishReason } : {},
+    ...response.responseId !== void 0 ? { responseId: response.responseId } : {},
+    ...response.responseModel !== void 0 ? { responseModel: response.responseModel } : {}
+  };
+  const format = record.request?.responseFormat?.type;
+  if (format !== "json_object" && format !== "json_schema") return output;
+  if (output.content === "" && output.tool_calls) return output;
+  try {
+    return { ...output, json: JSON.parse(output.content) };
+  } catch {
+    throw new AxlError(
+      "INVALID_MODEL_CALL_JSON",
+      `model call ${record.id} did not return valid JSON`
+    );
+  }
+}
+function providerOutput(response, record) {
+  return modelCallOutput(
+    {
+      content: response.content,
+      ...response.thinking_content !== void 0 ? { thinking: response.thinking_content } : {},
+      ...response.tool_calls !== void 0 ? { tool_calls: response.tool_calls } : {},
+      ...response.finishReason !== void 0 ? { finishReason: response.finishReason } : {},
+      ...response.rawFinishReason !== void 0 ? { rawFinishReason: response.rawFinishReason } : {},
+      ...response.responseId !== void 0 ? { responseId: response.responseId } : {},
+      ...response.responseModel !== void 0 ? { responseModel: response.responseModel } : {}
+    },
+    record
+  );
+}
+function recordedModelCall() {
+  return async (input) => {
+    const record = readInput(input);
+    if (record.captured.redacted)
+      throw new AxlError("RECORDED_OUTPUT_UNAVAILABLE", `model call ${record.id} was redacted`);
+    if (record.outcome.status !== "ok" || !("response" in record.outcome))
+      throw new AxlError(
+        "RECORDED_OUTPUT_UNAVAILABLE",
+        `model call ${record.id} has no complete successful output (${record.outcome.status})`
+      );
+    return { output: modelCallOutput(record.outcome.response, record) };
+  };
+}
+function replayRequest(record) {
+  const request = record.request;
+  if (!request || !record.captured.replayable || record.captured.continuityCaptured !== true || record.captured.redacted || record.captured.truncated || record.captured.omitted.length > 0 || (request.providerOptionKeys?.length ?? 0) > 0)
+    throw new AxlError(
+      "MODEL_CALL_NOT_REPLAYABLE",
+      `model call ${record.id} does not contain an exact replayable request`
+    );
+  return request;
+}
+function chatOptions(request, model) {
+  if (!request.options.model)
+    throw new AxlError("INVALID_MODEL_CALL_INPUT", "captured request model is empty");
+  const effort = request.options.effort;
+  if (effort !== void 0 && !["none", "low", "medium", "high", "xhigh", "max"].includes(effort))
+    throw new AxlError("INVALID_MODEL_CALL_INPUT", `unsupported captured effort ${effort}`);
+  const toolChoice = request.options.toolChoice;
+  if (toolChoice !== void 0 && toolChoice !== "auto" && toolChoice !== "none" && toolChoice !== "required" && !(toolChoice !== null && typeof toolChoice === "object" && !Array.isArray(toolChoice) && toolChoice.type === "function" && Object.keys(toolChoice).every((key) => key === "type" || key === "function") && toolChoice.function !== null && typeof toolChoice.function === "object" && !Array.isArray(toolChoice.function) && typeof toolChoice.function.name === "string" && Object.keys(toolChoice.function).every((key) => key === "name")))
+    throw new AxlError("INVALID_MODEL_CALL_INPUT", "unsupported captured tool choice");
+  return {
+    model,
+    ...request.options.temperature !== void 0 ? { temperature: request.options.temperature } : {},
+    ...request.options.maxTokens !== void 0 ? { maxTokens: request.options.maxTokens } : {},
+    ...request.options.thinkingBudget !== void 0 ? { thinkingBudget: request.options.thinkingBudget } : {},
+    ...request.options.includeThoughts !== void 0 ? { includeThoughts: request.options.includeThoughts } : {},
+    ...request.options.stop !== void 0 ? { stop: [...request.options.stop] } : {},
+    ...request.options.promptCache !== void 0 ? { promptCache: request.options.promptCache } : {},
+    ...effort !== void 0 ? { effort } : {},
+    ...toolChoice !== void 0 ? { toolChoice } : {},
+    ...request.tools ? {
+      tools: request.tools.map((tool) => ({
+        type: "function",
+        function: {
+          name: tool.name,
+          description: tool.description ?? "",
+          parameters: tool.parameters,
+          ...tool.strict !== void 0 ? { strict: tool.strict } : {}
+        }
+      }))
+    } : {},
+    ...request.responseFormat ? { responseFormat: request.responseFormat } : {}
+  };
+}
+function chatMessages(request) {
+  return request.messages.map((message) => {
+    if (!["system", "user", "assistant", "tool"].includes(message.role))
+      throw new AxlError("MODEL_CALL_NOT_REPLAYABLE", "captured message cannot be reconstructed");
+    let content;
+    if (message.input) {
+      const text = message.content ?? "";
+      let offset = 0;
+      const parts = [];
+      for (const part of message.input.parts) {
+        if (part.type !== "text" || offset + part.characters > text.length)
+          throw new AxlError(
+            "MODEL_CALL_NOT_REPLAYABLE",
+            "captured content parts cannot be reconstructed"
+          );
+        parts.push({ type: "text", text: text.slice(offset, offset + part.characters) });
+        offset += part.characters;
+      }
+      if (offset !== text.length)
+        throw new AxlError(
+          "MODEL_CALL_NOT_REPLAYABLE",
+          "captured text lengths do not match content"
+        );
+      content = parts;
+    } else if (message.content !== null) {
+      content = message.content;
+    } else {
+      throw new AxlError("MODEL_CALL_NOT_REPLAYABLE", "captured message content is absent");
+    }
+    return {
+      role: message.role,
+      content,
+      ...message.name !== void 0 ? { name: message.name } : {},
+      ...message.agent !== void 0 ? { agent: message.agent } : {},
+      ...message.origin !== void 0 ? { origin: message.origin } : {},
+      ...message.tool_calls !== void 0 ? { tool_calls: message.tool_calls } : {},
+      ...message.tool_call_id !== void 0 ? { tool_call_id: message.tool_call_id } : {},
+      ...message.providerMetadata !== void 0 ? { providerMetadata: message.providerMetadata } : {}
+    };
+  });
+}
+function replayModelCall(options = {}) {
+  return async (input, runtime) => {
+    const record = readInput(input);
+    const request = replayRequest(record);
+    const messages = chatMessages(request);
+    const capturedOptions = chatOptions(request, request.options.model);
+    if (!runtime)
+      throw new AxlError("MODEL_CALL_RUNTIME_REQUIRED", "replay requires an AxlRuntime");
+    const uri = options.model ?? `${record.provider}:${request.options.model}`;
+    if (!uri.includes(":"))
+      throw new AxlError("MODEL_CALL_MODEL_UNRESOLVED", "model override must use provider:model");
+    let resolved;
+    try {
+      resolved = runtime.resolveProvider(uri);
+    } catch {
+      throw new AxlError(
+        "MODEL_CALL_MODEL_UNRESOLVED",
+        `cannot resolve ${uri}; supply an explicit provider:model override`
+      );
+    }
+    if (!options.model && resolved.provider.name !== record.provider)
+      throw new AxlError(
+        "MODEL_CALL_MODEL_UNRESOLVED",
+        `resolved ${uri} to a different provider; supply an explicit override`
+      );
+    const callOptions = {
+      ...capturedOptions,
+      model: options.model ? resolved.model : request.options.model
+    };
+    if (record.kind === "chat") {
+      const response = await resolved.provider.chat(messages, callOptions);
+      return { output: providerOutput(response, record) };
+    }
+    let content = "";
+    let thinking = "";
+    const calls = /* @__PURE__ */ new Map();
+    let done;
+    for await (const chunk of resolved.provider.stream(messages, callOptions)) {
+      if (chunk.type === "text_delta") content += chunk.content;
+      else if (chunk.type === "thinking_delta") thinking += chunk.content;
+      else if (chunk.type === "tool_call_delta") {
+        const call = calls.get(chunk.id) ?? { id: chunk.id, name: "", arguments: "" };
+        if (chunk.name) call.name = chunk.name;
+        if (chunk.arguments) call.arguments += chunk.arguments;
+        calls.set(chunk.id, call);
+      } else if (chunk.type === "done") {
+        done = chunk;
+        break;
+      }
+    }
+    if (!done)
+      throw new AxlError("MODEL_CALL_STREAM_INCOMPLETE", "replayed stream ended without done");
+    return {
+      output: providerOutput(
+        {
+          content,
+          ...thinking ? { thinking_content: thinking } : {},
+          ...calls.size ? {
+            tool_calls: [...calls.values()].map((call) => ({
+              id: call.id,
+              type: "function",
+              function: { name: call.name, arguments: call.arguments }
+            }))
+          } : {},
+          ...done.finishReason !== void 0 ? { finishReason: done.finishReason } : {},
+          ...done.rawFinishReason !== void 0 ? { rawFinishReason: done.rawFinishReason } : {},
+          ...done.responseId !== void 0 ? { responseId: done.responseId } : {},
+          ...done.responseModel !== void 0 ? { responseModel: done.responseModel } : {}
+        },
+        record
+      )
+    };
+  };
+}
+
+// src/llm-scorer.ts
+import { z as z2 } from "zod";
 import { extractJson, zodToJsonSchema } from "@axlsdk/axl";
 function isZodIssueLike(value) {
   if (!value || typeof value !== "object") return false;
@@ -125,7 +526,7 @@ function isZodIssueLike(value) {
   return Array.isArray(issue.path) && issue.path.every((segment) => typeof segment === "string" || typeof segment === "number") && typeof issue.message === "string";
 }
 function llmScorer(config) {
-  const schema = config.schema ?? z.object({ score: z.number().min(0).max(1), reasoning: z.string() });
+  const schema = config.schema ?? z2.object({ score: z2.number().min(0).max(1), reasoning: z2.string() });
   const schemaJson = JSON.stringify(zodToJsonSchema(schema), null, 2);
   return {
     name: config.name,
@@ -217,11 +618,15 @@ export {
   evaluateScorerTolerance,
   isBudgetStopped,
   llmScorer,
+  loadModelCalls,
+  modelCallDataset,
   normalizeScorerResult,
   pairedBootstrapCI,
   parseRequestRecords,
   readAccounting,
+  recordedModelCall,
   refusedWork,
+  replayModelCall,
   rescore,
   resolveCaptureLimits,
   runEval,