@hue-run/sdk 0.9.0 → 0.11.0

package/dist/evals/scorer-publication.js

@@ -30,6 +30,47 @@ const worldOutcomeMetrics = [
     "process_constraints",
     "task_success",
 ].map((name) => ({ name, type: "boolean" }));
+/** `hue.conversion_outcome.v2`: v1's rubric with the destination reported as `recipient`,
+ * `thread` and `subject`. */
+const conversionOutcomeV2Metrics = [
+    "completed_run",
+    "saved_draft",
+    "correct_destination",
+    "recipient",
+    "thread",
+    "subject",
+    "content",
+    "unrelated_preserved",
+    "process_constraints",
+    "task_success",
+].map((name) => ({ name, type: "boolean" }));
+/** `hue.outcome_assertions.v2`'s counts; `.v3` adds the judge counts. */
+const outcomeAssertionMetrics = [
+    { name: "task_success", type: "boolean" },
+    ...["assertions_passed", "assertions_failed", "advisory_failed", "agent_mistakes"].map((name) => ({ name, type: "number", min: 0 })),
+];
+const outcomeAssertionV3Metrics = [
+    ...outcomeAssertionMetrics,
+    ...["judges_passed", "judges_failed", "judges_advisory"].map((name) => ({
+        name,
+        type: "number",
+        min: 0,
+    })),
+];
+/** The judge pin a `hue.outcome_assertions.v3` version carries; every field is required. */
+const outcomeJudgeConfig = z.strictObject({
+    model: z.string().min(1).max(200),
+    provider: z.string().min(1).max(64),
+    template: z.string().regex(/^[a-f0-9]{64}$/),
+    samples: z.number().int().min(1).max(9),
+    temperature: z.number().min(0).max(2),
+    maxOutputTokens: z.number().int().min(1).max(32_000),
+    timeoutMs: z.number().int().min(1_000).max(300_000),
+});
+/** A Hue-executed entry whose metrics are fixed by the entry itself. */
+const fixedMetrics = (fixed) => metrics
+    .default(fixed)
+    .refine((value) => digest(value) === digest(fixed), "World outcome metrics are fixed by the pinned entry");
 const jsonValue = z.custom((value) => {
     try {
         json(value);
@@ -85,9 +126,23 @@ const sdkScorerPublication = z.union([
     z.strictObject({
         kind: z.literal("world_outcome"),
         entry: z.literal("hue.conversion_outcome.v1"),
-        metrics: metrics
-            .default(worldOutcomeMetrics)
-            .refine((value) => digest(value) === digest(worldOutcomeMetrics), "World outcome metrics are fixed by the pinned entry"),
+        metrics: fixedMetrics(worldOutcomeMetrics),
+    }),
+    z.strictObject({
+        kind: z.literal("world_outcome"),
+        entry: z.literal("hue.conversion_outcome.v2"),
+        metrics: fixedMetrics(conversionOutcomeV2Metrics),
+    }),
+    z.strictObject({
+        kind: z.literal("world_outcome"),
+        entry: z.literal("hue.outcome_assertions.v2"),
+        metrics: fixedMetrics(outcomeAssertionMetrics),
+    }),
+    z.strictObject({
+        kind: z.literal("world_outcome"),
+        entry: z.literal("hue.outcome_assertions.v3"),
+        metrics: fixedMetrics(outcomeAssertionV3Metrics),
+        config: z.strictObject({ judge: outcomeJudgeConfig }),
     }),
     z.strictObject({ kind: z.literal("manual"), metrics }),
     z.strictObject({ kind: z.literal("llm_judge"), config: judgeConfig, metrics }),

package/dist/evals/simulation.d.ts

@@ -4,8 +4,8 @@ import type { EnvironmentTool } from "../environment/tools.js";
 import type { EnvironmentIdentity, PublishableEnvironmentDefinition, WorldHandoff } from "../environment/types.js";
 import { type EvaluationClient } from "./client.js";
 import type { ActualAgentManifestInputV2, AttemptConnectionBundleV2, RequestedAttemptProviderV2 } from "./attempt.js";
-import { type RunnerReport } from "./runner.js";
-import type { ExperimentCase, Identity, JsonValue, LocalScorer, ScorerDefinition, SimulationMcpCapability } from "./types.js";
+import { type RunnerReport, type TelemetryNotAccepted } from "./runner.js";
+import type { ExperimentCase, Identity, JsonValue, LocalFile, LocalScorer, ScorerDefinition, SimulationMcpCapability, TargetResult } from "./types.js";
 /** Repository-authored scorer identity and its public definition or local binding. */
 export interface RepositorySimulationScorer extends Identity {
     /** Scorer definition to publish, or a local callback bound to its source digest. */
@@ -121,6 +121,12 @@ export interface SimulationTargetContext {
     /** Credential-bearing provider connections for this callback only. Hue never
      * checkpoints, logs or adds this response to parity digests. */
     connectionBundle?: AttemptConnectionBundleV2;
+    /** Verified copies of the case's agent-visible input files (`source`, templates,
+     * references, originals); evaluator-only files are withheld. Empty when the case has none. */
+    files: LocalFile[];
+    /** Private directory for this case, removed after it; return generated files with
+     * `withFiles`. */
+    outputDirectory: string;
     /** Cancellation is cooperative: pass this signal into the real agent/provider call. */
     signal?: AbortSignal;
 }
@@ -152,6 +158,12 @@ export interface RunSimulationOptions {
         /** Bounded explanation for the omission. */
         reason: string;
     };
+    /** What a case does when required telemetry is not accepted; see
+     * `traceNotAccepted` of {@link runExperiment}. Defaults to `"stop"`. */
+    traceNotAccepted?: "stop" | "fail_case";
+    /** Called as each case is failed for its telemetry; see `onTelemetryNotAccepted` of
+     * {@link runExperiment}. */
+    onTelemetryNotAccepted?(entry: TelemetryNotAccepted): void | Promise<void>;
     /** Local scorer callbacks bound by their declared source digests. */
     localScorers?: LocalScorer[];
     /** Cases in flight, 1–64; defaults to 1. */
@@ -190,8 +202,9 @@ export interface RunSimulationOptions {
         /** Selected MCP surface. */
         surfaceKey: "google.gmail/mcp" | "slack/mcp";
     };
-    /** Invokes the existing local agent exactly once for this attempt. */
-    target(inputs: JsonValue, context: SimulationTargetContext): JsonValue | undefined | Promise<JsonValue | undefined>;
+    /** Invokes the existing local agent exactly once for this attempt; return the output, or
+     * `withFiles(output, files)` when it generated files. */
+    target(inputs: JsonValue, context: SimulationTargetContext): JsonValue | TargetResult | undefined | Promise<JsonValue | TargetResult | undefined>;
     /** Receives bounded nonsecret lifecycle progress. */
     onProgress?(event: SimulationProgress): void | Promise<void>;
 }

package/dist/evals/simulation.js

@@ -430,6 +430,12 @@ export async function runSimulation(options) {
             checkpointDirectory: join(store.directory, `experiment-${experimentId}`),
             persistResultContent: options.persistResultContent,
             traceEvidence: options.traceEvidence,
+            ...(options.traceNotAccepted ? { traceNotAccepted: options.traceNotAccepted } : {}),
+            ...(options.onTelemetryNotAccepted
+                ? {
+                    onTelemetryNotAccepted: (entry) => options.onTelemetryNotAccepted(entry),
+                }
+                : {}),
             environmentEvidence: "required",
             scorers: bindings,
             concurrency: options.concurrency,
@@ -474,6 +480,8 @@ export async function runSimulation(options) {
                     ...(targetContext.connectionBundle
                         ? { connectionBundle: structuredClone(targetContext.connectionBundle) }
                         : {}),
+                    files: structuredClone(targetContext.files),
+                    outputDirectory: targetContext.outputDirectory,
                     signal: targetContext.signal,
                 }),
             }),

package/dist/evals/types.d.ts

@@ -202,6 +202,33 @@ export type ScorerDefinition = {
     entry: "hue.conversion_outcome.v1";
     /** The seven fixed boolean metrics defined by the entry. */
     metrics: MetricDefinition[];
+} | {
+    /** Scored inside Hue using immutable world evidence; the local runner defers it. */
+    kind: "world_outcome";
+    /** v1's rubric with the destination reported as `recipient`, `thread` and `subject`. */
+    entry: "hue.conversion_outcome.v2";
+    /** The ten fixed boolean metrics defined by the entry. */
+    metrics: MetricDefinition[];
+} | {
+    /** A case's reviewed outcome assertions, scored inside Hue; the local runner defers it. */
+    kind: "world_outcome";
+    /** Pinned Hue-executed assertions evaluator. */
+    entry: "hue.outcome_assertions.v2";
+    /** `task_success` and the fixed assertion counts defined by the entry. */
+    metrics: MetricDefinition[];
+} | {
+    /** The v2 assertions plus the case's calibrated judges, scored inside Hue; the local runner
+     * defers it. */
+    kind: "world_outcome";
+    /** Pinned Hue-executed assertions-and-judges evaluator. */
+    entry: "hue.outcome_assertions.v3";
+    /** `task_success`, the assertion counts and the judge counts defined by the entry. */
+    metrics: MetricDefinition[];
+    /** The judge pin: model, template and sampling policy belong to the version. */
+    config: {
+        /** The judge every case scored by this version uses. */
+        judge: OutcomeJudgeConfig;
+    };
 } | {
     /** Scored by a person in Hue; the local runner defers it. */
     kind: "manual";
@@ -215,6 +242,23 @@ export type ScorerDefinition = {
     /** Metrics the judge reports. */
     metrics: MetricDefinition[];
 };
+/** The judge a `hue.outcome_assertions.v3` version pins. */
+export interface OutcomeJudgeConfig {
+    /** A provider-prefixed model id, such as `anthropic/claude-fable-5.1`. */
+    model: string;
+    /** The model's provider. */
+    provider: string;
+    /** SHA-256 hex digest of the judge template: its fixed instructions and evidence rendering. */
+    template: string;
+    /** Answers sampled per rubric item, 1–9; a majority decides. */
+    samples: number;
+    /** Sampling temperature, 0–2. */
+    temperature: number;
+    /** Output token limit per answer. */
+    maxOutputTokens: number;
+    /** Per-answer timeout in milliseconds. */
+    timeoutMs: number;
+}
 /** Configuration of a hosted judge scorer. */
 export interface JudgeConfig {
     /** Judge model identifier. */
@@ -264,6 +308,10 @@ export interface StoredResult extends ResultSummary {
     error: TypedError | null;
     /** Source digest of the local scorer that produced it, when applicable. */
     sourceDigest: string | null;
+    /** True only for a `skipped` result Hue's own outcome scoring recorded because the evaluator
+     * does not apply to the case (it has no outcome criteria or conversion rubric to grade); a
+     * submitted result never reads true. Absent from servers that predate it. */
+    notApplicable?: boolean;
 }
 /** A hosted judge job and its charge accounting. */
 export interface JudgeJob {

package/dist/evals/verdicts.d.ts

@@ -16,6 +16,10 @@ export interface WaitForResultsOptions {
     pollIntervalMillis?: number;
     /** Stops waiting early; the partial state is returned with `complete: false`. */
     signal?: AbortSignal;
+    /** Pinned versions of Hue's own judges (definition kind `world_judge`), whose results Hue can
+     * mark advisory. Any evaluator can write the same evidence, so no other result is advisory.
+     * {@link collectExperimentVerdicts} reads them from the experiment. */
+    judgeScorerVersionIds?: string[];
 }
 /** One stored result joined to its evaluation item. */
 export interface VerdictResult {
@@ -35,6 +39,15 @@ export interface VerdictResult {
     explanation: string | null;
     /** Scorer failure for `state: "error"`, otherwise `null`. */
     error: TypedError | null;
+    /** Whether Hue recorded that the evaluator does not apply to the case; such a result counts as
+     * neither a pass nor a failure. Absent means false. */
+    notApplicable?: boolean;
+    /** What a not-applicable evaluator needs that the case lacks, such as `outcome_criteria`. */
+    requires?: string;
+    /** Whether Hue recorded the result as advisory, as it does for every judge today: a result of a
+     * pinned Hue judge whose evidence says `advisory: true`, not an error, and whose metrics carry
+     * no `passed`. It is shown with its verdict but never decides a case. Absent means false. */
+    advisory?: boolean;
 }
 /** Outcome of {@link waitForResults}. */
 export interface VerdictResults {
@@ -65,10 +78,18 @@ export interface CaseVerdict {
     externalKey: string;
     /** Subject scored for the case, or `null` before completion. */
     subjectId: string | null;
-    /** `passed` and `failed` are scored verdicts; `error`, `skipped` and `pending` carry no verdict. */
+    /** `passed` and `failed` are scored verdicts; `error`, `skipped` and `pending` carry no verdict.
+     * Only the evaluators that apply to the case decide it; one where none applies is an error. */
     state: "passed" | "failed" | "error" | "skipped" | "pending";
-    /** True only for `state: "passed"`: every scored metric passed and no pin errored or is missing. */
+    /** True only for `state: "passed"`: every scored metric of an applicable pin passed and no
+     * applicable pin errored or is missing. */
     passed: boolean;
+    /** Pinned scorer versions Hue recorded as not applicable to the case; always set by
+     * {@link summarizeVerdicts}. */
+    notApplicable?: string[];
+    /** Pinned scorer versions whose results for the case are advisory: their metrics are listed
+     * but never decide the case; always set by {@link summarizeVerdicts}. */
+    advisory?: string[];
     /** Reported metrics across pinned scorers, in result order. */
     metrics: (Metric & {
         /** Scorer version that reported the metric. */
@@ -97,6 +118,9 @@ export interface VerdictSummary {
         skipped: number;
         /** Cases still missing a result. */
         pending: number;
+        /** Results of evaluators that do not apply to their case, a case can have several; always
+         * set by {@link summarizeVerdicts}. */
+        notApplicable?: number;
     };
 }
 /** Whether one reported metric counts as passing: an explicit `passed`, else a true boolean. */

package/dist/evals/verdicts.js

@@ -42,6 +42,7 @@ export async function waitForResults(client, options) {
         throw new RangeError("timeoutMillis must be ≥ 0");
     if (!Number.isInteger(interval) || interval < 250 || interval > 60_000)
         throw new RangeError("pollIntervalMillis must be 250–60000");
+    const judges = new Set(options.judgeScorerVersionIds ?? []);
     const wanted = options.subjectIds ? new Set(options.subjectIds) : undefined;
     const pins = [...new Set(options.scorerVersionIds)];
     const deadline = Date.now() + timeout;
@@ -66,6 +67,16 @@ export async function waitForResults(client, options) {
                     if (!id)
                         continue;
                     const stored = await client.getResult(id);
+                    // Only Hue's own flag on a skipped result marks it not applicable; the evidence alone
+                    // never does, and a scored or errored result keeps its verdict whatever it carries.
+                    const notApplicable = stored.state === "skipped" && stored.notApplicable === true;
+                    const evidence = stored.evidence;
+                    // Any evaluator can write `advisory: true` in its evidence; only Hue's own judges'
+                    // results are advisory, never as an error or with a metric that says it passed.
+                    const advisory = judges.has(stored.scorerVersionId) &&
+                        stored.state !== "error" &&
+                        evidence?.advisory === true &&
+                        !(stored.state === "scored" && stored.metrics.some((metric) => "passed" in metric));
                     results.push({
                         id: stored.id,
                         itemId: stored.itemId,
@@ -75,6 +86,11 @@ export async function waitForResults(client, options) {
                         metrics: stored.state === "scored" ? stored.metrics : [],
                         explanation: stored.explanation ?? null,
                         error: stored.error ?? null,
+                        notApplicable,
+                        advisory,
+                        ...(notApplicable && typeof evidence?.requires === "string"
+                            ? { requires: evidence.requires }
+                            : {}),
                     });
                 }
             return { complete, items, results };
@@ -105,11 +121,23 @@ export function summarizeVerdicts(results, options) {
         const subjectId = subjectByCase.get(item.id) ?? null;
         const own = subjectId ? results.results.filter((result) => result.subjectId === subjectId) : [];
         const metrics = own.flatMap((result) => result.metrics.map((metric) => ({ ...metric, scorerVersionId: result.scorerVersionId })));
-        const errors = own.filter((result) => result.state === "error");
-        const scored = own.filter((result) => result.state === "scored");
+        // An evaluator that does not apply to the case neither passes nor fails it.
+        const skippedAsInapplicable = (result) => result.state === "skipped" && result.notApplicable === true;
+        const isAdvisory = (result) => result.advisory === true &&
+            result.state !== "error" &&
+            result.metrics.every((metric) => metric.passed === undefined);
+        const inapplicable = own.filter(skippedAsInapplicable);
+        const applicable = own.filter((result) => !skippedAsInapplicable(result));
+        // An advisory result, such as a judge's, is shown but never decides the case, as in Hue.
+        const advisory = applicable.filter(isAdvisory);
+        const deciding = applicable.filter((result) => !isAdvisory(result));
+        const errors = deciding.filter((result) => result.state === "error");
+        const scored = deciding.filter((result) => result.state === "scored");
         const failing = scored.filter((result) => !result.metrics.every(metricPassed));
         const missing = [...pins].some((pin) => !own.some((result) => result.scorerVersionId === pin));
-        const state = errors.length
+        const noneApplies = !missing && own.length > 0 && !applicable.length;
+        const noneDecides = !missing && applicable.length > 0 && !deciding.length;
+        const state = errors.length || noneApplies || noneDecides
             ? "error"
             : failing.length
                 ? "failed"
@@ -118,23 +146,53 @@ export function summarizeVerdicts(results, options) {
                     : scored.length
                         ? "passed"
                         : "skipped";
+        const requires = [
+            ...new Set(inapplicable.flatMap((result) => (result.requires ? [result.requires] : []))),
+        ];
         return {
             caseId: item.id,
             externalKey: item.externalKey,
             subjectId,
             state,
             passed: state === "passed",
+            notApplicable: inapplicable.map((result) => result.scorerVersionId),
+            advisory: advisory.map((result) => result.scorerVersionId),
             metrics,
-            explanations: own
-                .filter((result) => result.state !== "scored" || failing.includes(result))
-                .map((result) => result.explanation)
-                .filter((explanation) => !!explanation),
+            explanations: [
+                ...(noneApplies
+                    ? [
+                        `No pinned evaluator applies to this case${requires.length ? ` (they need ${requires.join(" or ")})` : ""}; pin one that grades it`,
+                    ]
+                    : []),
+                ...(noneDecides
+                    ? [
+                        "Only advisory evaluators ran for this case, and they never decide it; pin one that grades it",
+                        ...advisory
+                            .filter((result) => result.state === "skipped")
+                            .map((result) => `An advisory evaluator skipped the case: ${result.explanation ?? "no reason given"}`),
+                    ]
+                    : []),
+                ...deciding
+                    .filter((result) => result.state !== "scored" || failing.includes(result))
+                    .map((result) => result.explanation)
+                    .filter((explanation) => !!explanation),
+            ],
             errors: errors.map((result) => result.error?.type ?? "ScorerError"),
         };
     });
-    const totals = { cases: cases.length, passed: 0, failed: 0, error: 0, skipped: 0, pending: 0 };
-    for (const item of cases)
+    const totals = {
+        cases: cases.length,
+        passed: 0,
+        failed: 0,
+        error: 0,
+        skipped: 0,
+        pending: 0,
+        notApplicable: 0,
+    };
+    for (const item of cases) {
         totals[item.state]++;
+        totals.notApplicable += item.notApplicable?.length ?? 0;
+    }
     return { cases, totals };
 }
 /** Diffs the current summary against a baseline by case key. */
@@ -169,10 +227,15 @@ export async function collectExperimentVerdicts(client, options) {
     const experiment = await client.getExperiment(options.experimentId);
     const scorerVersionIds = experiment.evaluation.scorerVersions.map((version) => version.id);
     const { experimentId: _experimentId, ...wait } = options;
+    // Hue's own judges, the only evaluators whose results can be advisory.
+    const judgeScorerVersionIds = experiment.evaluation.scorerVersions
+        .filter((version) => version.definition?.kind === "world_judge")
+        .map((version) => version.id);
     const results = await waitForResults(client, {
         ...wait,
         runId: experiment.evaluation.id,
         scorerVersionIds,
+        judgeScorerVersionIds,
     });
     const experimentItems = await allPages((after) => client.listExperimentItems(experiment.id, { after }));
     const linked = new Set(experimentItems.map((item) => item.execution?.subjectId).filter((id) => !!id));

package/dist/evals.d.ts

@@ -1,9 +1,10 @@
-export { createEvaluationClient, EvaluationClient, HueApiError } from "./evals/client.js";
+export { ArtifactSizeError, createEvaluationClient, EvaluationClient, HueApiError, } from "./evals/client.js";
 export type { EvaluationClientOptions } from "./evals/client.js";
 export { runExperiment, rescore, UncertainExecutionError, OutcomeSerializationError, TargetCancelledError, TargetOutcomeUncertainError, } from "./evals/runner.js";
-export type { RunExperimentOptions, RunExperimentTargetContext, RescoreOptions, RunnerReport, } from "./evals/runner.js";
+export type { RunExperimentOptions, RunExperimentTargetContext, RescoreOptions, RunnerReport, TelemetryIssueCount, TelemetryNotAccepted, } from "./evals/runner.js";
 export { TargetResult, withFiles } from "./evals/types.js";
-export { outputContentTypes, outputFileLimits, OutputFileError, safeFilename, targetFileRoles, } from "./evals/files.js";
+export { CaseFileError, outputContentTypes, outputFileLimits, OutputFileError, safeFilename, targetFileRoles, } from "./evals/files.js";
+export type { CaseFileErrorCode } from "./evals/files.js";
 export { runSimulation } from "./evals/simulation.js";
 export type { RepositorySimulationCase, RepositorySimulationScorer, RunSimulationOptions, SimulationDefinition, SimulationProgress, SimulationReport, SimulationScenario, SimulationTargetContext, } from "./evals/simulation.js";
 export { actualAgentManifestV2, agentManifestDigestV2, attemptBaselineV2, attemptBindingRead, attemptConnectionBundleV2, attemptIdentityV2, dependencyManifestV2, dependencyProviderV2, expectedAgentManifestV2, executionManifestDigestV2, parityEvidenceV2, preflightFindingV2, preflightReportV2, prepareAttemptInputV2, projectMcpConnectionV2, secretFreeBindingV2, surfaceBindingV2, } from "./evals/attempt.js";

package/dist/evals.js

@@ -1,7 +1,7 @@
-export { createEvaluationClient, EvaluationClient, HueApiError } from "./evals/client.js";
+export { ArtifactSizeError, createEvaluationClient, EvaluationClient, HueApiError, } from "./evals/client.js";
 export { runExperiment, rescore, UncertainExecutionError, OutcomeSerializationError, TargetCancelledError, TargetOutcomeUncertainError, } from "./evals/runner.js";
 export { TargetResult, withFiles } from "./evals/types.js";
-export { outputContentTypes, outputFileLimits, OutputFileError, safeFilename, targetFileRoles, } from "./evals/files.js";
+export { CaseFileError, outputContentTypes, outputFileLimits, OutputFileError, safeFilename, targetFileRoles, } from "./evals/files.js";
 export { runSimulation } from "./evals/simulation.js";
 export { actualAgentManifestV2, agentManifestDigestV2, attemptBaselineV2, attemptBindingRead, attemptConnectionBundleV2, attemptIdentityV2, dependencyManifestV2, dependencyProviderV2, expectedAgentManifestV2, executionManifestDigestV2, parityEvidenceV2, preflightFindingV2, preflightReportV2, prepareAttemptInputV2, projectMcpConnectionV2, secretFreeBindingV2, surfaceBindingV2, } from "./evals/attempt.js";
 export { builtins, defineLocalScorer, scoreLocally } from "./evals/scorers.js";

package/dist/inline-files.d.ts

@@ -1,5 +1,9 @@
 /** Inline file content larger than this many UTF-8 bytes is exported as its digest instead. */
 export declare const INLINE_FILE_LIMIT: number;
+/** Message text one record inspects for inline files in all, in UTF-16 code units. */
+export declare const INLINE_FILE_TEXT_PER_RECORD: number;
+/** Whether an attribute is one of the recorded message attributes that can inline files. */
+export declare function isMessageKey(key: string): boolean;
 /**
  * Replaces inline files longer than {@link INLINE_FILE_LIMIT} in a recorded message attribute
  * with their SHA-256 and byte size, so a span that inlines a large file exports the file's

package/dist/inline-files.js

@@ -3,32 +3,64 @@ import { MAX_BODY_BYTES } from "./config.js";
 /** Inline file content larger than this many UTF-8 bytes is exported as its digest instead. */
 export const INLINE_FILE_LIMIT = 64 * 1024;
 const MAX_INLINE_FILE_TEXT = 8 * MAX_BODY_BYTES;
+/** Message text one record inspects for inline files in all, in UTF-16 code units. */
+export const INLINE_FILE_TEXT_PER_RECORD = 2 * MAX_INLINE_FILE_TEXT;
 /** Message attributes whose JSON can inline files: GenAI blob parts and AI SDK 6 file parts. */
 const messageKeys = new Set([
     "gen_ai.input.messages",
     "gen_ai.output.messages",
     "ai.prompt.messages",
 ]);
+/** Whether an attribute is one of the recorded message attributes that can inline files. */
+export function isMessageKey(key) {
+    return messageKeys.has(key);
+}
 /** Strict base64: alphabet characters only, padded to a multiple of four. */
 const base64 = /^[A-Za-z0-9+/]*={0,2}$/;
-const base64DataUrl = /^data:[^,]*;base64,/;
-function binaryMimeType(value) {
-    return (typeof value === "string" &&
-        !/^text\//i.test(value) &&
-        value.toLowerCase() !== "application/json");
+/** An RFC 2397 `data:` URL's header, up to the comma before its data. Its `;` parameters are read
+ * in code: a pattern repeated per parameter fails on a pathological count (Node's stack overflows,
+ * Bun stops matching). */
+const dataUrl = /^data:([^,]*),/;
+/** Whether a `data:` URL header's parameters, after its media type, include `base64`. */
+function base64Header(header) {
+    const semicolon = header.indexOf(";");
+    // One `;`-separated parameter is exactly `base64`, without splitting them all.
+    return semicolon !== -1 && `${header.slice(semicolon)};`.includes(";base64;");
+}
+const hexPair = /^[0-9A-Fa-f]{2}$/;
+/** RFC 2397 data without `;base64`: percent-escaped octets, other characters as UTF-8. Undefined
+ * when a `%` does not start an escape. */
+function percentDecoded(payload) {
+    // Escapes only shrink the text, so its UTF-8 length bounds the bytes.
+    const bytes = Buffer.alloc(Buffer.byteLength(payload, "utf8"));
+    let length = 0;
+    let start = 0;
+    for (let index = payload.indexOf("%"); index !== -1; index = payload.indexOf("%", start)) {
+        const hex = payload.slice(index + 1, index + 3);
+        if (!hexPair.test(hex))
+            return undefined;
+        length += bytes.write(payload.slice(start, index), length, "utf8");
+        bytes[length++] = Number.parseInt(hex, 16);
+        start = index + 3;
+    }
+    length += bytes.write(payload.slice(start), length, "utf8");
+    return bytes.subarray(0, length);
 }
 /**
- * The bytes an inline file part carries: base64 (plain or as a `data:` URL) is decoded, and
- * anything else, such as a text file's content, is taken as UTF-8.
+ * The bytes an inline file part carries, the file's own bytes whatever its media type: a `data:`
+ * URL decoded by its own encoding (base64 with `;base64`, percent-escapes otherwise), content in
+ * the base64 alphabet decoded, and anything else, such as a text file's own text, as UTF-8. A
+ * `data:` URL whose data does not decode is taken as UTF-8 too.
  */
-function fileBytes(content, mimeType) {
-    const prefix = base64DataUrl.exec(content)?.[0].length ?? 0;
-    const payload = content.slice(prefix);
-    return (prefix > 0 || binaryMimeType(mimeType)) &&
-        payload.length % 4 === 0 &&
-        base64.test(payload)
-        ? Buffer.from(payload, "base64")
-        : Buffer.from(content, "utf8");
+function fileBytes(content) {
+    const url = dataUrl.exec(content);
+    const payload = url ? content.slice(url[0].length) : content;
+    const decoded = url && !base64Header(url[1])
+        ? percentDecoded(payload)
+        : payload.length % 4 === 0 && base64.test(payload)
+            ? Buffer.from(payload, "base64")
+            : undefined;
+    return decoded ?? Buffer.from(content, "utf8");
 }
 /** The key holding a part's inline content: GenAI `blob` parts and AI SDK 6 `file` parts. */
 function contentKey(part) {
@@ -44,9 +76,8 @@ function hashNode(value, state, depth) {
     const part = value;
     const key = contentKey(part);
     const inline = key === undefined ? undefined : part[key];
-    const mimeType = part.mime_type ?? part.mediaType;
     if (key !== undefined && typeof inline === "string") {
-        const bytes = fileBytes(inline, mimeType);
+        const bytes = fileBytes(inline);
         if (bytes.byteLength > INLINE_FILE_LIMIT) {
             const { [key]: _omitted, ...rest } = part;
             state.changed = true;

package/dist/provider-tools.d.ts

@@ -1,14 +1,19 @@
 import type { HostedToolProvider } from "./types.js";
-/** One provider-executed tool call. `arguments` and `result` are content. */
+/** One provider-executed tool call. `arguments`, `result` and `errorText` are content. */
 export interface HostedToolCall {
     name: string;
     callId?: string;
+    /** 0-based position of the call's item in the response: the OpenAI `output` index or the
+     * Anthropic `content` block index. */
+    position: number;
     /** The provider's label (OpenAI `server_label`) or name (Anthropic `server_name`) for the MCP server. */
     server?: string;
     arguments?: unknown;
     result?: unknown;
     /** Low-cardinality failure marker for `error.type`; absent when the call succeeded. */
     errorType?: string;
+    /** The provider's own error text for a failed OpenAI MCP call, unscrubbed and unbounded. */
+    errorText?: string;
 }
 /** One `mcp_list_tools` result: the tools a hosted MCP server offered. */
 export interface HostedToolListing {
@@ -23,6 +28,12 @@ export interface HostedToolActivity {
     /** Items that looked like hosted calls but could not be read. */
     skipped: number;
 }
+/**
+ * A provider's error text as a failed call exports it under content capture: credentials
+ * scrubbed first, then at most 1,024 code points with `…` marking a cut, lone surrogates
+ * replaced and NUL removed. The Python SDK produces the same text.
+ */
+export declare function providerErrorDescription(value: string): string;
 /**
  * Reads the hosted tool calls out of a provider response: the `output` items of an OpenAI
  * Responses API response, or the `content` blocks of an Anthropic Messages API response. An array