@browserbasehq/orca 3.4.0-preview.0 → 3.5.0-preview.1

package/dist/esm/lib/v3/verifier/evidenceNormalization.d.ts

@@ -0,0 +1,7 @@
+import type { AgentStepFinishedEvent } from "../types/public/agentEvidenceEvents.js";
+import type { AgentEvidence } from "./types.js";
+export declare const REDACTED_INLINE_IMAGE = "[redacted inline image payload]";
+export declare function collectInlineImagePayloads(value: unknown, actionName?: string, out?: string[]): string[];
+export declare function redactInlineImagePayloads(value: unknown, actionName?: string): unknown;
+export declare function mergeAgentEvidence(...parts: Array<AgentEvidence | undefined>): AgentEvidence;
+export declare function buildAgentEvidenceFromStepFinished(event: AgentStepFinishedEvent): AgentEvidence;

package/dist/esm/lib/v3/verifier/evidenceNormalization.js

@@ -0,0 +1,93 @@
+export const REDACTED_INLINE_IMAGE = "[redacted inline image payload]";
+const INLINE_IMAGE_KEYS = new Set(["screenshotBase64"]);
+function shouldRedactBase64Key(key, actionName) {
+    return (INLINE_IMAGE_KEYS.has(key) ||
+        (actionName === "screenshot" && key === "base64"));
+}
+export function collectInlineImagePayloads(value, actionName, out = []) {
+    if (!value || typeof value !== "object")
+        return out;
+    if (Buffer.isBuffer(value))
+        return out;
+    if (Array.isArray(value)) {
+        for (const item of value) {
+            collectInlineImagePayloads(item, actionName, out);
+        }
+        return out;
+    }
+    for (const [key, nested] of Object.entries(value)) {
+        if (shouldRedactBase64Key(key, actionName) && typeof nested === "string") {
+            out.push(nested);
+            continue;
+        }
+        collectInlineImagePayloads(nested, actionName, out);
+    }
+    return out;
+}
+export function redactInlineImagePayloads(value, actionName) {
+    if (!value || typeof value !== "object")
+        return value;
+    if (Buffer.isBuffer(value))
+        return value;
+    if (Array.isArray(value)) {
+        return value.map((item) => redactInlineImagePayloads(item, actionName));
+    }
+    const out = {};
+    for (const [key, nested] of Object.entries(value)) {
+        out[key] =
+            shouldRedactBase64Key(key, actionName) && typeof nested === "string"
+                ? REDACTED_INLINE_IMAGE
+                : redactInlineImagePayloads(nested, actionName);
+    }
+    return out;
+}
+export function mergeAgentEvidence(...parts) {
+    return {
+        modalities: parts.flatMap((p) => p?.modalities ?? []),
+    };
+}
+export function buildAgentEvidenceFromStepFinished(event) {
+    const modalities = [];
+    if (event.reasoning) {
+        modalities.push({ type: "text", content: event.reasoning });
+    }
+    const result = event.toolOutput.result;
+    if (result === undefined || result === null) {
+        return { modalities };
+    }
+    if (typeof result === "string") {
+        modalities.push({ type: "text", content: result });
+    }
+    else if (typeof result === "number" ||
+        typeof result === "boolean" ||
+        typeof result === "bigint") {
+        modalities.push({ type: "text", content: String(result) });
+    }
+    else if (Buffer.isBuffer(result)) {
+        modalities.push({
+            type: "image",
+            bytes: result,
+            mediaType: "image/png",
+        });
+    }
+    else if (typeof result === "object") {
+        for (const imageBase64 of collectInlineImagePayloads(result, event.actionName)) {
+            try {
+                modalities.push({
+                    type: "image",
+                    bytes: Buffer.from(imageBase64, "base64"),
+                    mediaType: "image/png",
+                });
+            }
+            catch {
+                // Malformed base64; skip the image and keep the JSON modality.
+            }
+        }
+        modalities.push({
+            type: "json",
+            content: redactInlineImagePayloads(result, event.actionName),
+        });
+    }
+    return { modalities };
+}
+//# sourceMappingURL=evidenceNormalization.js.map

package/dist/esm/lib/v3/verifier/evidenceNormalization.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"evidenceNormalization.js","sourceRoot":"","sources":["../../../../../lib/v3/verifier/evidenceNormalization.ts"],"names":[],"mappings":"AAGA,MAAM,CAAC,MAAM,qBAAqB,GAAG,iCAAiC,CAAC;AAEvE,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC;AAExD,SAAS,qBAAqB,CAAC,GAAW,EAAE,UAAmB;IAC7D,OAAO,CACL,iBAAiB,CAAC,GAAG,CAAC,GAAG,CAAC;QAC1B,CAAC,UAAU,KAAK,YAAY,IAAI,GAAG,KAAK,QAAQ,CAAC,CAClD,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,0BAA0B,CACxC,KAAc,EACd,UAAmB,EACnB,MAAgB,EAAE;IAElB,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,GAAG,CAAC;IACpD,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,GAAG,CAAC;IAEvC,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACzB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,0BAA0B,CAAC,IAAI,EAAE,UAAU,EAAE,GAAG,CAAC,CAAC;QACpD,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;IAED,KAAK,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QAClD,IAAI,qBAAqB,CAAC,GAAG,EAAE,UAAU,CAAC,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;YACzE,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACjB,SAAS;QACX,CAAC;QACD,0BAA0B,CAAC,MAAM,EAAE,UAAU,EAAE,GAAG,CAAC,CAAC;IACtD,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,UAAU,yBAAyB,CACvC,KAAc,EACd,UAAmB;IAEnB,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IACtD,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAEzC,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,yBAAyB,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC,CAAC;IAC1E,CAAC;IAED,MAAM,GAAG,GAA4B,EAAE,CAAC;IACxC,KAAK,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QAClD,GAAG,CAAC,GAAG,CAAC;YACN,qBAAqB,CAAC,GAAG,EAAE,UAAU,CAAC,IAAI,OAAO,MAAM,KAAK,QAAQ;gBAClE,CAAC,CAAC,qBAAqB;gBACvB,CAAC,CAAC,yBAAyB,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;IACtD,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,UAAU,kBAAkB,CAChC,GAAG,KAAuC;IAE1C,OAAO;QACL,UAAU,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,UAAU,IAAI,EAAE,CAAC;KACtD,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,kCAAkC,CAChD,KAA6B;IAE7B,MAAM,UAAU,GAAgC,EAAE,CAAC;IACnD,IAAI,KAAK,CAAC,SAAS,EAAE,CAAC;QACpB,UAAU,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC;IAC9D,CAAC;IAED,MAAM,MAAM,GAAG,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC;IACvC,IAAI,MAAM,KAAK,SAAS,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;QAC5C,OAAO,EAAE,UAAU,EAAE,CAAC;IACxB,CAAC;IAED,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;QAC/B,UAAU,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;IACrD,CAAC;SAAM,IACL,OAAO,MAAM,KAAK,QAAQ;QAC1B,OAAO,MAAM,KAAK,SAAS;QAC3B,OAAO,MAAM,KAAK,QAAQ,EAC1B,CAAC;QACD,UAAU,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IAC7D,CAAC;SAAM,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;QACnC,UAAU,CAAC,IAAI,CAAC;YACd,IAAI,EAAE,OAAO;YACb,KAAK,EAAE,MAAM;YACb,SAAS,EAAE,WAAW;SACvB,CAAC,CAAC;IACL,CAAC;SAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;QACtC,KAAK,MAAM,WAAW,IAAI,0BAA0B,CAClD,MAAM,EACN,KAAK,CAAC,UAAU,CACjB,EAAE,CAAC;YACF,IAAI,CAAC;gBACH,UAAU,CAAC,IAAI,CAAC;oBACd,IAAI,EAAE,OAAO;oBACb,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,WAAW,EAAE,QAAQ,CAAC;oBACzC,SAAS,EAAE,WAAW;iBACvB,CAAC,CAAC;YACL,CAAC;YAAC,MAAM,CAAC;gBACP,+DAA+D;YACjE,CAAC;QACH,CAAC;QACD,UAAU,CAAC,IAAI,CAAC;YACd,IAAI,EAAE,MAAM;YACZ,OAAO,EAAE,yBAAyB,CAAC,MAAM,EAAE,KAAK,CAAC,UAAU,CAAC;SAC7D,CAAC,CAAC;IACL,CAAC;IAED,OAAO,EAAE,UAAU,EAAE,CAAC;AACxB,CAAC","sourcesContent":["import type { AgentStepFinishedEvent } from \"../types/public/agentEvidenceEvents.js\";\nimport type { AgentEvidence } from \"./types.js\";\n\nexport const REDACTED_INLINE_IMAGE = \"[redacted inline image payload]\";\n\nconst INLINE_IMAGE_KEYS = new Set([\"screenshotBase64\"]);\n\nfunction shouldRedactBase64Key(key: string, actionName?: string): boolean {\n  return (\n    INLINE_IMAGE_KEYS.has(key) ||\n    (actionName === \"screenshot\" && key === \"base64\")\n  );\n}\n\nexport function collectInlineImagePayloads(\n  value: unknown,\n  actionName?: string,\n  out: string[] = [],\n): string[] {\n  if (!value || typeof value !== \"object\") return out;\n  if (Buffer.isBuffer(value)) return out;\n\n  if (Array.isArray(value)) {\n    for (const item of value) {\n      collectInlineImagePayloads(item, actionName, out);\n    }\n    return out;\n  }\n\n  for (const [key, nested] of Object.entries(value)) {\n    if (shouldRedactBase64Key(key, actionName) && typeof nested === \"string\") {\n      out.push(nested);\n      continue;\n    }\n    collectInlineImagePayloads(nested, actionName, out);\n  }\n  return out;\n}\n\nexport function redactInlineImagePayloads(\n  value: unknown,\n  actionName?: string,\n): unknown {\n  if (!value || typeof value !== \"object\") return value;\n  if (Buffer.isBuffer(value)) return value;\n\n  if (Array.isArray(value)) {\n    return value.map((item) => redactInlineImagePayloads(item, actionName));\n  }\n\n  const out: Record<string, unknown> = {};\n  for (const [key, nested] of Object.entries(value)) {\n    out[key] =\n      shouldRedactBase64Key(key, actionName) && typeof nested === \"string\"\n        ? REDACTED_INLINE_IMAGE\n        : redactInlineImagePayloads(nested, actionName);\n  }\n  return out;\n}\n\nexport function mergeAgentEvidence(\n  ...parts: Array<AgentEvidence | undefined>\n): AgentEvidence {\n  return {\n    modalities: parts.flatMap((p) => p?.modalities ?? []),\n  };\n}\n\nexport function buildAgentEvidenceFromStepFinished(\n  event: AgentStepFinishedEvent,\n): AgentEvidence {\n  const modalities: AgentEvidence[\"modalities\"] = [];\n  if (event.reasoning) {\n    modalities.push({ type: \"text\", content: event.reasoning });\n  }\n\n  const result = event.toolOutput.result;\n  if (result === undefined || result === null) {\n    return { modalities };\n  }\n\n  if (typeof result === \"string\") {\n    modalities.push({ type: \"text\", content: result });\n  } else if (\n    typeof result === \"number\" ||\n    typeof result === \"boolean\" ||\n    typeof result === \"bigint\"\n  ) {\n    modalities.push({ type: \"text\", content: String(result) });\n  } else if (Buffer.isBuffer(result)) {\n    modalities.push({\n      type: \"image\",\n      bytes: result,\n      mediaType: \"image/png\",\n    });\n  } else if (typeof result === \"object\") {\n    for (const imageBase64 of collectInlineImagePayloads(\n      result,\n      event.actionName,\n    )) {\n      try {\n        modalities.push({\n          type: \"image\",\n          bytes: Buffer.from(imageBase64, \"base64\"),\n          mediaType: \"image/png\",\n        });\n      } catch {\n        // Malformed base64; skip the image and keep the JSON modality.\n      }\n    }\n    modalities.push({\n      type: \"json\",\n      content: redactInlineImagePayloads(result, event.actionName),\n    });\n  }\n\n  return { modalities };\n}\n"]}

package/dist/esm/lib/v3/verifier/index.d.ts

@@ -0,0 +1,6 @@
+/**
+ * Public re-exports for the verifier subsystem.
+ */
+export type { AgentEvidence, AgentEvidenceModality, CriterionScore, EvaluationResult, FirstPointOfFailure, ProbeEvidence, Rubric, RubricCriterion, TaskSpec, TaskValidity, ToolOutput, Trajectory, TrajectoryStatus, TrajectoryStep, TrajectoryUsage, Verifier, VerifierFinding, VerifierRawSteps, } from "./types.js";
+export { buildAgentEvidenceFromStepFinished, collectInlineImagePayloads, mergeAgentEvidence, redactInlineImagePayloads, REDACTED_INLINE_IMAGE, } from "./evidenceNormalization.js";
+export { loadTrajectoryFromDisk, nextResultFilename, normalizeRubric, shouldPersistTrajectory, writeTrajectoryDir, } from "./trajectory.js";

package/dist/esm/lib/v3/verifier/index.js

@@ -0,0 +1,3 @@
+export { buildAgentEvidenceFromStepFinished, collectInlineImagePayloads, mergeAgentEvidence, redactInlineImagePayloads, REDACTED_INLINE_IMAGE, } from "./evidenceNormalization.js";
+export { loadTrajectoryFromDisk, nextResultFilename, normalizeRubric, shouldPersistTrajectory, writeTrajectoryDir, } from "./trajectory.js";
+//# sourceMappingURL=index.js.map

package/dist/esm/lib/v3/verifier/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../../lib/v3/verifier/index.ts"],"names":[],"mappings":"AAuBA,OAAO,EACL,kCAAkC,EAClC,0BAA0B,EAC1B,kBAAkB,EAClB,yBAAyB,EACzB,qBAAqB,GACtB,MAAM,4BAA4B,CAAC;AACpC,OAAO,EACL,sBAAsB,EACtB,kBAAkB,EAClB,eAAe,EACf,uBAAuB,EACvB,kBAAkB,GACnB,MAAM,iBAAiB,CAAC","sourcesContent":["/**\n * Public re-exports for the verifier subsystem.\n */\nexport type {\n  AgentEvidence,\n  AgentEvidenceModality,\n  CriterionScore,\n  EvaluationResult,\n  FirstPointOfFailure,\n  ProbeEvidence,\n  Rubric,\n  RubricCriterion,\n  TaskSpec,\n  TaskValidity,\n  ToolOutput,\n  Trajectory,\n  TrajectoryStatus,\n  TrajectoryStep,\n  TrajectoryUsage,\n  Verifier,\n  VerifierFinding,\n  VerifierRawSteps,\n} from \"./types.js\";\nexport {\n  buildAgentEvidenceFromStepFinished,\n  collectInlineImagePayloads,\n  mergeAgentEvidence,\n  redactInlineImagePayloads,\n  REDACTED_INLINE_IMAGE,\n} from \"./evidenceNormalization.js\";\nexport {\n  loadTrajectoryFromDisk,\n  nextResultFilename,\n  normalizeRubric,\n  shouldPersistTrajectory,\n  writeTrajectoryDir,\n} from \"./trajectory.js\";\n"]}

package/dist/esm/lib/v3/verifier/trajectory.d.ts

@@ -0,0 +1,50 @@
+import type { Rubric, Trajectory } from "./types.js";
+/**
+ * Convert dataset or generated rubric JSON into the public Stagehand shape.
+ * Snake-case dataset fields are accepted here so serialized quirks do not leak
+ * into the canonical rubric type.
+ */
+export declare function normalizeRubric(rubric: unknown): Rubric | undefined;
+/**
+ * Hydrate a Trajectory from the on-disk directory layout written by
+ * TrajectoryRecorder.persist(). Used by the offline re-scoring CLI (`bench
+ * verify`) and by any consumer that wants to feed a saved trajectory back
+ * into V3Evaluator.verify() without running an agent.
+ *
+ * Reverses the recorder's serialization tweaks:
+ *   - `probeEvidence.screenshotPath` → read file into `probeEvidence.screenshot`.
+ *   - Image modalities in `agentEvidence.modalities` carry `imagePath` on
+ *     disk instead of raw Buffer; legacy `bytesBase64` fixtures are also
+ *     accepted.
+ *
+ * @param dir absolute or cwd-relative path to a `<run-id>/<task-id>/` directory.
+ */
+export declare function loadTrajectoryFromDisk(dir: string): Promise<Trajectory>;
+/**
+ * Build a `result*.json` filename for persisted evaluator output.
+ *
+ * Convention: the live run writes `result.json`; offline re-score attempts use
+ * a label-based name (e.g., `result_rescore-2026-05-11.json`) so they coexist
+ * without collisions and remain easy to diff.
+ */
+export declare function nextResultFilename(label?: string): string;
+/**
+ * Default persistence policy: explicit override, then env, then "on unless CI".
+ */
+export declare function shouldPersistTrajectory(override: boolean | undefined): boolean;
+/**
+ * Write the on-disk trajectory layout under `dir`:
+ *
+ *   <dir>/
+ *     ├── task_data.json
+ *     ├── trajectory.json    (screenshots referenced by path)
+ *     ├── screenshots/
+ *     │   ├── probe/<N>.png
+ *     │   └── agent/<N>[_M].png
+ *     ├── scores/            (empty; populated separately)
+ *     └── core.log
+ *
+ * Image bytes are externalized to PNG files; the in-memory Trajectory is left
+ * untouched so callers can keep using it after persistence.
+ */
+export declare function writeTrajectoryDir(dir: string, trajectory: Trajectory): Promise<void>;

package/dist/esm/lib/v3/verifier/trajectory.js

@@ -0,0 +1,273 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+import { redactInlineImagePayloads } from "./evidenceNormalization.js";
+/**
+ * Convert dataset or generated rubric JSON into the public Stagehand shape.
+ * Snake-case dataset fields are accepted here so serialized quirks do not leak
+ * into the canonical rubric type.
+ */
+export function normalizeRubric(rubric) {
+    if (rubric == null)
+        return undefined;
+    if (typeof rubric !== "object") {
+        throw new TypeError("Rubric must be an object");
+    }
+    const rawRubric = rubric;
+    if (!Array.isArray(rawRubric.items)) {
+        throw new TypeError("Rubric is missing an items array");
+    }
+    return {
+        items: rawRubric.items.map((item) => {
+            const criterion = normalizeRequiredString(item.criterion, "criterion");
+            const description = normalizeRequiredString(item.description, "description");
+            const maxPoints = normalizeMaxPoints(item);
+            if (typeof maxPoints !== "number" || !Number.isFinite(maxPoints)) {
+                throw new TypeError(`Rubric criterion "${criterion}" is missing a numeric maxPoints value`);
+            }
+            return {
+                criterion,
+                description,
+                maxPoints,
+                ...(typeof item.condition === "string" && {
+                    condition: item.condition,
+                }),
+            };
+        }),
+    };
+}
+function normalizeRequiredString(value, fieldName) {
+    if (typeof value === "string" && value.length) {
+        return value;
+    }
+    throw new TypeError(`Rubric criterion is missing a ${fieldName} value`);
+}
+function normalizeMaxPoints(item) {
+    return item.maxPoints ?? item.max_points;
+}
+function normalizeResultLabel(label) {
+    return (label ?? `rescore-${new Date().toISOString()}`).replace(/[^A-Za-z0-9._-]/g, "_");
+}
+// ─────────────────────────────────────────────────────────────────────────────
+// On-disk loader
+// ─────────────────────────────────────────────────────────────────────────────
+/**
+ * Hydrate a Trajectory from the on-disk directory layout written by
+ * TrajectoryRecorder.persist(). Used by the offline re-scoring CLI (`bench
+ * verify`) and by any consumer that wants to feed a saved trajectory back
+ * into V3Evaluator.verify() without running an agent.
+ *
+ * Reverses the recorder's serialization tweaks:
+ *   - `probeEvidence.screenshotPath` → read file into `probeEvidence.screenshot`.
+ *   - Image modalities in `agentEvidence.modalities` carry `imagePath` on
+ *     disk instead of raw Buffer; legacy `bytesBase64` fixtures are also
+ *     accepted.
+ *
+ * @param dir absolute or cwd-relative path to a `<run-id>/<task-id>/` directory.
+ */
+export async function loadTrajectoryFromDisk(dir) {
+    const fs = await import("node:fs/promises");
+    const path = await import("node:path");
+    const trajectoryDir = path.resolve(dir);
+    const trajectoryPath = path.join(trajectoryDir, "trajectory.json");
+    const raw = await fs.readFile(trajectoryPath, "utf8");
+    const parsed = JSON.parse(raw);
+    const resolveWithinTrajectoryDir = (candidate, fieldName = "screenshotPath") => {
+        const resolved = path.resolve(trajectoryDir, candidate);
+        const relative = path.relative(trajectoryDir, resolved);
+        const outside = relative === ".." ||
+            relative.startsWith(`..${path.sep}`) ||
+            path.isAbsolute(relative);
+        if (outside) {
+            throw new Error(`Trajectory ${fieldName} escapes trajectory directory: ${candidate}`);
+        }
+        return resolved;
+    };
+    const hydrateProbeScreenshot = async (probe) => {
+        if (probe?.screenshotPath && !probe.screenshot) {
+            const resolved = resolveWithinTrajectoryDir(probe.screenshotPath);
+            try {
+                probe.screenshot = await fs.readFile(resolved);
+            }
+            catch {
+                // Missing screenshot file: leave probe.screenshot unset. The verifier's
+                // evidence_insufficient path will handle it.
+            }
+        }
+    };
+    for (const step of parsed.steps) {
+        // Rehydrate tier-2 probe screenshot from its on-disk file reference.
+        await hydrateProbeScreenshot(step.probeEvidence);
+        // Decode image modalities from disk references back to Buffer.
+        if (step.agentEvidence?.modalities) {
+            const modalities = [];
+            for (const m of step.agentEvidence.modalities) {
+                // The on-disk shape carries imagePath/bytesBase64 instead of bytes,
+                // so we look through `unknown` rather than rely on the typed union.
+                const raw = m;
+                if (m.type === "image" && typeof raw.bytesBase64 === "string") {
+                    modalities.push({
+                        type: "image",
+                        bytes: Buffer.from(raw.bytesBase64, "base64"),
+                        mediaType: m.mediaType,
+                    });
+                    continue;
+                }
+                if (m.type === "image" && typeof raw.imagePath === "string") {
+                    const resolved = resolveWithinTrajectoryDir(raw.imagePath, "imagePath");
+                    try {
+                        modalities.push({
+                            type: "image",
+                            bytes: await fs.readFile(resolved),
+                            mediaType: m.mediaType,
+                        });
+                    }
+                    catch {
+                        // Missing agent image file: omit that image modality. The
+                        // verifier's evidence_insufficient path will handle missing bytes.
+                    }
+                    continue;
+                }
+                modalities.push(m);
+            }
+            step.agentEvidence.modalities = modalities;
+        }
+    }
+    await hydrateProbeScreenshot(parsed.finalObservation);
+    return parsed;
+}
+/**
+ * Build a `result*.json` filename for persisted evaluator output.
+ *
+ * Convention: the live run writes `result.json`; offline re-score attempts use
+ * a label-based name (e.g., `result_rescore-2026-05-11.json`) so they coexist
+ * without collisions and remain easy to diff.
+ */
+export function nextResultFilename(label) {
+    return `result_${normalizeResultLabel(label)}.json`;
+}
+/**
+ * Default persistence policy: explicit override, then env, then "on unless CI".
+ */
+export function shouldPersistTrajectory(override) {
+    if (override !== undefined)
+        return override;
+    const env = process.env.VERIFIER_PERSIST_TRAJECTORIES?.toLowerCase();
+    if (env === "1" || env === "true")
+        return true;
+    if (env === "0" || env === "false")
+        return false;
+    return !process.env.CI;
+}
+/**
+ * Write the on-disk trajectory layout under `dir`:
+ *
+ *   <dir>/
+ *     ├── task_data.json
+ *     ├── trajectory.json    (screenshots referenced by path)
+ *     ├── screenshots/
+ *     │   ├── probe/<N>.png
+ *     │   └── agent/<N>[_M].png
+ *     ├── scores/            (empty; populated separately)
+ *     └── core.log
+ *
+ * Image bytes are externalized to PNG files; the in-memory Trajectory is left
+ * untouched so callers can keep using it after persistence.
+ */
+export async function writeTrajectoryDir(dir, trajectory) {
+    await fs.mkdir(dir, { recursive: true });
+    await fs.mkdir(path.join(dir, "screenshots", "probe"), { recursive: true });
+    await fs.mkdir(path.join(dir, "screenshots", "agent"), { recursive: true });
+    const serializableSteps = [];
+    // A single post-turn probe is fanned across every step of a multi-tool turn,
+    // and a single agent screenshot is shared across every action a CUA provider
+    // chose from it, so the same Buffer is shared by reference. Dedupe by
+    // identity: write the PNG once and point every sharing step at the same file.
+    const probePathByBuffer = new Map();
+    const agentPathByBuffer = new Map();
+    for (const [i, step] of trajectory.steps.entries()) {
+        const probe = { ...step.probeEvidence };
+        if (probe.screenshot) {
+            let relPath = probePathByBuffer.get(probe.screenshot);
+            if (!relPath) {
+                relPath = `screenshots/probe/${i + 1}.png`;
+                await fs.writeFile(path.join(dir, relPath), probe.screenshot);
+                probePathByBuffer.set(probe.screenshot, relPath);
+            }
+            probe.screenshotPath = relPath;
+            delete probe.screenshot;
+        }
+        const imageModalities = step.agentEvidence.modalities.filter((m) => m.type === "image");
+        const multipleImages = imageModalities.length > 1;
+        let imageSeq = 0;
+        const modalities = [];
+        for (const m of step.agentEvidence.modalities) {
+            if (m.type !== "image") {
+                modalities.push(m.type === "json"
+                    ? {
+                        ...m,
+                        content: redactInlineImagePayloads(m.content, step.actionName),
+                    }
+                    : m);
+                continue;
+            }
+            let relPath = agentPathByBuffer.get(m.bytes);
+            if (!relPath) {
+                const suffix = multipleImages ? `_${imageSeq}` : "";
+                relPath = `screenshots/agent/${i + 1}${suffix}.png`;
+                await fs.writeFile(path.join(dir, relPath), m.bytes);
+                agentPathByBuffer.set(m.bytes, relPath);
+            }
+            modalities.push({
+                type: "image",
+                imagePath: relPath,
+                mediaType: m.mediaType,
+            });
+            imageSeq += 1;
+        }
+        serializableSteps.push({
+            ...step,
+            probeEvidence: probe,
+            agentEvidence: { modalities },
+            toolOutput: {
+                ...step.toolOutput,
+                result: redactInlineImagePayloads(step.toolOutput.result, step.actionName),
+            },
+        });
+    }
+    const finalObservation = trajectory.finalObservation === undefined
+        ? undefined
+        : { ...trajectory.finalObservation };
+    if (finalObservation?.screenshot) {
+        const relPath = "screenshots/probe/final.png";
+        await fs.writeFile(path.join(dir, relPath), finalObservation.screenshot);
+        finalObservation.screenshotPath = relPath;
+        delete finalObservation.screenshot;
+    }
+    // Image modalities carry imagePath instead of raw bytes on disk; cast
+    // through unknown rather than widen Trajectory's type contract.
+    const serialized = {
+        ...trajectory,
+        steps: serializableSteps,
+        ...(finalObservation ? { finalObservation } : {}),
+    };
+    await fs.writeFile(path.join(dir, "trajectory.json"), JSON.stringify(serialized, null, 2));
+    await fs.writeFile(path.join(dir, "task_data.json"), JSON.stringify({
+        task: trajectory.task,
+        status: trajectory.status,
+        finalAnswer: trajectory.finalAnswer ?? null,
+    }, null, 2));
+    await fs.mkdir(path.join(dir, "scores"), { recursive: true });
+    await fs.writeFile(path.join(dir, "core.log"), coreLog(trajectory));
+}
+function coreLog(trajectory) {
+    return (trajectory.steps
+        .map((step, i) => JSON.stringify({
+        step: i,
+        action: step.actionName,
+        url: step.probeEvidence.url ?? null,
+        ok: step.toolOutput.ok,
+        reasoning: step.reasoning || undefined,
+    }))
+        .join("\n") + "\n");
+}
+//# sourceMappingURL=trajectory.js.map

package/dist/esm/lib/v3/verifier/trajectory.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"trajectory.js","sourceRoot":"","sources":["../../../../../lib/v3/verifier/trajectory.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAClC,OAAO,IAAI,MAAM,WAAW,CAAC;AAQ7B,OAAO,EAAE,yBAAyB,EAAE,MAAM,4BAA4B,CAAC;AAkBvE;;;;GAIG;AACH,MAAM,UAAU,eAAe,CAAC,MAAe;IAC7C,IAAI,MAAM,IAAI,IAAI;QAAE,OAAO,SAAS,CAAC;IACrC,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;QAC/B,MAAM,IAAI,SAAS,CAAC,0BAA0B,CAAC,CAAC;IAClD,CAAC;IAED,MAAM,SAAS,GAAG,MAAmB,CAAC;IACtC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC;QACpC,MAAM,IAAI,SAAS,CAAC,kCAAkC,CAAC,CAAC;IAC1D,CAAC;IAED,OAAO;QACL,KAAK,EAAE,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;YAClC,MAAM,SAAS,GAAG,uBAAuB,CAAC,IAAI,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC;YACvE,MAAM,WAAW,GAAG,uBAAuB,CACzC,IAAI,CAAC,WAAW,EAChB,aAAa,CACd,CAAC;YACF,MAAM,SAAS,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC;YAE3C,IAAI,OAAO,SAAS,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;gBACjE,MAAM,IAAI,SAAS,CACjB,qBAAqB,SAAS,wCAAwC,CACvE,CAAC;YACJ,CAAC;YAED,OAAO;gBACL,SAAS;gBACT,WAAW;gBACX,SAAS;gBACT,GAAG,CAAC,OAAO,IAAI,CAAC,SAAS,KAAK,QAAQ,IAAI;oBACxC,SAAS,EAAE,IAAI,CAAC,SAAS;iBAC1B,CAAC;aACH,CAAC;QACJ,CAAC,CAAC;KACH,CAAC;AACJ,CAAC;AAED,SAAS,uBAAuB,CAAC,KAAc,EAAE,SAAiB;IAChE,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;QAC9C,OAAO,KAAK,CAAC;IACf,CAAC;IAED,MAAM,IAAI,SAAS,CAAC,iCAAiC,SAAS,QAAQ,CAAC,CAAC;AAC1E,CAAC;AAED,SAAS,kBAAkB,CAAC,IAAwB;IAClD,OAAO,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,UAAU,CAAC;AAC3C,CAAC;AAED,SAAS,oBAAoB,CAAC,KAAc;IAC1C,OAAO,CAAC,KAAK,IAAI,WAAW,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC,OAAO,CAC7D,kBAAkB,EAClB,GAAG,CACJ,CAAC;AACJ,CAAC;AAED,gFAAgF;AAChF,iBAAiB;AACjB,gFAAgF;AAEhF;;;;;;;;;;;;;GAaG;AACH,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAAC,GAAW;IACtD,MAAM,EAAE,GAAG,MAAM,MAAM,CAAC,kBAAkB,CAAC,CAAC;IAC5C,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,WAAW,CAAC,CAAC;IACvC,MAAM,aAAa,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAExC,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,iBAAiB,CAAC,CAAC;IACnE,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC,CAAC;IACtD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAsB5B,CAAC;IAEF,MAAM,0BAA0B,GAAG,CACjC,SAAiB,EACjB,SAAS,GAAG,gBAAgB,EACpB,EAAE;QACV,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,SAAS,CAAC,CAAC;QACxD,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,aAAa,EAAE,QAAQ,CAAC,CAAC;QACxD,MAAM,OAAO,GACX,QAAQ,KAAK,IAAI;YACjB,QAAQ,CAAC,UAAU,CAAC,KAAK,IAAI,CAAC,GAAG,EAAE,CAAC;YACpC,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;QAE5B,IAAI,OAAO,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CACb,cAAc,SAAS,kCAAkC,SAAS,EAAE,CACrE,CAAC;QACJ,CAAC;QAED,OAAO,QAAQ,CAAC;IAClB,CAAC,CAAC;IAEF,MAAM,sBAAsB,GAAG,KAAK,EAClC,KAAyC,EAC1B,EAAE;QACjB,IAAI,KAAK,EAAE,cAAc,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,CAAC;YAC/C,MAAM,QAAQ,GAAG,0BAA0B,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;YAClE,IAAI,CAAC;gBACH,KAAK,CAAC,UAAU,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;YACjD,CAAC;YAAC,MAAM,CAAC;gBACP,wEAAwE;gBACxE,6CAA6C;YAC/C,CAAC;QACH,CAAC;IACH,CAAC,CAAC;IAEF,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;QAChC,qEAAqE;QACrE,MAAM,sBAAsB,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAEjD,+DAA+D;QAC/D,IAAI,IAAI,CAAC,aAAa,EAAE,UAAU,EAAE,CAAC;YACnC,MAAM,UAAU,GAA4B,EAAE,CAAC;YAC/C,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,aAAa,CAAC,UAAU,EAAE,CAAC;gBAC9C,oEAAoE;gBACpE,oEAAoE;gBACpE,MAAM,GAAG,GAAG,CAGX,CAAC;gBACF,IAAI,CAAC,CAAC,IAAI,KAAK,OAAO,IAAI,OAAO,GAAG,CAAC,WAAW,KAAK,QAAQ,EAAE,CAAC;oBAC9D,UAAU,CAAC,IAAI,CAAC;wBACd,IAAI,EAAE,OAAgB;wBACtB,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,QAAQ,CAAC;wBAC7C,SAAS,EAAE,CAAC,CAAC,SAAS;qBACvB,CAAC,CAAC;oBACH,SAAS;gBACX,CAAC;gBACD,IAAI,CAAC,CAAC,IAAI,KAAK,OAAO,IAAI,OAAO,GAAG,CAAC,SAAS,KAAK,QAAQ,EAAE,CAAC;oBAC5D,MAAM,QAAQ,GAAG,0BAA0B,CACzC,GAAG,CAAC,SAAS,EACb,WAAW,CACZ,CAAC;oBACF,IAAI,CAAC;wBACH,UAAU,CAAC,IAAI,CAAC;4BACd,IAAI,EAAE,OAAgB;4BACtB,KAAK,EAAE,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC;4BAClC,SAAS,EAAE,CAAC,CAAC,SAAS;yBACvB,CAAC,CAAC;oBACL,CAAC;oBAAC,MAAM,CAAC;wBACP,0DAA0D;wBAC1D,mEAAmE;oBACrE,CAAC;oBACD,SAAS;gBACX,CAAC;gBACD,UAAU,CAAC,IAAI,CAAC,CAA0B,CAAC,CAAC;YAC9C,CAAC;YACD,IAAI,CAAC,aAAa,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7C,CAAC;IACH,CAAC;IAED,MAAM,sBAAsB,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC;IAEtD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,kBAAkB,CAAC,KAAc;IAC/C,OAAO,UAAU,oBAAoB,CAAC,KAAK,CAAC,OAAO,CAAC;AACtD,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,uBAAuB,CACrC,QAA6B;IAE7B,IAAI,QAAQ,KAAK,SAAS;QAAE,OAAO,QAAQ,CAAC;IAC5C,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,6BAA6B,EAAE,WAAW,EAAE,CAAC;IACrE,IAAI,GAAG,KAAK,GAAG,IAAI,GAAG,KAAK,MAAM;QAAE,OAAO,IAAI,CAAC;IAC/C,IAAI,GAAG,KAAK,GAAG,IAAI,GAAG,KAAK,OAAO;QAAE,OAAO,KAAK,CAAC;IACjD,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;AACzB,CAAC;AAED;;;;;;;;;;;;;;GAcG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,GAAW,EACX,UAAsB;IAEtB,MAAM,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACzC,MAAM,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,aAAa,EAAE,OAAO,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC5E,MAAM,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,aAAa,EAAE,OAAO,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE5E,MAAM,iBAAiB,GAAc,EAAE,CAAC;IACxC,6EAA6E;IAC7E,6EAA6E;IAC7E,sEAAsE;IACtE,8EAA8E;IAC9E,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAAkB,CAAC;IACpD,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAAkB,CAAC;IACpD,KAAK,MAAM,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,UAAU,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;QACnD,MAAM,KAAK,GAAkB,EAAE,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;QACvD,IAAI,KAAK,CAAC,UAAU,EAAE,CAAC;YACrB,IAAI,OAAO,GAAG,iBAAiB,CAAC,GAAG,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;YACtD,IAAI,CAAC,OAAO,EAAE,CAAC;gBACb,OAAO,GAAG,qBAAqB,CAAC,GAAG,CAAC,MAAM,CAAC;gBAC3C,MAAM,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,EAAE,KAAK,CAAC,UAAU,CAAC,CAAC;gBAC9D,iBAAiB,CAAC,GAAG,CAAC,KAAK,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;YACnD,CAAC;YACD,KAAK,CAAC,cAAc,GAAG,OAAO,CAAC;YAC/B,OAAO,KAAK,CAAC,UAAU,CAAC;QAC1B,CAAC;QAED,MAAM,eAAe,GAAG,IAAI,CAAC,aAAa,CAAC,UAAU,CAAC,MAAM,CAC1D,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,OAAO,CAC1B,CAAC;QACF,MAAM,cAAc,GAAG,eAAe,CAAC,MAAM,GAAG,CAAC,CAAC;QAClD,IAAI,QAAQ,GAAG,CAAC,CAAC;QACjB,MAAM,UAAU,GAAc,EAAE,CAAC;QACjC,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,aAAa,CAAC,UAAU,EAAE,CAAC;YAC9C,IAAI,CAAC,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;gBACvB,UAAU,CAAC,IAAI,CACb,CAAC,CAAC,IAAI,KAAK,MAAM;oBACf,CAAC,CAAC;wBACE,GAAG,CAAC;wBACJ,OAAO,EAAE,yBAAyB,CAAC,CAAC,CAAC,OAAO,EAAE,IAAI,CAAC,UAAU,CAAC;qBAC/D;oBACH,CAAC,CAAC,CAAC,CACN,CAAC;gBACF,SAAS;YACX,CAAC;YACD,IAAI,OAAO,GAAG,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;YAC7C,IAAI,CAAC,OAAO,EAAE,CAAC;gBACb,MAAM,MAAM,GAAG,cAAc,CAAC,CAAC,CAAC,IAAI,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACpD,OAAO,GAAG,qBAAqB,CAAC,GAAG,CAAC,GAAG,MAAM,MAAM,CAAC;gBACpD,MAAM,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC;gBACrD,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;YAC1C,CAAC;YACD,UAAU,CAAC,IAAI,CAAC;gBACd,IAAI,EAAE,OAAO;gBACb,SAAS,EAAE,OAAO;gBAClB,SAAS,EAAE,CAAC,CAAC,SAAS;aACvB,CAAC,CAAC;YACH,QAAQ,IAAI,CAAC,CAAC;QAChB,CAAC;QACD,iBAAiB,CAAC,IAAI,CAAC;YACrB,GAAG,IAAI;YACP,aAAa,EAAE,KAAK;YACpB,aAAa,EAAE,EAAE,UAAU,EAAE;YAC7B,UAAU,EAAE;gBACV,GAAG,IAAI,CAAC,UAAU;gBAClB,MAAM,EAAE,yBAAyB,CAC/B,IAAI,CAAC,UAAU,CAAC,MAAM,EACtB,IAAI,CAAC,UAAU,CAChB;aACF;SACF,CAAC,CAAC;IACL,CAAC;IAED,MAAM,gBAAgB,GACpB,UAAU,CAAC,gBAAgB,KAAK,SAAS;QACvC,CAAC,CAAC,SAAS;QACX,CAAC,CAAC,EAAE,GAAG,UAAU,CAAC,gBAAgB,EAAE,CAAC;IACzC,IAAI,gBAAgB,EAAE,UAAU,EAAE,CAAC;QACjC,MAAM,OAAO,GAAG,6BAA6B,CAAC;QAC9C,MAAM,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,EAAE,gBAAgB,CAAC,UAAU,CAAC,CAAC;QACzE,gBAAgB,CAAC,cAAc,GAAG,OAAO,CAAC;QAC1C,OAAO,gBAAgB,CAAC,UAAU,CAAC;IACrC,CAAC;IAED,sEAAsE;IACtE,gEAAgE;IAChE,MAAM,UAAU,GAAG;QACjB,GAAG,UAAU;QACb,KAAK,EAAE,iBAAiB;QACxB,GAAG,CAAC,gBAAgB,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACvC,CAAC;IAEb,MAAM,EAAE,CAAC,SAAS,CAChB,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,iBAAiB,CAAC,EACjC,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC,CACpC,CAAC;IAEF,MAAM,EAAE,CAAC,SAAS,CAChB,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,gBAAgB,CAAC,EAChC,IAAI,CAAC,SAAS,CACZ;QACE,IAAI,EAAE,UAAU,CAAC,IAAI;QACrB,MAAM,EAAE,UAAU,CAAC,MAAM;QACzB,WAAW,EAAE,UAAU,CAAC,WAAW,IAAI,IAAI;KAC5C,EACD,IAAI,EACJ,CAAC,CACF,CACF,CAAC;IAEF,MAAM,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,QAAQ,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC9D,MAAM,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,UAAU,CAAC,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;AACtE,CAAC;AAED,SAAS,OAAO,CAAC,UAAsB;IACrC,OAAO,CACL,UAAU,CAAC,KAAK;SACb,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CACf,IAAI,CAAC,SAAS,CAAC;QACb,IAAI,EAAE,CAAC;QACP,MAAM,EAAE,IAAI,CAAC,UAAU;QACvB,GAAG,EAAE,IAAI,CAAC,aAAa,CAAC,GAAG,IAAI,IAAI;QACnC,EAAE,EAAE,IAAI,CAAC,UAAU,CAAC,EAAE;QACtB,SAAS,EAAE,IAAI,CAAC,SAAS,IAAI,SAAS;KACvC,CAAC,CACH;SACA,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CACrB,CAAC;AACJ,CAAC","sourcesContent":["import fs from \"node:fs/promises\";\nimport path from \"node:path\";\nimport type {\n  AgentEvidenceModality,\n  ProbeEvidence,\n  Rubric,\n  Trajectory,\n  TrajectoryStep,\n} from \"./types.js\";\nimport { redactInlineImagePayloads } from \"./evidenceNormalization.js\";\n\ntype RawRubricCriterion = {\n  criterion: unknown;\n  description: unknown;\n  max_points?: unknown;\n  maxPoints?: unknown;\n  condition?: unknown;\n};\n\ntype RawRubric = {\n  items?: unknown;\n};\n\ntype PersistedProbeEvidence = ProbeEvidence & {\n  screenshotPath?: string;\n};\n\n/**\n * Convert dataset or generated rubric JSON into the public Stagehand shape.\n * Snake-case dataset fields are accepted here so serialized quirks do not leak\n * into the canonical rubric type.\n */\nexport function normalizeRubric(rubric: unknown): Rubric | undefined {\n  if (rubric == null) return undefined;\n  if (typeof rubric !== \"object\") {\n    throw new TypeError(\"Rubric must be an object\");\n  }\n\n  const rawRubric = rubric as RawRubric;\n  if (!Array.isArray(rawRubric.items)) {\n    throw new TypeError(\"Rubric is missing an items array\");\n  }\n\n  return {\n    items: rawRubric.items.map((item) => {\n      const criterion = normalizeRequiredString(item.criterion, \"criterion\");\n      const description = normalizeRequiredString(\n        item.description,\n        \"description\",\n      );\n      const maxPoints = normalizeMaxPoints(item);\n\n      if (typeof maxPoints !== \"number\" || !Number.isFinite(maxPoints)) {\n        throw new TypeError(\n          `Rubric criterion \"${criterion}\" is missing a numeric maxPoints value`,\n        );\n      }\n\n      return {\n        criterion,\n        description,\n        maxPoints,\n        ...(typeof item.condition === \"string\" && {\n          condition: item.condition,\n        }),\n      };\n    }),\n  };\n}\n\nfunction normalizeRequiredString(value: unknown, fieldName: string): string {\n  if (typeof value === \"string\" && value.length) {\n    return value;\n  }\n\n  throw new TypeError(`Rubric criterion is missing a ${fieldName} value`);\n}\n\nfunction normalizeMaxPoints(item: RawRubricCriterion): unknown {\n  return item.maxPoints ?? item.max_points;\n}\n\nfunction normalizeResultLabel(label?: string): string {\n  return (label ?? `rescore-${new Date().toISOString()}`).replace(\n    /[^A-Za-z0-9._-]/g,\n    \"_\",\n  );\n}\n\n// ─────────────────────────────────────────────────────────────────────────────\n// On-disk loader\n// ─────────────────────────────────────────────────────────────────────────────\n\n/**\n * Hydrate a Trajectory from the on-disk directory layout written by\n * TrajectoryRecorder.persist(). Used by the offline re-scoring CLI (`bench\n * verify`) and by any consumer that wants to feed a saved trajectory back\n * into V3Evaluator.verify() without running an agent.\n *\n * Reverses the recorder's serialization tweaks:\n *   - `probeEvidence.screenshotPath` → read file into `probeEvidence.screenshot`.\n *   - Image modalities in `agentEvidence.modalities` carry `imagePath` on\n *     disk instead of raw Buffer; legacy `bytesBase64` fixtures are also\n *     accepted.\n *\n * @param dir absolute or cwd-relative path to a `<run-id>/<task-id>/` directory.\n */\nexport async function loadTrajectoryFromDisk(dir: string): Promise<Trajectory> {\n  const fs = await import(\"node:fs/promises\");\n  const path = await import(\"node:path\");\n  const trajectoryDir = path.resolve(dir);\n\n  const trajectoryPath = path.join(trajectoryDir, \"trajectory.json\");\n  const raw = await fs.readFile(trajectoryPath, \"utf8\");\n  const parsed = JSON.parse(raw) as Trajectory & {\n    finalObservation?: PersistedProbeEvidence;\n    steps: Array<\n      TrajectoryStep & {\n        agentEvidence: {\n          modalities: Array<\n            | { type: \"text\"; content: string }\n            | {\n                type: \"image\";\n                mediaType: string;\n                // On-disk forms. Current writer externalizes bytes to\n                // imagePath; bytesBase64 is accepted for older fixtures.\n                bytes?: unknown;\n                bytesBase64?: string;\n                imagePath?: string;\n              }\n            | { type: \"json\"; content: unknown }\n          >;\n        };\n        probeEvidence: PersistedProbeEvidence;\n      }\n    >;\n  };\n\n  const resolveWithinTrajectoryDir = (\n    candidate: string,\n    fieldName = \"screenshotPath\",\n  ): string => {\n    const resolved = path.resolve(trajectoryDir, candidate);\n    const relative = path.relative(trajectoryDir, resolved);\n    const outside =\n      relative === \"..\" ||\n      relative.startsWith(`..${path.sep}`) ||\n      path.isAbsolute(relative);\n\n    if (outside) {\n      throw new Error(\n        `Trajectory ${fieldName} escapes trajectory directory: ${candidate}`,\n      );\n    }\n\n    return resolved;\n  };\n\n  const hydrateProbeScreenshot = async (\n    probe: PersistedProbeEvidence | undefined,\n  ): Promise<void> => {\n    if (probe?.screenshotPath && !probe.screenshot) {\n      const resolved = resolveWithinTrajectoryDir(probe.screenshotPath);\n      try {\n        probe.screenshot = await fs.readFile(resolved);\n      } catch {\n        // Missing screenshot file: leave probe.screenshot unset. The verifier's\n        // evidence_insufficient path will handle it.\n      }\n    }\n  };\n\n  for (const step of parsed.steps) {\n    // Rehydrate tier-2 probe screenshot from its on-disk file reference.\n    await hydrateProbeScreenshot(step.probeEvidence);\n\n    // Decode image modalities from disk references back to Buffer.\n    if (step.agentEvidence?.modalities) {\n      const modalities: AgentEvidenceModality[] = [];\n      for (const m of step.agentEvidence.modalities) {\n        // The on-disk shape carries imagePath/bytesBase64 instead of bytes,\n        // so we look through `unknown` rather than rely on the typed union.\n        const raw = m as unknown as {\n          bytesBase64?: string;\n          imagePath?: string;\n        };\n        if (m.type === \"image\" && typeof raw.bytesBase64 === \"string\") {\n          modalities.push({\n            type: \"image\" as const,\n            bytes: Buffer.from(raw.bytesBase64, \"base64\"),\n            mediaType: m.mediaType,\n          });\n          continue;\n        }\n        if (m.type === \"image\" && typeof raw.imagePath === \"string\") {\n          const resolved = resolveWithinTrajectoryDir(\n            raw.imagePath,\n            \"imagePath\",\n          );\n          try {\n            modalities.push({\n              type: \"image\" as const,\n              bytes: await fs.readFile(resolved),\n              mediaType: m.mediaType,\n            });\n          } catch {\n            // Missing agent image file: omit that image modality. The\n            // verifier's evidence_insufficient path will handle missing bytes.\n          }\n          continue;\n        }\n        modalities.push(m as AgentEvidenceModality);\n      }\n      step.agentEvidence.modalities = modalities;\n    }\n  }\n\n  await hydrateProbeScreenshot(parsed.finalObservation);\n\n  return parsed;\n}\n\n/**\n * Build a `result*.json` filename for persisted evaluator output.\n *\n * Convention: the live run writes `result.json`; offline re-score attempts use\n * a label-based name (e.g., `result_rescore-2026-05-11.json`) so they coexist\n * without collisions and remain easy to diff.\n */\nexport function nextResultFilename(label?: string): string {\n  return `result_${normalizeResultLabel(label)}.json`;\n}\n\n/**\n * Default persistence policy: explicit override, then env, then \"on unless CI\".\n */\nexport function shouldPersistTrajectory(\n  override: boolean | undefined,\n): boolean {\n  if (override !== undefined) return override;\n  const env = process.env.VERIFIER_PERSIST_TRAJECTORIES?.toLowerCase();\n  if (env === \"1\" || env === \"true\") return true;\n  if (env === \"0\" || env === \"false\") return false;\n  return !process.env.CI;\n}\n\n/**\n * Write the on-disk trajectory layout under `dir`:\n *\n *   <dir>/\n *     ├── task_data.json\n *     ├── trajectory.json    (screenshots referenced by path)\n *     ├── screenshots/\n *     │   ├── probe/<N>.png\n *     │   └── agent/<N>[_M].png\n *     ├── scores/            (empty; populated separately)\n *     └── core.log\n *\n * Image bytes are externalized to PNG files; the in-memory Trajectory is left\n * untouched so callers can keep using it after persistence.\n */\nexport async function writeTrajectoryDir(\n  dir: string,\n  trajectory: Trajectory,\n): Promise<void> {\n  await fs.mkdir(dir, { recursive: true });\n  await fs.mkdir(path.join(dir, \"screenshots\", \"probe\"), { recursive: true });\n  await fs.mkdir(path.join(dir, \"screenshots\", \"agent\"), { recursive: true });\n\n  const serializableSteps: unknown[] = [];\n  // A single post-turn probe is fanned across every step of a multi-tool turn,\n  // and a single agent screenshot is shared across every action a CUA provider\n  // chose from it, so the same Buffer is shared by reference. Dedupe by\n  // identity: write the PNG once and point every sharing step at the same file.\n  const probePathByBuffer = new Map<Buffer, string>();\n  const agentPathByBuffer = new Map<Buffer, string>();\n  for (const [i, step] of trajectory.steps.entries()) {\n    const probe: ProbeEvidence = { ...step.probeEvidence };\n    if (probe.screenshot) {\n      let relPath = probePathByBuffer.get(probe.screenshot);\n      if (!relPath) {\n        relPath = `screenshots/probe/${i + 1}.png`;\n        await fs.writeFile(path.join(dir, relPath), probe.screenshot);\n        probePathByBuffer.set(probe.screenshot, relPath);\n      }\n      probe.screenshotPath = relPath;\n      delete probe.screenshot;\n    }\n\n    const imageModalities = step.agentEvidence.modalities.filter(\n      (m) => m.type === \"image\",\n    );\n    const multipleImages = imageModalities.length > 1;\n    let imageSeq = 0;\n    const modalities: unknown[] = [];\n    for (const m of step.agentEvidence.modalities) {\n      if (m.type !== \"image\") {\n        modalities.push(\n          m.type === \"json\"\n            ? {\n                ...m,\n                content: redactInlineImagePayloads(m.content, step.actionName),\n              }\n            : m,\n        );\n        continue;\n      }\n      let relPath = agentPathByBuffer.get(m.bytes);\n      if (!relPath) {\n        const suffix = multipleImages ? `_${imageSeq}` : \"\";\n        relPath = `screenshots/agent/${i + 1}${suffix}.png`;\n        await fs.writeFile(path.join(dir, relPath), m.bytes);\n        agentPathByBuffer.set(m.bytes, relPath);\n      }\n      modalities.push({\n        type: \"image\",\n        imagePath: relPath,\n        mediaType: m.mediaType,\n      });\n      imageSeq += 1;\n    }\n    serializableSteps.push({\n      ...step,\n      probeEvidence: probe,\n      agentEvidence: { modalities },\n      toolOutput: {\n        ...step.toolOutput,\n        result: redactInlineImagePayloads(\n          step.toolOutput.result,\n          step.actionName,\n        ),\n      },\n    });\n  }\n\n  const finalObservation: ProbeEvidence | undefined =\n    trajectory.finalObservation === undefined\n      ? undefined\n      : { ...trajectory.finalObservation };\n  if (finalObservation?.screenshot) {\n    const relPath = \"screenshots/probe/final.png\";\n    await fs.writeFile(path.join(dir, relPath), finalObservation.screenshot);\n    finalObservation.screenshotPath = relPath;\n    delete finalObservation.screenshot;\n  }\n\n  // Image modalities carry imagePath instead of raw bytes on disk; cast\n  // through unknown rather than widen Trajectory's type contract.\n  const serialized = {\n    ...trajectory,\n    steps: serializableSteps,\n    ...(finalObservation ? { finalObservation } : {}),\n  } as unknown;\n\n  await fs.writeFile(\n    path.join(dir, \"trajectory.json\"),\n    JSON.stringify(serialized, null, 2),\n  );\n\n  await fs.writeFile(\n    path.join(dir, \"task_data.json\"),\n    JSON.stringify(\n      {\n        task: trajectory.task,\n        status: trajectory.status,\n        finalAnswer: trajectory.finalAnswer ?? null,\n      },\n      null,\n      2,\n    ),\n  );\n\n  await fs.mkdir(path.join(dir, \"scores\"), { recursive: true });\n  await fs.writeFile(path.join(dir, \"core.log\"), coreLog(trajectory));\n}\n\nfunction coreLog(trajectory: Trajectory): string {\n  return (\n    trajectory.steps\n      .map((step, i) =>\n        JSON.stringify({\n          step: i,\n          action: step.actionName,\n          url: step.probeEvidence.url ?? null,\n          ok: step.toolOutput.ok,\n          reasoning: step.reasoning || undefined,\n        }),\n      )\n      .join(\"\\n\") + \"\\n\"\n  );\n}\n"]}