@browserbasehq/orca 3.5.0-vertex-test → 3.6.0-gemini-cua

package/dist/cjs/lib/v3/verifier/trajectory.js

@@ -0,0 +1,316 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.normalizeRubric = normalizeRubric;
+exports.loadTrajectoryFromDisk = loadTrajectoryFromDisk;
+exports.nextResultFilename = nextResultFilename;
+exports.shouldPersistTrajectory = shouldPersistTrajectory;
+exports.writeTrajectoryDir = writeTrajectoryDir;
+const promises_1 = __importDefault(require("node:fs/promises"));
+const node_path_1 = __importDefault(require("node:path"));
+const evidenceNormalization_js_1 = require("./evidenceNormalization.js");
+/**
+ * Convert dataset or generated rubric JSON into the public Stagehand shape.
+ * Snake-case dataset fields are accepted here so serialized quirks do not leak
+ * into the canonical rubric type.
+ */
+function normalizeRubric(rubric) {
+    if (rubric == null)
+        return undefined;
+    if (typeof rubric !== "object") {
+        throw new TypeError("Rubric must be an object");
+    }
+    const rawRubric = rubric;
+    if (!Array.isArray(rawRubric.items)) {
+        throw new TypeError("Rubric is missing an items array");
+    }
+    return {
+        items: rawRubric.items.map((item) => {
+            const criterion = normalizeRequiredString(item.criterion, "criterion");
+            const description = normalizeRequiredString(item.description, "description");
+            const maxPoints = normalizeMaxPoints(item);
+            if (typeof maxPoints !== "number" || !Number.isFinite(maxPoints)) {
+                throw new TypeError(`Rubric criterion "${criterion}" is missing a numeric maxPoints value`);
+            }
+            return {
+                criterion,
+                description,
+                maxPoints,
+                ...(typeof item.condition === "string" && {
+                    condition: item.condition,
+                }),
+            };
+        }),
+    };
+}
+function normalizeRequiredString(value, fieldName) {
+    if (typeof value === "string" && value.length) {
+        return value;
+    }
+    throw new TypeError(`Rubric criterion is missing a ${fieldName} value`);
+}
+function normalizeMaxPoints(item) {
+    return item.maxPoints ?? item.max_points;
+}
+function normalizeResultLabel(label) {
+    return (label ?? `rescore-${new Date().toISOString()}`).replace(/[^A-Za-z0-9._-]/g, "_");
+}
+// ─────────────────────────────────────────────────────────────────────────────
+// On-disk loader
+// ─────────────────────────────────────────────────────────────────────────────
+/**
+ * Hydrate a Trajectory from the on-disk directory layout written by
+ * TrajectoryRecorder.persist(). Used by the offline re-scoring CLI (`bench
+ * verify`) and by any consumer that wants to feed a saved trajectory back
+ * into V3Evaluator.verify() without running an agent.
+ *
+ * Reverses the recorder's serialization tweaks:
+ *   - `probeEvidence.screenshotPath` → read file into `probeEvidence.screenshot`.
+ *   - Image modalities in `agentEvidence.modalities` carry `imagePath` on disk
+ *     (relative to the trajectory dir). Legacy `bytesBase64` is also accepted
+ *     for trajectories written before the externalization change.
+ *
+ * @param dir absolute or cwd-relative path to a `<run-id>/<task-id>/` directory.
+ */
+async function loadTrajectoryFromDisk(dir) {
+    const fs = await Promise.resolve().then(() => __importStar(require("node:fs/promises")));
+    const path = await Promise.resolve().then(() => __importStar(require("node:path")));
+    const trajectoryDir = path.resolve(dir);
+    const trajectoryPath = path.join(trajectoryDir, "trajectory.json");
+    const raw = await fs.readFile(trajectoryPath, "utf8");
+    const parsed = JSON.parse(raw);
+    const resolveWithinTrajectoryDir = (candidate, fieldName = "screenshotPath") => {
+        const resolved = path.resolve(trajectoryDir, candidate);
+        const relative = path.relative(trajectoryDir, resolved);
+        const outside = relative === ".." ||
+            relative.startsWith(`..${path.sep}`) ||
+            path.isAbsolute(relative);
+        if (outside) {
+            throw new Error(`Trajectory ${fieldName} escapes trajectory directory: ${candidate}`);
+        }
+        return resolved;
+    };
+    const hydrateProbeScreenshot = async (probe) => {
+        if (probe?.screenshotPath && !probe.screenshot) {
+            const resolved = resolveWithinTrajectoryDir(probe.screenshotPath);
+            try {
+                probe.screenshot = await fs.readFile(resolved);
+            }
+            catch {
+                // Missing screenshot file: leave probe.screenshot unset. The verifier's
+                // evidence_insufficient path will handle it.
+            }
+        }
+    };
+    for (const step of parsed.steps) {
+        // Rehydrate tier-2 probe screenshot from its on-disk file reference.
+        await hydrateProbeScreenshot(step.probeEvidence);
+        // Rehydrate tier-1 image modalities. New shape: resolve imagePath → Buffer.
+        // Legacy shape: decode bytesBase64 → Buffer.
+        if (step.agentEvidence?.modalities) {
+            const resolved = [];
+            for (const m of step.agentEvidence.modalities) {
+                if (m.type !== "image") {
+                    resolved.push(m);
+                    continue;
+                }
+                const raw = m;
+                if (typeof raw.imagePath === "string") {
+                    const imagePath = resolveWithinTrajectoryDir(raw.imagePath, "imagePath");
+                    try {
+                        const bytes = await fs.readFile(imagePath);
+                        resolved.push({ type: "image", bytes, mediaType: m.mediaType });
+                    }
+                    catch {
+                        // Missing tier-1 image: drop this modality. evidence_insufficient
+                        // handles the downstream effect.
+                    }
+                }
+                else if (typeof raw.bytesBase64 === "string") {
+                    resolved.push({
+                        type: "image",
+                        bytes: Buffer.from(raw.bytesBase64, "base64"),
+                        mediaType: m.mediaType,
+                    });
+                }
+                else {
+                    resolved.push(m);
+                }
+            }
+            step.agentEvidence.modalities = resolved;
+        }
+    }
+    await hydrateProbeScreenshot(parsed.finalObservation);
+    return parsed;
+}
+/**
+ * Build a `result*.json` filename for persisted evaluator output.
+ *
+ * Convention: the live run writes `result.json`; offline re-score attempts use
+ * a label-based name (e.g., `result_rescore-2026-05-11.json`) so they coexist
+ * without collisions and remain easy to diff.
+ */
+function nextResultFilename(label) {
+    return `result_${normalizeResultLabel(label)}.json`;
+}
+/**
+ * Default persistence policy: explicit override, then env, then "on unless CI".
+ */
+function shouldPersistTrajectory(override) {
+    if (override !== undefined)
+        return override;
+    const env = process.env.VERIFIER_PERSIST_TRAJECTORIES?.toLowerCase();
+    if (env === "1" || env === "true")
+        return true;
+    if (env === "0" || env === "false")
+        return false;
+    return !process.env.CI;
+}
+/**
+ * Write the on-disk trajectory layout under `dir`:
+ *
+ *   <dir>/
+ *     ├── task_data.json
+ *     ├── trajectory.json    (screenshots referenced by path)
+ *     ├── screenshots/
+ *     │   ├── probe/<N>.png
+ *     │   └── agent/<N>[_M].png
+ *     ├── scores/            (empty; populated separately)
+ *     └── core.log
+ *
+ * Image bytes are externalized to PNG files; the in-memory Trajectory is left
+ * untouched so callers can keep using it after persistence.
+ */
+async function writeTrajectoryDir(dir, trajectory) {
+    await promises_1.default.mkdir(dir, { recursive: true });
+    await promises_1.default.mkdir(node_path_1.default.join(dir, "screenshots", "probe"), { recursive: true });
+    await promises_1.default.mkdir(node_path_1.default.join(dir, "screenshots", "agent"), { recursive: true });
+    const serializableSteps = [];
+    // A single post-turn probe is fanned across every step of a multi-tool turn,
+    // and a single agent screenshot is shared across every action a CUA provider
+    // chose from it, so the same Buffer is shared by reference. Dedupe by
+    // identity: write the PNG once and point every sharing step at the same file.
+    const probePathByBuffer = new Map();
+    const agentPathByBuffer = new Map();
+    for (const [i, step] of trajectory.steps.entries()) {
+        const probe = { ...step.probeEvidence };
+        if (probe.screenshot) {
+            let relPath = probePathByBuffer.get(probe.screenshot);
+            if (!relPath) {
+                relPath = `screenshots/probe/${i + 1}.png`;
+                await promises_1.default.writeFile(node_path_1.default.join(dir, relPath), probe.screenshot);
+                probePathByBuffer.set(probe.screenshot, relPath);
+            }
+            probe.screenshotPath = relPath;
+            delete probe.screenshot;
+        }
+        const imageModalities = step.agentEvidence.modalities.filter((m) => m.type === "image");
+        const multipleImages = imageModalities.length > 1;
+        let imageSeq = 0;
+        const modalities = [];
+        for (const m of step.agentEvidence.modalities) {
+            if (m.type !== "image") {
+                modalities.push(m.type === "json"
+                    ? {
+                        ...m,
+                        content: (0, evidenceNormalization_js_1.redactInlineImagePayloads)(m.content, step.actionName),
+                    }
+                    : m);
+                continue;
+            }
+            let relPath = agentPathByBuffer.get(m.bytes);
+            if (!relPath) {
+                const suffix = multipleImages ? `_${imageSeq}` : "";
+                relPath = `screenshots/agent/${i + 1}${suffix}.png`;
+                await promises_1.default.writeFile(node_path_1.default.join(dir, relPath), m.bytes);
+                agentPathByBuffer.set(m.bytes, relPath);
+            }
+            modalities.push({
+                type: "image",
+                imagePath: relPath,
+                mediaType: m.mediaType,
+            });
+            imageSeq += 1;
+        }
+        serializableSteps.push({
+            ...step,
+            probeEvidence: probe,
+            agentEvidence: { modalities },
+            toolOutput: {
+                ...step.toolOutput,
+                result: (0, evidenceNormalization_js_1.redactInlineImagePayloads)(step.toolOutput.result, step.actionName),
+            },
+        });
+    }
+    const finalObservation = trajectory.finalObservation === undefined
+        ? undefined
+        : { ...trajectory.finalObservation };
+    if (finalObservation?.screenshot) {
+        const relPath = "screenshots/probe/final.png";
+        await promises_1.default.writeFile(node_path_1.default.join(dir, relPath), finalObservation.screenshot);
+        finalObservation.screenshotPath = relPath;
+        delete finalObservation.screenshot;
+    }
+    // Image modalities carry imagePath instead of raw bytes on disk; cast
+    // through unknown rather than widen Trajectory's type contract.
+    const serialized = {
+        ...trajectory,
+        steps: serializableSteps,
+        ...(finalObservation ? { finalObservation } : {}),
+    };
+    await promises_1.default.writeFile(node_path_1.default.join(dir, "trajectory.json"), JSON.stringify(serialized, null, 2));
+    await promises_1.default.writeFile(node_path_1.default.join(dir, "task_data.json"), JSON.stringify({
+        task: trajectory.task,
+        status: trajectory.status,
+        finalAnswer: trajectory.finalAnswer ?? null,
+    }, null, 2));
+    await promises_1.default.mkdir(node_path_1.default.join(dir, "scores"), { recursive: true });
+    await promises_1.default.writeFile(node_path_1.default.join(dir, "core.log"), coreLog(trajectory));
+}
+function coreLog(trajectory) {
+    return (trajectory.steps
+        .map((step, i) => JSON.stringify({
+        step: i,
+        action: step.actionName,
+        url: step.probeEvidence.url ?? null,
+        ok: step.toolOutput.ok,
+        reasoning: step.reasoning || undefined,
+    }))
+        .join("\n") + "\n");
+}
+//# sourceMappingURL=trajectory.js.map

package/dist/cjs/lib/v3/verifier/trajectory.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"trajectory.js","sourceRoot":"","sources":["../../../../../lib/v3/verifier/trajectory.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAgCA,0CAoCC;AAuCD,wDAgHC;AASD,gDAEC;AAKD,0DAQC;AAiBD,gDAiHC;AArXD,gEAAkC;AAClC,0DAA6B;AAQ7B,yEAAuE;AAkBvE;;;;GAIG;AACH,SAAgB,eAAe,CAAC,MAAe;IAC7C,IAAI,MAAM,IAAI,IAAI;QAAE,OAAO,SAAS,CAAC;IACrC,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;QAC/B,MAAM,IAAI,SAAS,CAAC,0BAA0B,CAAC,CAAC;IAClD,CAAC;IAED,MAAM,SAAS,GAAG,MAAmB,CAAC;IACtC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC;QACpC,MAAM,IAAI,SAAS,CAAC,kCAAkC,CAAC,CAAC;IAC1D,CAAC;IAED,OAAO;QACL,KAAK,EAAE,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;YAClC,MAAM,SAAS,GAAG,uBAAuB,CAAC,IAAI,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC;YACvE,MAAM,WAAW,GAAG,uBAAuB,CACzC,IAAI,CAAC,WAAW,EAChB,aAAa,CACd,CAAC;YACF,MAAM,SAAS,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC;YAE3C,IAAI,OAAO,SAAS,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;gBACjE,MAAM,IAAI,SAAS,CACjB,qBAAqB,SAAS,wCAAwC,CACvE,CAAC;YACJ,CAAC;YAED,OAAO;gBACL,SAAS;gBACT,WAAW;gBACX,SAAS;gBACT,GAAG,CAAC,OAAO,IAAI,CAAC,SAAS,KAAK,QAAQ,IAAI;oBACxC,SAAS,EAAE,IAAI,CAAC,SAAS;iBAC1B,CAAC;aACH,CAAC;QACJ,CAAC,CAAC;KACH,CAAC;AACJ,CAAC;AAED,SAAS,uBAAuB,CAAC,KAAc,EAAE,SAAiB;IAChE,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;QAC9C,OAAO,KAAK,CAAC;IACf,CAAC;IAED,MAAM,IAAI,SAAS,CAAC,iCAAiC,SAAS,QAAQ,CAAC,CAAC;AAC1E,CAAC;AAED,SAAS,kBAAkB,CAAC,IAAwB;IAClD,OAAO,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,UAAU,CAAC;AAC3C,CAAC;AAED,SAAS,oBAAoB,CAAC,KAAc;IAC1C,OAAO,CAAC,KAAK,IAAI,WAAW,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC,OAAO,CAC7D,kBAAkB,EAClB,GAAG,CACJ,CAAC;AACJ,CAAC;AAED,gFAAgF;AAChF,iBAAiB;AACjB,gFAAgF;AAEhF;;;;;;;;;;;;;GAaG;AACI,KAAK,UAAU,sBAAsB,CAAC,GAAW;IACtD,MAAM,EAAE,GAAG,wDAAa,kBAAkB,GAAC,CAAC;IAC5C,MAAM,IAAI,GAAG,wDAAa,WAAW,GAAC,CAAC;IACvC,MAAM,aAAa,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAExC,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,iBAAiB,CAAC,CAAC;IACnE,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC,CAAC;IACtD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAuB5B,CAAC;IAEF,MAAM,0BAA0B,GAAG,CACjC,SAAiB,EACjB,SAAS,GAAG,gBAAgB,EACpB,EAAE;QACV,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,SAAS,CAAC,CAAC;QACxD,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,aAAa,EAAE,QAAQ,CAAC,CAAC;QACxD,MAAM,OAAO,GACX,QAAQ,KAAK,IAAI;YACjB,QAAQ,CAAC,UAAU,CAAC,KAAK,IAAI,CAAC,GAAG,EAAE,CAAC;YACpC,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;QAE5B,IAAI,OAAO,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CACb,cAAc,SAAS,kCAAkC,SAAS,EAAE,CACrE,CAAC;QACJ,CAAC;QAED,OAAO,QAAQ,CAAC;IAClB,CAAC,CAAC;IAEF,MAAM,sBAAsB,GAAG,KAAK,EAClC,KAAyC,EAC1B,EAAE;QACjB,IAAI,KAAK,EAAE,cAAc,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,CAAC;YAC/C,MAAM,QAAQ,GAAG,0BAA0B,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;YAClE,IAAI,CAAC;gBACH,KAAK,CAAC,UAAU,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;YACjD,CAAC;YAAC,MAAM,CAAC;gBACP,wEAAwE;gBACxE,6CAA6C;YAC/C,CAAC;QACH,CAAC;IACH,CAAC,CAAC;IAEF,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;QAChC,qEAAqE;QACrE,MAAM,sBAAsB,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAEjD,4EAA4E;QAC5E,6CAA6C;QAC7C,IAAI,IAAI,CAAC,aAAa,EAAE,UAAU,EAAE,CAAC;YACnC,MAAM,QAAQ,GAA4B,EAAE,CAAC;YAC7C,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,aAAa,CAAC,UAAU,EAAE,CAAC;gBAC9C,IAAI,CAAC,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;oBACvB,QAAQ,CAAC,IAAI,CAAC,CAA0B,CAAC,CAAC;oBAC1C,SAAS;gBACX,CAAC;gBACD,MAAM,GAAG,GAAG,CAGX,CAAC;gBACF,IAAI,OAAO,GAAG,CAAC,SAAS,KAAK,QAAQ,EAAE,CAAC;oBACtC,MAAM,SAAS,GAAG,0BAA0B,CAC1C,GAAG,CAAC,SAAS,EACb,WAAW,CACZ,CAAC;oBACF,IAAI,CAAC;wBACH,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;wBAC3C,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC;oBAClE,CAAC;oBAAC,MAAM,CAAC;wBACP,kEAAkE;wBAClE,iCAAiC;oBACnC,CAAC;gBACH,CAAC;qBAAM,IAAI,OAAO,GAAG,CAAC,WAAW,KAAK,QAAQ,EAAE,CAAC;oBAC/C,QAAQ,CAAC,IAAI,CAAC;wBACZ,IAAI,EAAE,OAAO;wBACb,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,QAAQ,CAAC;wBAC7C,SAAS,EAAE,CAAC,CAAC,SAAS;qBACvB,CAAC,CAAC;gBACL,CAAC;qBAAM,CAAC;oBACN,QAAQ,CAAC,IAAI,CAAC,CAA0B,CAAC,CAAC;gBAC5C,CAAC;YACH,CAAC;YACD,IAAI,CAAC,aAAa,CAAC,UAAU,GAAG,QAAQ,CAAC;QAC3C,CAAC;IACH,CAAC;IAED,MAAM,sBAAsB,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC;IAEtD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;;;GAMG;AACH,SAAgB,kBAAkB,CAAC,KAAc;IAC/C,OAAO,UAAU,oBAAoB,CAAC,KAAK,CAAC,OAAO,CAAC;AACtD,CAAC;AAED;;GAEG;AACH,SAAgB,uBAAuB,CACrC,QAA6B;IAE7B,IAAI,QAAQ,KAAK,SAAS;QAAE,OAAO,QAAQ,CAAC;IAC5C,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,6BAA6B,EAAE,WAAW,EAAE,CAAC;IACrE,IAAI,GAAG,KAAK,GAAG,IAAI,GAAG,KAAK,MAAM;QAAE,OAAO,IAAI,CAAC;IAC/C,IAAI,GAAG,KAAK,GAAG,IAAI,GAAG,KAAK,OAAO;QAAE,OAAO,KAAK,CAAC;IACjD,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;AACzB,CAAC;AAED;;;;;;;;;;;;;;GAcG;AACI,KAAK,UAAU,kBAAkB,CACtC,GAAW,EACX,UAAsB;IAEtB,MAAM,kBAAE,CAAC,KAAK,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACzC,MAAM,kBAAE,CAAC,KAAK,CAAC,mBAAI,CAAC,IAAI,CAAC,GAAG,EAAE,aAAa,EAAE,OAAO,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC5E,MAAM,kBAAE,CAAC,KAAK,CAAC,mBAAI,CAAC,IAAI,CAAC,GAAG,EAAE,aAAa,EAAE,OAAO,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE5E,MAAM,iBAAiB,GAAc,EAAE,CAAC;IACxC,6EAA6E;IAC7E,6EAA6E;IAC7E,sEAAsE;IACtE,8EAA8E;IAC9E,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAAkB,CAAC;IACpD,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAAkB,CAAC;IACpD,KAAK,MAAM,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,UAAU,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;QACnD,MAAM,KAAK,GAAkB,EAAE,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;QACvD,IAAI,KAAK,CAAC,UAAU,EAAE,CAAC;YACrB,IAAI,OAAO,GAAG,iBAAiB,CAAC,GAAG,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;YACtD,IAAI,CAAC,OAAO,EAAE,CAAC;gBACb,OAAO,GAAG,qBAAqB,CAAC,GAAG,CAAC,MAAM,CAAC;gBAC3C,MAAM,kBAAE,CAAC,SAAS,CAAC,mBAAI,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,EAAE,KAAK,CAAC,UAAU,CAAC,CAAC;gBAC9D,iBAAiB,CAAC,GAAG,CAAC,KAAK,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;YACnD,CAAC;YACD,KAAK,CAAC,cAAc,GAAG,OAAO,CAAC;YAC/B,OAAO,KAAK,CAAC,UAAU,CAAC;QAC1B,CAAC;QAED,MAAM,eAAe,GAAG,IAAI,CAAC,aAAa,CAAC,UAAU,CAAC,MAAM,CAC1D,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,OAAO,CAC1B,CAAC;QACF,MAAM,cAAc,GAAG,eAAe,CAAC,MAAM,GAAG,CAAC,CAAC;QAClD,IAAI,QAAQ,GAAG,CAAC,CAAC;QACjB,MAAM,UAAU,GAAc,EAAE,CAAC;QACjC,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,aAAa,CAAC,UAAU,EAAE,CAAC;YAC9C,IAAI,CAAC,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;gBACvB,UAAU,CAAC,IAAI,CACb,CAAC,CAAC,IAAI,KAAK,MAAM;oBACf,CAAC,CAAC;wBACE,GAAG,CAAC;wBACJ,OAAO,EAAE,IAAA,oDAAyB,EAAC,CAAC,CAAC,OAAO,EAAE,IAAI,CAAC,UAAU,CAAC;qBAC/D;oBACH,CAAC,CAAC,CAAC,CACN,CAAC;gBACF,SAAS;YACX,CAAC;YACD,IAAI,OAAO,GAAG,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;YAC7C,IAAI,CAAC,OAAO,EAAE,CAAC;gBACb,MAAM,MAAM,GAAG,cAAc,CAAC,CAAC,CAAC,IAAI,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACpD,OAAO,GAAG,qBAAqB,CAAC,GAAG,CAAC,GAAG,MAAM,MAAM,CAAC;gBACpD,MAAM,kBAAE,CAAC,SAAS,CAAC,mBAAI,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC;gBACrD,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;YAC1C,CAAC;YACD,UAAU,CAAC,IAAI,CAAC;gBACd,IAAI,EAAE,OAAO;gBACb,SAAS,EAAE,OAAO;gBAClB,SAAS,EAAE,CAAC,CAAC,SAAS;aACvB,CAAC,CAAC;YACH,QAAQ,IAAI,CAAC,CAAC;QAChB,CAAC;QACD,iBAAiB,CAAC,IAAI,CAAC;YACrB,GAAG,IAAI;YACP,aAAa,EAAE,KAAK;YACpB,aAAa,EAAE,EAAE,UAAU,EAAE;YAC7B,UAAU,EAAE;gBACV,GAAG,IAAI,CAAC,UAAU;gBAClB,MAAM,EAAE,IAAA,oDAAyB,EAC/B,IAAI,CAAC,UAAU,CAAC,MAAM,EACtB,IAAI,CAAC,UAAU,CAChB;aACF;SACF,CAAC,CAAC;IACL,CAAC;IAED,MAAM,gBAAgB,GACpB,UAAU,CAAC,gBAAgB,KAAK,SAAS;QACvC,CAAC,CAAC,SAAS;QACX,CAAC,CAAC,EAAE,GAAG,UAAU,CAAC,gBAAgB,EAAE,CAAC;IACzC,IAAI,gBAAgB,EAAE,UAAU,EAAE,CAAC;QACjC,MAAM,OAAO,GAAG,6BAA6B,CAAC;QAC9C,MAAM,kBAAE,CAAC,SAAS,CAAC,mBAAI,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,EAAE,gBAAgB,CAAC,UAAU,CAAC,CAAC;QACzE,gBAAgB,CAAC,cAAc,GAAG,OAAO,CAAC;QAC1C,OAAO,gBAAgB,CAAC,UAAU,CAAC;IACrC,CAAC;IAED,sEAAsE;IACtE,gEAAgE;IAChE,MAAM,UAAU,GAAG;QACjB,GAAG,UAAU;QACb,KAAK,EAAE,iBAAiB;QACxB,GAAG,CAAC,gBAAgB,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACvC,CAAC;IAEb,MAAM,kBAAE,CAAC,SAAS,CAChB,mBAAI,CAAC,IAAI,CAAC,GAAG,EAAE,iBAAiB,CAAC,EACjC,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC,CACpC,CAAC;IAEF,MAAM,kBAAE,CAAC,SAAS,CAChB,mBAAI,CAAC,IAAI,CAAC,GAAG,EAAE,gBAAgB,CAAC,EAChC,IAAI,CAAC,SAAS,CACZ;QACE,IAAI,EAAE,UAAU,CAAC,IAAI;QACrB,MAAM,EAAE,UAAU,CAAC,MAAM;QACzB,WAAW,EAAE,UAAU,CAAC,WAAW,IAAI,IAAI;KAC5C,EACD,IAAI,EACJ,CAAC,CACF,CACF,CAAC;IAEF,MAAM,kBAAE,CAAC,KAAK,CAAC,mBAAI,CAAC,IAAI,CAAC,GAAG,EAAE,QAAQ,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC9D,MAAM,kBAAE,CAAC,SAAS,CAAC,mBAAI,CAAC,IAAI,CAAC,GAAG,EAAE,UAAU,CAAC,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;AACtE,CAAC;AAED,SAAS,OAAO,CAAC,UAAsB;IACrC,OAAO,CACL,UAAU,CAAC,KAAK;SACb,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CACf,IAAI,CAAC,SAAS,CAAC;QACb,IAAI,EAAE,CAAC;QACP,MAAM,EAAE,IAAI,CAAC,UAAU;QACvB,GAAG,EAAE,IAAI,CAAC,aAAa,CAAC,GAAG,IAAI,IAAI;QACnC,EAAE,EAAE,IAAI,CAAC,UAAU,CAAC,EAAE;QACtB,SAAS,EAAE,IAAI,CAAC,SAAS,IAAI,SAAS;KACvC,CAAC,CACH;SACA,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CACrB,CAAC;AACJ,CAAC","sourcesContent":["import fs from \"node:fs/promises\";\nimport path from \"node:path\";\nimport type {\n  AgentEvidenceModality,\n  ProbeEvidence,\n  Rubric,\n  Trajectory,\n  TrajectoryStep,\n} from \"./types.js\";\nimport { redactInlineImagePayloads } from \"./evidenceNormalization.js\";\n\ntype RawRubricCriterion = {\n  criterion: unknown;\n  description: unknown;\n  max_points?: unknown;\n  maxPoints?: unknown;\n  condition?: unknown;\n};\n\ntype RawRubric = {\n  items?: unknown;\n};\n\ntype PersistedProbeEvidence = ProbeEvidence & {\n  screenshotPath?: string;\n};\n\n/**\n * Convert dataset or generated rubric JSON into the public Stagehand shape.\n * Snake-case dataset fields are accepted here so serialized quirks do not leak\n * into the canonical rubric type.\n */\nexport function normalizeRubric(rubric: unknown): Rubric | undefined {\n  if (rubric == null) return undefined;\n  if (typeof rubric !== \"object\") {\n    throw new TypeError(\"Rubric must be an object\");\n  }\n\n  const rawRubric = rubric as RawRubric;\n  if (!Array.isArray(rawRubric.items)) {\n    throw new TypeError(\"Rubric is missing an items array\");\n  }\n\n  return {\n    items: rawRubric.items.map((item) => {\n      const criterion = normalizeRequiredString(item.criterion, \"criterion\");\n      const description = normalizeRequiredString(\n        item.description,\n        \"description\",\n      );\n      const maxPoints = normalizeMaxPoints(item);\n\n      if (typeof maxPoints !== \"number\" || !Number.isFinite(maxPoints)) {\n        throw new TypeError(\n          `Rubric criterion \"${criterion}\" is missing a numeric maxPoints value`,\n        );\n      }\n\n      return {\n        criterion,\n        description,\n        maxPoints,\n        ...(typeof item.condition === \"string\" && {\n          condition: item.condition,\n        }),\n      };\n    }),\n  };\n}\n\nfunction normalizeRequiredString(value: unknown, fieldName: string): string {\n  if (typeof value === \"string\" && value.length) {\n    return value;\n  }\n\n  throw new TypeError(`Rubric criterion is missing a ${fieldName} value`);\n}\n\nfunction normalizeMaxPoints(item: RawRubricCriterion): unknown {\n  return item.maxPoints ?? item.max_points;\n}\n\nfunction normalizeResultLabel(label?: string): string {\n  return (label ?? `rescore-${new Date().toISOString()}`).replace(\n    /[^A-Za-z0-9._-]/g,\n    \"_\",\n  );\n}\n\n// ─────────────────────────────────────────────────────────────────────────────\n// On-disk loader\n// ─────────────────────────────────────────────────────────────────────────────\n\n/**\n * Hydrate a Trajectory from the on-disk directory layout written by\n * TrajectoryRecorder.persist(). Used by the offline re-scoring CLI (`bench\n * verify`) and by any consumer that wants to feed a saved trajectory back\n * into V3Evaluator.verify() without running an agent.\n *\n * Reverses the recorder's serialization tweaks:\n *   - `probeEvidence.screenshotPath` → read file into `probeEvidence.screenshot`.\n *   - Image modalities in `agentEvidence.modalities` carry `imagePath` on disk\n *     (relative to the trajectory dir). Legacy `bytesBase64` is also accepted\n *     for trajectories written before the externalization change.\n *\n * @param dir absolute or cwd-relative path to a `<run-id>/<task-id>/` directory.\n */\nexport async function loadTrajectoryFromDisk(dir: string): Promise<Trajectory> {\n  const fs = await import(\"node:fs/promises\");\n  const path = await import(\"node:path\");\n  const trajectoryDir = path.resolve(dir);\n\n  const trajectoryPath = path.join(trajectoryDir, \"trajectory.json\");\n  const raw = await fs.readFile(trajectoryPath, \"utf8\");\n  const parsed = JSON.parse(raw) as Trajectory & {\n    finalObservation?: PersistedProbeEvidence;\n    steps: Array<\n      TrajectoryStep & {\n        agentEvidence: {\n          modalities: Array<\n            | { type: \"text\"; content: string }\n            | {\n                type: \"image\";\n                mediaType: string;\n                // Accept both shapes:\n                //   imagePath    — new, relative path under the trajectory dir\n                //   bytesBase64  — legacy, inline base64\n                bytes?: unknown;\n                imagePath?: string;\n                bytesBase64?: string;\n              }\n            | { type: \"json\"; content: unknown }\n          >;\n        };\n        probeEvidence: PersistedProbeEvidence;\n      }\n    >;\n  };\n\n  const resolveWithinTrajectoryDir = (\n    candidate: string,\n    fieldName = \"screenshotPath\",\n  ): string => {\n    const resolved = path.resolve(trajectoryDir, candidate);\n    const relative = path.relative(trajectoryDir, resolved);\n    const outside =\n      relative === \"..\" ||\n      relative.startsWith(`..${path.sep}`) ||\n      path.isAbsolute(relative);\n\n    if (outside) {\n      throw new Error(\n        `Trajectory ${fieldName} escapes trajectory directory: ${candidate}`,\n      );\n    }\n\n    return resolved;\n  };\n\n  const hydrateProbeScreenshot = async (\n    probe: PersistedProbeEvidence | undefined,\n  ): Promise<void> => {\n    if (probe?.screenshotPath && !probe.screenshot) {\n      const resolved = resolveWithinTrajectoryDir(probe.screenshotPath);\n      try {\n        probe.screenshot = await fs.readFile(resolved);\n      } catch {\n        // Missing screenshot file: leave probe.screenshot unset. The verifier's\n        // evidence_insufficient path will handle it.\n      }\n    }\n  };\n\n  for (const step of parsed.steps) {\n    // Rehydrate tier-2 probe screenshot from its on-disk file reference.\n    await hydrateProbeScreenshot(step.probeEvidence);\n\n    // Rehydrate tier-1 image modalities. New shape: resolve imagePath → Buffer.\n    // Legacy shape: decode bytesBase64 → Buffer.\n    if (step.agentEvidence?.modalities) {\n      const resolved: AgentEvidenceModality[] = [];\n      for (const m of step.agentEvidence.modalities) {\n        if (m.type !== \"image\") {\n          resolved.push(m as AgentEvidenceModality);\n          continue;\n        }\n        const raw = m as unknown as {\n          imagePath?: string;\n          bytesBase64?: string;\n        };\n        if (typeof raw.imagePath === \"string\") {\n          const imagePath = resolveWithinTrajectoryDir(\n            raw.imagePath,\n            \"imagePath\",\n          );\n          try {\n            const bytes = await fs.readFile(imagePath);\n            resolved.push({ type: \"image\", bytes, mediaType: m.mediaType });\n          } catch {\n            // Missing tier-1 image: drop this modality. evidence_insufficient\n            // handles the downstream effect.\n          }\n        } else if (typeof raw.bytesBase64 === \"string\") {\n          resolved.push({\n            type: \"image\",\n            bytes: Buffer.from(raw.bytesBase64, \"base64\"),\n            mediaType: m.mediaType,\n          });\n        } else {\n          resolved.push(m as AgentEvidenceModality);\n        }\n      }\n      step.agentEvidence.modalities = resolved;\n    }\n  }\n\n  await hydrateProbeScreenshot(parsed.finalObservation);\n\n  return parsed;\n}\n\n/**\n * Build a `result*.json` filename for persisted evaluator output.\n *\n * Convention: the live run writes `result.json`; offline re-score attempts use\n * a label-based name (e.g., `result_rescore-2026-05-11.json`) so they coexist\n * without collisions and remain easy to diff.\n */\nexport function nextResultFilename(label?: string): string {\n  return `result_${normalizeResultLabel(label)}.json`;\n}\n\n/**\n * Default persistence policy: explicit override, then env, then \"on unless CI\".\n */\nexport function shouldPersistTrajectory(\n  override: boolean | undefined,\n): boolean {\n  if (override !== undefined) return override;\n  const env = process.env.VERIFIER_PERSIST_TRAJECTORIES?.toLowerCase();\n  if (env === \"1\" || env === \"true\") return true;\n  if (env === \"0\" || env === \"false\") return false;\n  return !process.env.CI;\n}\n\n/**\n * Write the on-disk trajectory layout under `dir`:\n *\n *   <dir>/\n *     ├── task_data.json\n *     ├── trajectory.json    (screenshots referenced by path)\n *     ├── screenshots/\n *     │   ├── probe/<N>.png\n *     │   └── agent/<N>[_M].png\n *     ├── scores/            (empty; populated separately)\n *     └── core.log\n *\n * Image bytes are externalized to PNG files; the in-memory Trajectory is left\n * untouched so callers can keep using it after persistence.\n */\nexport async function writeTrajectoryDir(\n  dir: string,\n  trajectory: Trajectory,\n): Promise<void> {\n  await fs.mkdir(dir, { recursive: true });\n  await fs.mkdir(path.join(dir, \"screenshots\", \"probe\"), { recursive: true });\n  await fs.mkdir(path.join(dir, \"screenshots\", \"agent\"), { recursive: true });\n\n  const serializableSteps: unknown[] = [];\n  // A single post-turn probe is fanned across every step of a multi-tool turn,\n  // and a single agent screenshot is shared across every action a CUA provider\n  // chose from it, so the same Buffer is shared by reference. Dedupe by\n  // identity: write the PNG once and point every sharing step at the same file.\n  const probePathByBuffer = new Map<Buffer, string>();\n  const agentPathByBuffer = new Map<Buffer, string>();\n  for (const [i, step] of trajectory.steps.entries()) {\n    const probe: ProbeEvidence = { ...step.probeEvidence };\n    if (probe.screenshot) {\n      let relPath = probePathByBuffer.get(probe.screenshot);\n      if (!relPath) {\n        relPath = `screenshots/probe/${i + 1}.png`;\n        await fs.writeFile(path.join(dir, relPath), probe.screenshot);\n        probePathByBuffer.set(probe.screenshot, relPath);\n      }\n      probe.screenshotPath = relPath;\n      delete probe.screenshot;\n    }\n\n    const imageModalities = step.agentEvidence.modalities.filter(\n      (m) => m.type === \"image\",\n    );\n    const multipleImages = imageModalities.length > 1;\n    let imageSeq = 0;\n    const modalities: unknown[] = [];\n    for (const m of step.agentEvidence.modalities) {\n      if (m.type !== \"image\") {\n        modalities.push(\n          m.type === \"json\"\n            ? {\n                ...m,\n                content: redactInlineImagePayloads(m.content, step.actionName),\n              }\n            : m,\n        );\n        continue;\n      }\n      let relPath = agentPathByBuffer.get(m.bytes);\n      if (!relPath) {\n        const suffix = multipleImages ? `_${imageSeq}` : \"\";\n        relPath = `screenshots/agent/${i + 1}${suffix}.png`;\n        await fs.writeFile(path.join(dir, relPath), m.bytes);\n        agentPathByBuffer.set(m.bytes, relPath);\n      }\n      modalities.push({\n        type: \"image\",\n        imagePath: relPath,\n        mediaType: m.mediaType,\n      });\n      imageSeq += 1;\n    }\n    serializableSteps.push({\n      ...step,\n      probeEvidence: probe,\n      agentEvidence: { modalities },\n      toolOutput: {\n        ...step.toolOutput,\n        result: redactInlineImagePayloads(\n          step.toolOutput.result,\n          step.actionName,\n        ),\n      },\n    });\n  }\n\n  const finalObservation: ProbeEvidence | undefined =\n    trajectory.finalObservation === undefined\n      ? undefined\n      : { ...trajectory.finalObservation };\n  if (finalObservation?.screenshot) {\n    const relPath = \"screenshots/probe/final.png\";\n    await fs.writeFile(path.join(dir, relPath), finalObservation.screenshot);\n    finalObservation.screenshotPath = relPath;\n    delete finalObservation.screenshot;\n  }\n\n  // Image modalities carry imagePath instead of raw bytes on disk; cast\n  // through unknown rather than widen Trajectory's type contract.\n  const serialized = {\n    ...trajectory,\n    steps: serializableSteps,\n    ...(finalObservation ? { finalObservation } : {}),\n  } as unknown;\n\n  await fs.writeFile(\n    path.join(dir, \"trajectory.json\"),\n    JSON.stringify(serialized, null, 2),\n  );\n\n  await fs.writeFile(\n    path.join(dir, \"task_data.json\"),\n    JSON.stringify(\n      {\n        task: trajectory.task,\n        status: trajectory.status,\n        finalAnswer: trajectory.finalAnswer ?? null,\n      },\n      null,\n      2,\n    ),\n  );\n\n  await fs.mkdir(path.join(dir, \"scores\"), { recursive: true });\n  await fs.writeFile(path.join(dir, \"core.log\"), coreLog(trajectory));\n}\n\nfunction coreLog(trajectory: Trajectory): string {\n  return (\n    trajectory.steps\n      .map((step, i) =>\n        JSON.stringify({\n          step: i,\n          action: step.actionName,\n          url: step.probeEvidence.url ?? null,\n          ok: step.toolOutput.ok,\n          reasoning: step.reasoning || undefined,\n        }),\n      )\n      .join(\"\\n\") + \"\\n\"\n  );\n}\n"]}