@browserbasehq/orca 3.5.1-preview.0 → 3.6.0-gemini-cua

package/dist/cjs/lib/v3/verifier/evidence.js

@@ -0,0 +1,432 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.isImageEvidence = isImageEvidence;
+exports.isTextEvidence = isTextEvidence;
+exports.loadAndReduceScreenshots = loadAndReduceScreenshots;
+exports.collectCanonicalEvidence = collectCanonicalEvidence;
+let sharpPromise = null;
+async function loadSharp() {
+    if (sharpPromise)
+        return sharpPromise;
+    sharpPromise = (async () => {
+        try {
+            // Dynamic specifier hides sharp from TypeScript's module resolver:
+            // sharp is an optional runtime dep (evals/server installs it; core
+            // consumers don't have to). try/catch handles missing-at-runtime.
+            const specifier = "sharp";
+            const mod = (await Promise.resolve(`${specifier}`).then(s => __importStar(require(s))));
+            return (mod.default ?? mod);
+        }
+        catch {
+            return null;
+        }
+    })();
+    return sharpPromise;
+}
+const DEFAULT_SSIM_THRESHOLD = 0.75;
+const DEFAULT_MSE_THRESHOLD = 30;
+const DEFAULT_IMAGE_RESIZE = 0.7;
+/** Discriminator helpers — kind === "image" for screenshots. */
+function isImageEvidence(e) {
+    return "bytes" in e && e.bytes instanceof Buffer;
+}
+function isTextEvidence(e) {
+    return ("content" in e && typeof e.content === "string");
+}
+/**
+ * Step 1 — load trajectory screenshots from disk (or memory), deduplicate,
+ * and downsize.
+ *
+ * Returns an array of canonical screenshots ready to feed into Step 2.
+ * Steps without a captured probe screenshot are skipped silently — they
+ * never reach the canonical array, but their action context still appears
+ * in the prompt's action history.
+ */
+async function loadAndReduceScreenshots(trajectory, opts = {}) {
+    const ssimThreshold = opts.ssimThreshold ?? DEFAULT_SSIM_THRESHOLD;
+    const mseThreshold = opts.mseThreshold ?? DEFAULT_MSE_THRESHOLD;
+    const imageResize = opts.imageResize ?? DEFAULT_IMAGE_RESIZE;
+    // Collect raw frames in chronological order. Per step we take tier-1
+    // agent-mirrored screenshots first (the exact bytes a CUA provider saw, plus
+    // any inline act() screenshots lifted into image modalities), then the
+    // tier-2 post-action probe. Buffers are populated either live or after
+    // loadTrajectoryFromDisk(); absent/empty buffers are skipped. Dedup collapses
+    // near-identical frames downstream, so including both tiers is safe.
+    const rawFrames = [];
+    for (let i = 0; i < trajectory.steps.length; i++) {
+        const step = trajectory.steps[i];
+        // tier-1: agent screenshots, in modality order, before the probe.
+        for (const m of step.agentEvidence?.modalities ?? []) {
+            if (m.type === "image" && m.bytes && m.bytes.length > 0) {
+                rawFrames.push({
+                    bytes: m.bytes,
+                    originalStepIndex: i,
+                });
+            }
+        }
+        // tier-2: post-action probe screenshot.
+        const buf = step.probeEvidence?.screenshot;
+        if (buf && buf.length > 0) {
+            rawFrames.push({
+                bytes: buf,
+                originalStepIndex: i,
+            });
+        }
+    }
+    // Terminal page observation captured after the agent finished. Preserves the
+    // legacy final-screenshot verification behavior; positioned after every step
+    // so it anchors as the trajectory's closing frame (always kept as "last").
+    const finalShot = trajectory.finalObservation?.screenshot;
+    if (finalShot && finalShot.length > 0) {
+        rawFrames.push({
+            bytes: finalShot,
+            originalStepIndex: trajectory.steps.length,
+        });
+    }
+    const sharp = await loadSharp();
+    const stepIndexToCanonical = new Map();
+    if (rawFrames.length === 0) {
+        return {
+            screenshots: [],
+            stepIndexToCanonical,
+            originalCount: 0,
+            keptCount: 0,
+            thresholds: {
+                ssim: ssimThreshold,
+                mse: mseThreshold,
+                resize: imageResize,
+            },
+        };
+    }
+    // ── Dedup ──────────────────────────────────────────────────────────────
+    // First + last always kept. Middle frames kept iff they're sufficiently
+    // dissimilar to the previously kept frame. Dissimilarity check mirrors
+    // ScreenshotCollector: quick MSE pass, escalate to SSIM only if MSE
+    // suggests the frames differ. If sharp is unavailable, keep everything.
+    const keptRaw = [];
+    // Track which raw-frame index each step maps to. Pre-fill with "the last
+    // kept frame so far" so dropped frames fall back to their surviving peer.
+    const rawIdxByStep = new Map();
+    let lastKeptIdx = 0;
+    for (let i = 0; i < rawFrames.length; i++) {
+        const frame = rawFrames[i];
+        const isFirst = i === 0;
+        const isLast = i === rawFrames.length - 1;
+        let keep = true;
+        let reason = sharp
+            ? "diverges"
+            : "no-dedup";
+        if (sharp && !isFirst && !isLast) {
+            const prev = keptRaw[keptRaw.length - 1];
+            try {
+                const mse = await calculateMSE(sharp, prev.bytes, frame.bytes);
+                if (mse < mseThreshold) {
+                    keep = false;
+                }
+                else {
+                    const ssim = await calculateSSIM(sharp, prev.bytes, frame.bytes);
+                    // Drop when "too similar" (SSIM at or above threshold). Mirrors
+                    // ScreenshotCollector.shouldKeep = ssim < threshold.
+                    if (ssim >= ssimThreshold)
+                        keep = false;
+                }
+            }
+            catch {
+                // Comparison error → keep the frame (safer to err on inclusion).
+                keep = true;
+            }
+        }
+        else if (isFirst) {
+            reason = "first";
+        }
+        else if (isLast) {
+            reason = "last";
+        }
+        if (keep) {
+            keptRaw.push({
+                bytes: frame.bytes,
+                originalStepIndex: frame.originalStepIndex,
+                keptReason: reason,
+            });
+            lastKeptIdx = keptRaw.length - 1;
+        }
+        rawIdxByStep.set(frame.originalStepIndex, lastKeptIdx);
+    }
+    // ── Resize ─────────────────────────────────────────────────────────────
+    const screenshots = await Promise.all(keptRaw.map(async (raw, canonicalIndex) => {
+        const resized = sharp
+            ? await resizePng(sharp, raw.bytes, imageResize)
+            : raw.bytes;
+        return {
+            canonicalIndex,
+            originalStepIndex: raw.originalStepIndex,
+            bytes: resized,
+            mediaType: "image/png",
+            keptReason: raw.keptReason,
+        };
+    }));
+    // Build step → canonical index lookup from the dropped-fallback table.
+    for (const [stepIdx, rawIdx] of rawIdxByStep.entries()) {
+        stepIndexToCanonical.set(stepIdx, rawIdx);
+    }
+    return {
+        screenshots,
+        stepIndexToCanonical,
+        originalCount: rawFrames.length,
+        keptCount: screenshots.length,
+        thresholds: {
+            ssim: ssimThreshold,
+            mse: mseThreshold,
+            resize: imageResize,
+        },
+    };
+}
+/**
+ * Collect a combined evidence-point list (images + ariaTree text snippets).
+ *
+ * Images (tier-1 agent screenshots + tier-2 probes + the final observation)
+ * go through {@link loadAndReduceScreenshots} (dedup + downscale).
+ * Text evidence is sourced from:
+ *   - tier-2 `probeEvidence.ariaTree` (and `finalObservation.ariaTree`)
+ *   - tier-1 text/json modalities in `agentEvidence`
+ *   - native `toolOutput.result`
+ *
+ * Text snippets are deduplicated on their full normalized content so a "stuck
+ * on the same page" agent doesn't produce a flood of identical snippets.
+ *
+ * `canonicalIndex` is unified across both kinds: the first image gets 0,
+ * the next image or text snippet gets 1, etc. The returned `loaded`
+ * (`screenshots[].canonicalIndex` and `stepIndexToCanonical`) is re-stamped
+ * into this same combined index space so every canonicalIndex in the result
+ * references one array.
+ */
+async function collectCanonicalEvidence(trajectory, opts = {}) {
+    const loaded = await loadAndReduceScreenshots(trajectory, opts);
+    const evidence = [];
+    const pending = [];
+    for (const shot of loaded.screenshots) {
+        pending.push({ kind: "image", shot });
+    }
+    const seenText = new Set();
+    const addTextEvidence = (stepIdx, source, text) => {
+        // Runtime guard, not just the type: trajectory JSON loaded from disk is
+        // unvalidated, so a malformed non-string field must not crash collection.
+        if (typeof text !== "string" || text.length === 0)
+            return;
+        const trimmed = text.length > 4000 ? text.slice(0, 4000) : text;
+        const normalized = trimmed.replace(/\s+/g, " ").trim();
+        if (normalized.length === 0)
+            return;
+        // Dedupe on the full normalized text — a length+prefix key would collapse
+        // distinct snippets that share a prefix. The 4k cap bounds key size.
+        if (seenText.has(normalized))
+            return;
+        seenText.add(normalized);
+        pending.push({
+            kind: "text",
+            stepIdx,
+            source,
+            content: trimmed,
+        });
+    };
+    for (let i = 0; i < trajectory.steps.length; i++) {
+        const step = trajectory.steps[i];
+        addTextEvidence(i, "probe-aria", step.probeEvidence?.ariaTree);
+        for (const modality of step.agentEvidence?.modalities ?? []) {
+            if (modality.type === "text") {
+                addTextEvidence(i, "agent-text", modality.content);
+            }
+            else if (modality.type === "json") {
+                addTextEvidence(i, "agent-json", safeStringifyEvidence(modality.content));
+            }
+        }
+        // Defensive: agentEvidence is derived from toolOutput today, but keeping
+        // the native result as a fallback preserves evidence if that mapping
+        // changes or an adapter omits modalities.
+        addTextEvidence(i, "tool-output", safeStringifyEvidence(step.toolOutput?.result));
+    }
+    // Terminal aria observation — mirrors the final screenshot frame so the
+    // closing page state is scoreable even when the last step had no probe.
+    addTextEvidence(trajectory.steps.length, "probe-aria", trajectory.finalObservation?.ariaTree);
+    // Sort by step position asc; ties → image before text so the
+    // "page state before the harness probed text" reads naturally.
+    pending.sort((a, b) => {
+        const pa = a.kind === "image" ? a.shot.originalStepIndex : a.stepIdx;
+        const pb = b.kind === "image" ? b.shot.originalStepIndex : b.stepIdx;
+        if (pa !== pb)
+            return pa - pb;
+        return a.kind === "image" ? -1 : 1;
+    });
+    // Stamp canonical indices in the combined ordering. Track the screenshot
+    // array-index → combined-index translation so the returned `loaded` can be
+    // re-mapped onto the same space (previously screenshots[].canonicalIndex and
+    // stepIndexToCanonical pointed into the images-only array and disagreed with
+    // `evidence` once text was interleaved).
+    const screenshotIdxToCombined = new Map();
+    for (const p of pending) {
+        if (p.kind === "image") {
+            const combinedIndex = evidence.length;
+            screenshotIdxToCombined.set(p.shot.canonicalIndex, combinedIndex);
+            evidence.push({ ...p.shot, canonicalIndex: combinedIndex });
+        }
+        else {
+            evidence.push({
+                canonicalIndex: evidence.length,
+                originalStepIndex: p.stepIdx,
+                source: p.source,
+                content: p.content,
+            });
+        }
+    }
+    const remappedScreenshots = loaded.screenshots.map((shot) => {
+        const combined = screenshotIdxToCombined.get(shot.canonicalIndex);
+        return combined === undefined
+            ? shot
+            : { ...shot, canonicalIndex: combined };
+    });
+    const remappedStepIndex = new Map();
+    for (const [stepIdx, screenshotIdx] of loaded.stepIndexToCanonical) {
+        const combined = screenshotIdxToCombined.get(screenshotIdx);
+        if (combined !== undefined)
+            remappedStepIndex.set(stepIdx, combined);
+    }
+    return {
+        evidence,
+        loaded: {
+            ...loaded,
+            screenshots: remappedScreenshots,
+            stepIndexToCanonical: remappedStepIndex,
+        },
+    };
+}
+// ─── Internals ────────────────────────────────────────────────────────────
+function safeStringifyEvidence(value) {
+    if (value === undefined || value === null)
+        return undefined;
+    // Strings pass through verbatim — JSON.stringify would wrap them in quotes.
+    if (typeof value === "string")
+        return value;
+    try {
+        return JSON.stringify(value);
+    }
+    catch {
+        return String(value);
+    }
+}
+/**
+ * Port of `imageResize` from packages/evals/utils. Re-encodes as PNG with
+ * palette + max compression so the downstream LLM call sees smaller bytes.
+ * No-op when sharp can't read the buffer dimensions.
+ */
+async function resizePng(sharp, bytes, scaleFactor) {
+    if (scaleFactor >= 0.999 && scaleFactor <= 1.001)
+        return bytes;
+    try {
+        const metadata = await sharp(bytes).metadata();
+        if (!metadata.width || !metadata.height)
+            return bytes;
+        const width = Math.max(1, Math.round(metadata.width * scaleFactor));
+        const height = Math.max(1, Math.round(metadata.height * scaleFactor));
+        return await sharp(bytes)
+            .resize(width, height, {
+            fit: "inside",
+            kernel: sharp.kernel.lanczos3,
+        })
+            .png({
+            compressionLevel: 9,
+            adaptiveFiltering: true,
+            palette: true,
+        })
+            .toBuffer();
+    }
+    catch {
+        return bytes;
+    }
+}
+/**
+ * Port of `ScreenshotCollector.calculateMSE`. Resamples both images to a
+ * fixed small size and computes the per-byte squared error mean. Lower
+ * means more similar.
+ */
+async function calculateMSE(sharp, img1, img2) {
+    const size = { width: 400, height: 300 };
+    const data1 = await sharp(img1).resize(size).raw().toBuffer();
+    const data2 = await sharp(img2).resize(size).raw().toBuffer();
+    if (data1.length !== data2.length)
+        return Number.MAX_SAFE_INTEGER;
+    let sum = 0;
+    for (let i = 0; i < data1.length; i++) {
+        const diff = data1[i] - data2[i];
+        sum += diff * diff;
+    }
+    return sum / data1.length;
+}
+/**
+ * Port of `ScreenshotCollector.calculateSSIM`. Simplified single-window
+ * SSIM on grayscale at 400×300. Returns a value in [0, 1] where 1 ==
+ * identical (after grayscale downsample).
+ */
+async function calculateSSIM(sharp, img1, img2) {
+    const size = { width: 400, height: 300 };
+    const gray1 = await sharp(img1).resize(size).grayscale().raw().toBuffer();
+    const gray2 = await sharp(img2).resize(size).grayscale().raw().toBuffer();
+    if (gray1.length !== gray2.length)
+        return 0;
+    const c1 = 0.01 * 0.01;
+    const c2 = 0.03 * 0.03;
+    let sum1 = 0;
+    let sum2 = 0;
+    let sum1Sq = 0;
+    let sum2Sq = 0;
+    let sum12 = 0;
+    const N = gray1.length;
+    for (let i = 0; i < N; i++) {
+        sum1 += gray1[i];
+        sum2 += gray2[i];
+        sum1Sq += gray1[i] * gray1[i];
+        sum2Sq += gray2[i] * gray2[i];
+        sum12 += gray1[i] * gray2[i];
+    }
+    const mean1 = sum1 / N;
+    const mean2 = sum2 / N;
+    const var1 = sum1Sq / N - mean1 * mean1;
+    const var2 = sum2Sq / N - mean2 * mean2;
+    const cov12 = sum12 / N - mean1 * mean2;
+    const numerator = (2 * mean1 * mean2 + c1) * (2 * cov12 + c2);
+    const denominator = (mean1 * mean1 + mean2 * mean2 + c1) * (var1 + var2 + c2);
+    return numerator / denominator;
+}
+//# sourceMappingURL=evidence.js.map

package/dist/cjs/lib/v3/verifier/evidence.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"evidence.js","sourceRoot":"","sources":["../../../../../lib/v3/verifier/evidence.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAkEA,0CAIC;AAED,wCAMC;AAWD,4DAgKC;AAqBD,4DAuIC;AA7WD,IAAI,YAAY,GAAiC,IAAI,CAAC;AAEtD,KAAK,UAAU,SAAS;IACtB,IAAI,YAAY;QAAE,OAAO,YAAY,CAAC;IACtC,YAAY,GAAG,CAAC,KAAK,IAA2B,EAAE;QAChD,IAAI,CAAC;YACH,mEAAmE;YACnE,mEAAmE;YACnE,kEAAkE;YAClE,MAAM,SAAS,GAAG,OAAO,CAAC;YAC1B,MAAM,GAAG,GAAG,CAAC,yBAAa,SAAS,uCAAC,CAE3B,CAAC;YACV,OAAO,CAAC,GAAG,CAAC,OAAO,IAAI,GAAG,CAAU,CAAC;QACvC,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC,CAAC,EAAE,CAAC;IACL,OAAO,YAAY,CAAC;AACtB,CAAC;AAED,MAAM,sBAAsB,GAAG,IAAI,CAAC;AACpC,MAAM,qBAAqB,GAAG,EAAE,CAAC;AACjC,MAAM,oBAAoB,GAAG,GAAG,CAAC;AAEjC,gEAAgE;AAChE,SAAgB,eAAe,CAC7B,CAAoB;IAEpB,OAAO,OAAO,IAAI,CAAC,IAAK,CAAyB,CAAC,KAAK,YAAY,MAAM,CAAC;AAC5E,CAAC;AAED,SAAgB,cAAc,CAC5B,CAAoB;IAEpB,OAAO,CACL,SAAS,IAAI,CAAC,IAAI,OAAQ,CAA2B,CAAC,OAAO,KAAK,QAAQ,CAC3E,CAAC;AACJ,CAAC;AAED;;;;;;;;GAQG;AACI,KAAK,UAAU,wBAAwB,CAC5C,UAAsB,EACtB,OAA4B,EAAE;IAE9B,MAAM,aAAa,GAAG,IAAI,CAAC,aAAa,IAAI,sBAAsB,CAAC;IACnE,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,IAAI,qBAAqB,CAAC;IAChE,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,IAAI,oBAAoB,CAAC;IAE7D,qEAAqE;IACrE,6EAA6E;IAC7E,uEAAuE;IACvE,uEAAuE;IACvE,8EAA8E;IAC9E,qEAAqE;IACrE,MAAM,SAAS,GAGV,EAAE,CAAC;IAER,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACjD,MAAM,IAAI,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QACjC,kEAAkE;QAClE,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,aAAa,EAAE,UAAU,IAAI,EAAE,EAAE,CAAC;YACrD,IAAI,CAAC,CAAC,IAAI,KAAK,OAAO,IAAI,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACxD,SAAS,CAAC,IAAI,CAAC;oBACb,KAAK,EAAE,CAAC,CAAC,KAAK;oBACd,iBAAiB,EAAE,CAAC;iBACrB,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QACD,wCAAwC;QACxC,MAAM,GAAG,GAAG,IAAI,CAAC,aAAa,EAAE,UAAU,CAAC;QAC3C,IAAI,GAAG,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1B,SAAS,CAAC,IAAI,CAAC;gBACb,KAAK,EAAE,GAAG;gBACV,iBAAiB,EAAE,CAAC;aACrB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,6EAA6E;IAC7E,6EAA6E;IAC7E,2EAA2E;IAC3E,MAAM,SAAS,GAAG,UAAU,CAAC,gBAAgB,EAAE,UAAU,CAAC;IAC1D,IAAI,SAAS,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtC,SAAS,CAAC,IAAI,CAAC;YACb,KAAK,EAAE,SAAS;YAChB,iBAAiB,EAAE,UAAU,CAAC,KAAK,CAAC,MAAM;SAC3C,CAAC,CAAC;IACL,CAAC;IAED,MAAM,KAAK,GAAG,MAAM,SAAS,EAAE,CAAC;IAChC,MAAM,oBAAoB,GAAG,IAAI,GAAG,EAAkB,CAAC;IAEvD,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC3B,OAAO;YACL,WAAW,EAAE,EAAE;YACf,oBAAoB;YACpB,aAAa,EAAE,CAAC;YAChB,SAAS,EAAE,CAAC;YACZ,UAAU,EAAE;gBACV,IAAI,EAAE,aAAa;gBACnB,GAAG,EAAE,YAAY;gBACjB,MAAM,EAAE,WAAW;aACpB;SACF,CAAC;IACJ,CAAC;IAED,0EAA0E;IAC1E,wEAAwE;IACxE,uEAAuE;IACvE,oEAAoE;IACpE,wEAAwE;IACxE,MAAM,OAAO,GAIR,EAAE,CAAC;IAER,yEAAyE;IACzE,0EAA0E;IAC1E,MAAM,YAAY,GAAG,IAAI,GAAG,EAAkB,CAAC;IAE/C,IAAI,WAAW,GAAG,CAAC,CAAC;IAEpB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC1C,MAAM,KAAK,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;QAC3B,MAAM,OAAO,GAAG,CAAC,KAAK,CAAC,CAAC;QACxB,MAAM,MAAM,GAAG,CAAC,KAAK,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC;QAE1C,IAAI,IAAI,GAAG,IAAI,CAAC;QAChB,IAAI,MAAM,GAAsC,KAAK;YACnD,CAAC,CAAC,UAAU;YACZ,CAAC,CAAC,UAAU,CAAC;QAEf,IAAI,KAAK,IAAI,CAAC,OAAO,IAAI,CAAC,MAAM,EAAE,CAAC;YACjC,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YACzC,IAAI,CAAC;gBACH,MAAM,GAAG,GAAG,MAAM,YAAY,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC;gBAC/D,IAAI,GAAG,GAAG,YAAY,EAAE,CAAC;oBACvB,IAAI,GAAG,KAAK,CAAC;gBACf,CAAC;qBAAM,CAAC;oBACN,MAAM,IAAI,GAAG,MAAM,aAAa,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC;oBACjE,gEAAgE;oBAChE,qDAAqD;oBACrD,IAAI,IAAI,IAAI,aAAa;wBAAE,IAAI,GAAG,KAAK,CAAC;gBAC1C,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,iEAAiE;gBACjE,IAAI,GAAG,IAAI,CAAC;YACd,CAAC;QACH,CAAC;aAAM,IAAI,OAAO,EAAE,CAAC;YACnB,MAAM,GAAG,OAAO,CAAC;QACnB,CAAC;aAAM,IAAI,MAAM,EAAE,CAAC;YAClB,MAAM,GAAG,MAAM,CAAC;QAClB,CAAC;QAED,IAAI,IAAI,EAAE,CAAC;YACT,OAAO,CAAC,IAAI,CAAC;gBACX,KAAK,EAAE,KAAK,CAAC,KAAK;gBAClB,iBAAiB,EAAE,KAAK,CAAC,iBAAiB;gBAC1C,UAAU,EAAE,MAAM;aACnB,CAAC,CAAC;YACH,WAAW,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;QACnC,CAAC;QACD,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,iBAAiB,EAAE,WAAW,CAAC,CAAC;IACzD,CAAC;IAED,0EAA0E;IAC1E,MAAM,WAAW,GAA0B,MAAM,OAAO,CAAC,GAAG,CAC1D,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,EAAE,cAAc,EAAgC,EAAE;QACtE,MAAM,OAAO,GAAG,KAAK;YACnB,CAAC,CAAC,MAAM,SAAS,CAAC,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,WAAW,CAAC;YAChD,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC;QACd,OAAO;YACL,cAAc;YACd,iBAAiB,EAAE,GAAG,CAAC,iBAAiB;YACxC,KAAK,EAAE,OAAO;YACd,SAAS,EAAE,WAAW;YACtB,UAAU,EAAE,GAAG,CAAC,UAAU;SAC3B,CAAC;IACJ,CAAC,CAAC,CACH,CAAC;IAEF,uEAAuE;IACvE,KAAK,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,IAAI,YAAY,CAAC,OAAO,EAAE,EAAE,CAAC;QACvD,oBAAoB,CAAC,GAAG,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IAC5C,CAAC;IAED,OAAO;QACL,WAAW;QACX,oBAAoB;QACpB,aAAa,EAAE,SAAS,CAAC,MAAM;QAC/B,SAAS,EAAE,WAAW,CAAC,MAAM;QAC7B,UAAU,EAAE;YACV,IAAI,EAAE,aAAa;YACnB,GAAG,EAAE,YAAY;YACjB,MAAM,EAAE,WAAW;SACpB;KACF,CAAC;AACJ,CAAC;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACI,KAAK,UAAU,wBAAwB,CAC5C,UAAsB,EACtB,OAA4B,EAAE;IAK9B,MAAM,MAAM,GAAG,MAAM,wBAAwB,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;IAChE,MAAM,QAAQ,GAAwB,EAAE,CAAC;IAazC,MAAM,OAAO,GAAc,EAAE,CAAC;IAE9B,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;QACtC,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;IACxC,CAAC;IAED,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAC;IACnC,MAAM,eAAe,GAAG,CACtB,OAAe,EACf,MAAuC,EACvC,IAAwB,EACxB,EAAE;QACF,wEAAwE;QACxE,0EAA0E;QAC1E,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAC1D,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAChE,MAAM,UAAU,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;QACvD,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QACpC,0EAA0E;QAC1E,qEAAqE;QACrE,IAAI,QAAQ,CAAC,GAAG,CAAC,UAAU,CAAC;YAAE,OAAO;QACrC,QAAQ,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QACzB,OAAO,CAAC,IAAI,CAAC;YACX,IAAI,EAAE,MAAM;YACZ,OAAO;YACP,MAAM;YACN,OAAO,EAAE,OAAO;SACjB,CAAC,CAAC;IACL,CAAC,CAAC;IAEF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACjD,MAAM,IAAI,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QACjC,eAAe,CAAC,CAAC,EAAE,YAAY,EAAE,IAAI,CAAC,aAAa,EAAE,QAAQ,CAAC,CAAC;QAE/D,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,aAAa,EAAE,UAAU,IAAI,EAAE,EAAE,CAAC;YAC5D,IAAI,QAAQ,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;gBAC7B,eAAe,CAAC,CAAC,EAAE,YAAY,EAAE,QAAQ,CAAC,OAAO,CAAC,CAAC;YACrD,CAAC;iBAAM,IAAI,QAAQ,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;gBACpC,eAAe,CACb,CAAC,EACD,YAAY,EACZ,qBAAqB,CAAC,QAAQ,CAAC,OAAO,CAAC,CACxC,CAAC;YACJ,CAAC;QACH,CAAC;QAED,yEAAyE;QACzE,qEAAqE;QACrE,0CAA0C;QAC1C,eAAe,CACb,CAAC,EACD,aAAa,EACb,qBAAqB,CAAC,IAAI,CAAC,UAAU,EAAE,MAAM,CAAC,CAC/C,CAAC;IACJ,CAAC;IAED,wEAAwE;IACxE,wEAAwE;IACxE,eAAe,CACb,UAAU,CAAC,KAAK,CAAC,MAAM,EACvB,YAAY,EACZ,UAAU,CAAC,gBAAgB,EAAE,QAAQ,CACtC,CAAC;IAEF,6DAA6D;IAC7D,+DAA+D;IAC/D,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACpB,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,KAAK,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;QACrE,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,KAAK,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;QACrE,IAAI,EAAE,KAAK,EAAE;YAAE,OAAO,EAAE,GAAG,EAAE,CAAC;QAC9B,OAAO,CAAC,CAAC,IAAI,KAAK,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACrC,CAAC,CAAC,CAAC;IAEH,yEAAyE;IACzE,2EAA2E;IAC3E,6EAA6E;IAC7E,6EAA6E;IAC7E,yCAAyC;IACzC,MAAM,uBAAuB,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC1D,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,IAAI,CAAC,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;YACvB,MAAM,aAAa,GAAG,QAAQ,CAAC,MAAM,CAAC;YACtC,uBAAuB,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,cAAc,EAAE,aAAa,CAAC,CAAC;YAClE,QAAQ,CAAC,IAAI,CAAC,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,cAAc,EAAE,aAAa,EAAE,CAAC,CAAC;QAC9D,CAAC;aAAM,CAAC;YACN,QAAQ,CAAC,IAAI,CAAC;gBACZ,cAAc,EAAE,QAAQ,CAAC,MAAM;gBAC/B,iBAAiB,EAAE,CAAC,CAAC,OAAO;gBAC5B,MAAM,EAAE,CAAC,CAAC,MAAM;gBAChB,OAAO,EAAE,CAAC,CAAC,OAAO;aACnB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,MAAM,mBAAmB,GAAG,MAAM,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QAC1D,MAAM,QAAQ,GAAG,uBAAuB,CAAC,GAAG,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QAClE,OAAO,QAAQ,KAAK,SAAS;YAC3B,CAAC,CAAC,IAAI;YACN,CAAC,CAAC,EAAE,GAAG,IAAI,EAAE,cAAc,EAAE,QAAQ,EAAE,CAAC;IAC5C,CAAC,CAAC,CAAC;IACH,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAAkB,CAAC;IACpD,KAAK,MAAM,CAAC,OAAO,EAAE,aAAa,CAAC,IAAI,MAAM,CAAC,oBAAoB,EAAE,CAAC;QACnE,MAAM,QAAQ,GAAG,uBAAuB,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;QAC5D,IAAI,QAAQ,KAAK,SAAS;YAAE,iBAAiB,CAAC,GAAG,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;IACvE,CAAC;IAED,OAAO;QACL,QAAQ;QACR,MAAM,EAAE;YACN,GAAG,MAAM;YACT,WAAW,EAAE,mBAAmB;YAChC,oBAAoB,EAAE,iBAAiB;SACxC;KACF,CAAC;AACJ,CAAC;AAED,6EAA6E;AAE7E,SAAS,qBAAqB,CAAC,KAAc;IAC3C,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,SAAS,CAAC;IAC5D,4EAA4E;IAC5E,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IAC5C,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;IAC/B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;IACvB,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,KAAK,UAAU,SAAS,CACtB,KAAY,EACZ,KAAa,EACb,WAAmB;IAEnB,IAAI,WAAW,IAAI,KAAK,IAAI,WAAW,IAAI,KAAK;QAAE,OAAO,KAAK,CAAC;IAC/D,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,KAAK,CAAC,CAAC,QAAQ,EAAE,CAAC;QAC/C,IAAI,CAAC,QAAQ,CAAC,KAAK,IAAI,CAAC,QAAQ,CAAC,MAAM;YAAE,OAAO,KAAK,CAAC;QACtD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,GAAG,WAAW,CAAC,CAAC,CAAC;QACpE,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,GAAG,WAAW,CAAC,CAAC,CAAC;QACtE,OAAO,MAAM,KAAK,CAAC,KAAK,CAAC;aACtB,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE;YACrB,GAAG,EAAE,QAAQ;YACb,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,QAAQ;SAC9B,CAAC;aACD,GAAG,CAAC;YACH,gBAAgB,EAAE,CAAC;YACnB,iBAAiB,EAAE,IAAI;YACvB,OAAO,EAAE,IAAI;SACd,CAAC;aACD,QAAQ,EAAE,CAAC;IAChB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,KAAK,UAAU,YAAY,CACzB,KAAY,EACZ,IAAY,EACZ,IAAY;IAEZ,MAAM,IAAI,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC;IACzC,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC;IAC9D,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC;IAC9D,IAAI,KAAK,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM;QAAE,OAAO,MAAM,CAAC,gBAAgB,CAAC;IAClE,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACjC,GAAG,IAAI,IAAI,GAAG,IAAI,CAAC;IACrB,CAAC;IACD,OAAO,GAAG,GAAG,KAAK,CAAC,MAAM,CAAC;AAC5B,CAAC;AAED;;;;GAIG;AACH,KAAK,UAAU,aAAa,CAC1B,KAAY,EACZ,IAAY,EACZ,IAAY;IAEZ,MAAM,IAAI,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC;IACzC,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC;IAC1E,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC;IAC1E,IAAI,KAAK,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM;QAAE,OAAO,CAAC,CAAC;IAE5C,MAAM,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC;IACvB,MAAM,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC;IACvB,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,MAAM,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC;IACvB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC3B,IAAI,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC;QACjB,IAAI,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC;QACjB,MAAM,IAAI,KAAK,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QAC9B,MAAM,IAAI,KAAK,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QAC9B,KAAK,IAAI,KAAK,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;IAC/B,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,CAAC;IACvB,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,CAAC;IACvB,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,GAAG,KAAK,GAAG,KAAK,CAAC;IACxC,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,GAAG,KAAK,GAAG,KAAK,CAAC;IACxC,MAAM,KAAK,GAAG,KAAK,GAAG,CAAC,GAAG,KAAK,GAAG,KAAK,CAAC;IACxC,MAAM,SAAS,GAAG,CAAC,CAAC,GAAG,KAAK,GAAG,KAAK,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,GAAG,KAAK,GAAG,EAAE,CAAC,CAAC;IAC9D,MAAM,WAAW,GAAG,CAAC,KAAK,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,GAAG,EAAE,CAAC,GAAG,CAAC,IAAI,GAAG,IAAI,GAAG,EAAE,CAAC,CAAC;IAC9E,OAAO,SAAS,GAAG,WAAW,CAAC;AACjC,CAAC","sourcesContent":["/**\n * Evidence loader: hydrate tier-1 agent screenshots, tier-2 probe screenshots,\n * and the terminal final observation from a trajectory; dedup near-identical\n * frames with MSE + SSIM; and downsize for the relevance LLM call. The first\n * and last frames are always kept so the verifier can cite the trajectory's\n * bookends. `sharp` is loaded dynamically; if unavailable, dedup/resize no-op.\n */\nimport type {\n  CanonicalEvidence,\n  CanonicalScreenshot,\n  CanonicalTextEvidence,\n  EvidenceLoadOptions,\n  EvidenceLoadResult,\n  Trajectory,\n} from \"./types.js\";\n\n// Lazy-loaded `sharp` namespace. When `sharp` is not installed, we fall back\n// to keep-everything-at-native-size. Keep this structural so core does not\n// need to publish sharp as a runtime dependency.\ninterface SharpImage {\n  metadata(): Promise<{ width?: number; height?: number }>;\n  resize(\n    width: number,\n    height?: number,\n    options?: { fit?: string; kernel?: unknown },\n  ): SharpImage;\n  resize(options: { width: number; height: number }): SharpImage;\n  raw(): SharpImage;\n  grayscale(): SharpImage;\n  png(options?: {\n    compressionLevel?: number;\n    adaptiveFiltering?: boolean;\n    palette?: boolean;\n  }): SharpImage;\n  toBuffer(): Promise<Buffer>;\n}\n\ntype Sharp = ((input: Buffer) => SharpImage) & {\n  kernel: { lanczos3: unknown };\n};\nlet sharpPromise: Promise<Sharp | null> | null = null;\n\nasync function loadSharp(): Promise<Sharp | null> {\n  if (sharpPromise) return sharpPromise;\n  sharpPromise = (async (): Promise<Sharp | null> => {\n    try {\n      // Dynamic specifier hides sharp from TypeScript's module resolver:\n      // sharp is an optional runtime dep (evals/server installs it; core\n      // consumers don't have to). try/catch handles missing-at-runtime.\n      const specifier = \"sharp\";\n      const mod = (await import(specifier)) as unknown as {\n        default?: Sharp;\n      } & Sharp;\n      return (mod.default ?? mod) as Sharp;\n    } catch {\n      return null;\n    }\n  })();\n  return sharpPromise;\n}\n\nconst DEFAULT_SSIM_THRESHOLD = 0.75;\nconst DEFAULT_MSE_THRESHOLD = 30;\nconst DEFAULT_IMAGE_RESIZE = 0.7;\n\n/** Discriminator helpers — kind === \"image\" for screenshots. */\nexport function isImageEvidence(\n  e: CanonicalEvidence,\n): e is CanonicalScreenshot {\n  return \"bytes\" in e && (e as CanonicalScreenshot).bytes instanceof Buffer;\n}\n\nexport function isTextEvidence(\n  e: CanonicalEvidence,\n): e is CanonicalTextEvidence {\n  return (\n    \"content\" in e && typeof (e as CanonicalTextEvidence).content === \"string\"\n  );\n}\n\n/**\n * Step 1 — load trajectory screenshots from disk (or memory), deduplicate,\n * and downsize.\n *\n * Returns an array of canonical screenshots ready to feed into Step 2.\n * Steps without a captured probe screenshot are skipped silently — they\n * never reach the canonical array, but their action context still appears\n * in the prompt's action history.\n */\nexport async function loadAndReduceScreenshots(\n  trajectory: Trajectory,\n  opts: EvidenceLoadOptions = {},\n): Promise<EvidenceLoadResult> {\n  const ssimThreshold = opts.ssimThreshold ?? DEFAULT_SSIM_THRESHOLD;\n  const mseThreshold = opts.mseThreshold ?? DEFAULT_MSE_THRESHOLD;\n  const imageResize = opts.imageResize ?? DEFAULT_IMAGE_RESIZE;\n\n  // Collect raw frames in chronological order. Per step we take tier-1\n  // agent-mirrored screenshots first (the exact bytes a CUA provider saw, plus\n  // any inline act() screenshots lifted into image modalities), then the\n  // tier-2 post-action probe. Buffers are populated either live or after\n  // loadTrajectoryFromDisk(); absent/empty buffers are skipped. Dedup collapses\n  // near-identical frames downstream, so including both tiers is safe.\n  const rawFrames: Array<{\n    bytes: Buffer;\n    originalStepIndex: number;\n  }> = [];\n\n  for (let i = 0; i < trajectory.steps.length; i++) {\n    const step = trajectory.steps[i];\n    // tier-1: agent screenshots, in modality order, before the probe.\n    for (const m of step.agentEvidence?.modalities ?? []) {\n      if (m.type === \"image\" && m.bytes && m.bytes.length > 0) {\n        rawFrames.push({\n          bytes: m.bytes,\n          originalStepIndex: i,\n        });\n      }\n    }\n    // tier-2: post-action probe screenshot.\n    const buf = step.probeEvidence?.screenshot;\n    if (buf && buf.length > 0) {\n      rawFrames.push({\n        bytes: buf,\n        originalStepIndex: i,\n      });\n    }\n  }\n\n  // Terminal page observation captured after the agent finished. Preserves the\n  // legacy final-screenshot verification behavior; positioned after every step\n  // so it anchors as the trajectory's closing frame (always kept as \"last\").\n  const finalShot = trajectory.finalObservation?.screenshot;\n  if (finalShot && finalShot.length > 0) {\n    rawFrames.push({\n      bytes: finalShot,\n      originalStepIndex: trajectory.steps.length,\n    });\n  }\n\n  const sharp = await loadSharp();\n  const stepIndexToCanonical = new Map<number, number>();\n\n  if (rawFrames.length === 0) {\n    return {\n      screenshots: [],\n      stepIndexToCanonical,\n      originalCount: 0,\n      keptCount: 0,\n      thresholds: {\n        ssim: ssimThreshold,\n        mse: mseThreshold,\n        resize: imageResize,\n      },\n    };\n  }\n\n  // ── Dedup ──────────────────────────────────────────────────────────────\n  // First + last always kept. Middle frames kept iff they're sufficiently\n  // dissimilar to the previously kept frame. Dissimilarity check mirrors\n  // ScreenshotCollector: quick MSE pass, escalate to SSIM only if MSE\n  // suggests the frames differ. If sharp is unavailable, keep everything.\n  const keptRaw: Array<{\n    bytes: Buffer;\n    originalStepIndex: number;\n    keptReason: CanonicalScreenshot[\"keptReason\"];\n  }> = [];\n\n  // Track which raw-frame index each step maps to. Pre-fill with \"the last\n  // kept frame so far\" so dropped frames fall back to their surviving peer.\n  const rawIdxByStep = new Map<number, number>();\n\n  let lastKeptIdx = 0;\n\n  for (let i = 0; i < rawFrames.length; i++) {\n    const frame = rawFrames[i];\n    const isFirst = i === 0;\n    const isLast = i === rawFrames.length - 1;\n\n    let keep = true;\n    let reason: CanonicalScreenshot[\"keptReason\"] = sharp\n      ? \"diverges\"\n      : \"no-dedup\";\n\n    if (sharp && !isFirst && !isLast) {\n      const prev = keptRaw[keptRaw.length - 1];\n      try {\n        const mse = await calculateMSE(sharp, prev.bytes, frame.bytes);\n        if (mse < mseThreshold) {\n          keep = false;\n        } else {\n          const ssim = await calculateSSIM(sharp, prev.bytes, frame.bytes);\n          // Drop when \"too similar\" (SSIM at or above threshold). Mirrors\n          // ScreenshotCollector.shouldKeep = ssim < threshold.\n          if (ssim >= ssimThreshold) keep = false;\n        }\n      } catch {\n        // Comparison error → keep the frame (safer to err on inclusion).\n        keep = true;\n      }\n    } else if (isFirst) {\n      reason = \"first\";\n    } else if (isLast) {\n      reason = \"last\";\n    }\n\n    if (keep) {\n      keptRaw.push({\n        bytes: frame.bytes,\n        originalStepIndex: frame.originalStepIndex,\n        keptReason: reason,\n      });\n      lastKeptIdx = keptRaw.length - 1;\n    }\n    rawIdxByStep.set(frame.originalStepIndex, lastKeptIdx);\n  }\n\n  // ── Resize ─────────────────────────────────────────────────────────────\n  const screenshots: CanonicalScreenshot[] = await Promise.all(\n    keptRaw.map(async (raw, canonicalIndex): Promise<CanonicalScreenshot> => {\n      const resized = sharp\n        ? await resizePng(sharp, raw.bytes, imageResize)\n        : raw.bytes;\n      return {\n        canonicalIndex,\n        originalStepIndex: raw.originalStepIndex,\n        bytes: resized,\n        mediaType: \"image/png\",\n        keptReason: raw.keptReason,\n      };\n    }),\n  );\n\n  // Build step → canonical index lookup from the dropped-fallback table.\n  for (const [stepIdx, rawIdx] of rawIdxByStep.entries()) {\n    stepIndexToCanonical.set(stepIdx, rawIdx);\n  }\n\n  return {\n    screenshots,\n    stepIndexToCanonical,\n    originalCount: rawFrames.length,\n    keptCount: screenshots.length,\n    thresholds: {\n      ssim: ssimThreshold,\n      mse: mseThreshold,\n      resize: imageResize,\n    },\n  };\n}\n\n/**\n * Collect a combined evidence-point list (images + ariaTree text snippets).\n *\n * Images (tier-1 agent screenshots + tier-2 probes + the final observation)\n * go through {@link loadAndReduceScreenshots} (dedup + downscale).\n * Text evidence is sourced from:\n *   - tier-2 `probeEvidence.ariaTree` (and `finalObservation.ariaTree`)\n *   - tier-1 text/json modalities in `agentEvidence`\n *   - native `toolOutput.result`\n *\n * Text snippets are deduplicated on their full normalized content so a \"stuck\n * on the same page\" agent doesn't produce a flood of identical snippets.\n *\n * `canonicalIndex` is unified across both kinds: the first image gets 0,\n * the next image or text snippet gets 1, etc. The returned `loaded`\n * (`screenshots[].canonicalIndex` and `stepIndexToCanonical`) is re-stamped\n * into this same combined index space so every canonicalIndex in the result\n * references one array.\n */\nexport async function collectCanonicalEvidence(\n  trajectory: Trajectory,\n  opts: EvidenceLoadOptions = {},\n): Promise<{\n  evidence: CanonicalEvidence[];\n  loaded: EvidenceLoadResult;\n}> {\n  const loaded = await loadAndReduceScreenshots(trajectory, opts);\n  const evidence: CanonicalEvidence[] = [];\n\n  // Interleave images and text by step position so the resulting array is\n  // (roughly) chronological. We collect texts per step, then merge by\n  // step position with images.\n  type Pending =\n    | { kind: \"image\"; shot: CanonicalScreenshot }\n    | {\n        kind: \"text\";\n        stepIdx: number;\n        source: CanonicalTextEvidence[\"source\"];\n        content: string;\n      };\n  const pending: Pending[] = [];\n\n  for (const shot of loaded.screenshots) {\n    pending.push({ kind: \"image\", shot });\n  }\n\n  const seenText = new Set<string>();\n  const addTextEvidence = (\n    stepIdx: number,\n    source: CanonicalTextEvidence[\"source\"],\n    text: string | undefined,\n  ) => {\n    // Runtime guard, not just the type: trajectory JSON loaded from disk is\n    // unvalidated, so a malformed non-string field must not crash collection.\n    if (typeof text !== \"string\" || text.length === 0) return;\n    const trimmed = text.length > 4000 ? text.slice(0, 4000) : text;\n    const normalized = trimmed.replace(/\\s+/g, \" \").trim();\n    if (normalized.length === 0) return;\n    // Dedupe on the full normalized text — a length+prefix key would collapse\n    // distinct snippets that share a prefix. The 4k cap bounds key size.\n    if (seenText.has(normalized)) return;\n    seenText.add(normalized);\n    pending.push({\n      kind: \"text\",\n      stepIdx,\n      source,\n      content: trimmed,\n    });\n  };\n\n  for (let i = 0; i < trajectory.steps.length; i++) {\n    const step = trajectory.steps[i];\n    addTextEvidence(i, \"probe-aria\", step.probeEvidence?.ariaTree);\n\n    for (const modality of step.agentEvidence?.modalities ?? []) {\n      if (modality.type === \"text\") {\n        addTextEvidence(i, \"agent-text\", modality.content);\n      } else if (modality.type === \"json\") {\n        addTextEvidence(\n          i,\n          \"agent-json\",\n          safeStringifyEvidence(modality.content),\n        );\n      }\n    }\n\n    // Defensive: agentEvidence is derived from toolOutput today, but keeping\n    // the native result as a fallback preserves evidence if that mapping\n    // changes or an adapter omits modalities.\n    addTextEvidence(\n      i,\n      \"tool-output\",\n      safeStringifyEvidence(step.toolOutput?.result),\n    );\n  }\n\n  // Terminal aria observation — mirrors the final screenshot frame so the\n  // closing page state is scoreable even when the last step had no probe.\n  addTextEvidence(\n    trajectory.steps.length,\n    \"probe-aria\",\n    trajectory.finalObservation?.ariaTree,\n  );\n\n  // Sort by step position asc; ties → image before text so the\n  // \"page state before the harness probed text\" reads naturally.\n  pending.sort((a, b) => {\n    const pa = a.kind === \"image\" ? a.shot.originalStepIndex : a.stepIdx;\n    const pb = b.kind === \"image\" ? b.shot.originalStepIndex : b.stepIdx;\n    if (pa !== pb) return pa - pb;\n    return a.kind === \"image\" ? -1 : 1;\n  });\n\n  // Stamp canonical indices in the combined ordering. Track the screenshot\n  // array-index → combined-index translation so the returned `loaded` can be\n  // re-mapped onto the same space (previously screenshots[].canonicalIndex and\n  // stepIndexToCanonical pointed into the images-only array and disagreed with\n  // `evidence` once text was interleaved).\n  const screenshotIdxToCombined = new Map<number, number>();\n  for (const p of pending) {\n    if (p.kind === \"image\") {\n      const combinedIndex = evidence.length;\n      screenshotIdxToCombined.set(p.shot.canonicalIndex, combinedIndex);\n      evidence.push({ ...p.shot, canonicalIndex: combinedIndex });\n    } else {\n      evidence.push({\n        canonicalIndex: evidence.length,\n        originalStepIndex: p.stepIdx,\n        source: p.source,\n        content: p.content,\n      });\n    }\n  }\n\n  const remappedScreenshots = loaded.screenshots.map((shot) => {\n    const combined = screenshotIdxToCombined.get(shot.canonicalIndex);\n    return combined === undefined\n      ? shot\n      : { ...shot, canonicalIndex: combined };\n  });\n  const remappedStepIndex = new Map<number, number>();\n  for (const [stepIdx, screenshotIdx] of loaded.stepIndexToCanonical) {\n    const combined = screenshotIdxToCombined.get(screenshotIdx);\n    if (combined !== undefined) remappedStepIndex.set(stepIdx, combined);\n  }\n\n  return {\n    evidence,\n    loaded: {\n      ...loaded,\n      screenshots: remappedScreenshots,\n      stepIndexToCanonical: remappedStepIndex,\n    },\n  };\n}\n\n// ─── Internals ────────────────────────────────────────────────────────────\n\nfunction safeStringifyEvidence(value: unknown): string | undefined {\n  if (value === undefined || value === null) return undefined;\n  // Strings pass through verbatim — JSON.stringify would wrap them in quotes.\n  if (typeof value === \"string\") return value;\n  try {\n    return JSON.stringify(value);\n  } catch {\n    return String(value);\n  }\n}\n\n/**\n * Port of `imageResize` from packages/evals/utils. Re-encodes as PNG with\n * palette + max compression so the downstream LLM call sees smaller bytes.\n * No-op when sharp can't read the buffer dimensions.\n */\nasync function resizePng(\n  sharp: Sharp,\n  bytes: Buffer,\n  scaleFactor: number,\n): Promise<Buffer> {\n  if (scaleFactor >= 0.999 && scaleFactor <= 1.001) return bytes;\n  try {\n    const metadata = await sharp(bytes).metadata();\n    if (!metadata.width || !metadata.height) return bytes;\n    const width = Math.max(1, Math.round(metadata.width * scaleFactor));\n    const height = Math.max(1, Math.round(metadata.height * scaleFactor));\n    return await sharp(bytes)\n      .resize(width, height, {\n        fit: \"inside\",\n        kernel: sharp.kernel.lanczos3,\n      })\n      .png({\n        compressionLevel: 9,\n        adaptiveFiltering: true,\n        palette: true,\n      })\n      .toBuffer();\n  } catch {\n    return bytes;\n  }\n}\n\n/**\n * Port of `ScreenshotCollector.calculateMSE`. Resamples both images to a\n * fixed small size and computes the per-byte squared error mean. Lower\n * means more similar.\n */\nasync function calculateMSE(\n  sharp: Sharp,\n  img1: Buffer,\n  img2: Buffer,\n): Promise<number> {\n  const size = { width: 400, height: 300 };\n  const data1 = await sharp(img1).resize(size).raw().toBuffer();\n  const data2 = await sharp(img2).resize(size).raw().toBuffer();\n  if (data1.length !== data2.length) return Number.MAX_SAFE_INTEGER;\n  let sum = 0;\n  for (let i = 0; i < data1.length; i++) {\n    const diff = data1[i] - data2[i];\n    sum += diff * diff;\n  }\n  return sum / data1.length;\n}\n\n/**\n * Port of `ScreenshotCollector.calculateSSIM`. Simplified single-window\n * SSIM on grayscale at 400×300. Returns a value in [0, 1] where 1 ==\n * identical (after grayscale downsample).\n */\nasync function calculateSSIM(\n  sharp: Sharp,\n  img1: Buffer,\n  img2: Buffer,\n): Promise<number> {\n  const size = { width: 400, height: 300 };\n  const gray1 = await sharp(img1).resize(size).grayscale().raw().toBuffer();\n  const gray2 = await sharp(img2).resize(size).grayscale().raw().toBuffer();\n  if (gray1.length !== gray2.length) return 0;\n\n  const c1 = 0.01 * 0.01;\n  const c2 = 0.03 * 0.03;\n  let sum1 = 0;\n  let sum2 = 0;\n  let sum1Sq = 0;\n  let sum2Sq = 0;\n  let sum12 = 0;\n  const N = gray1.length;\n  for (let i = 0; i < N; i++) {\n    sum1 += gray1[i];\n    sum2 += gray2[i];\n    sum1Sq += gray1[i] * gray1[i];\n    sum2Sq += gray2[i] * gray2[i];\n    sum12 += gray1[i] * gray2[i];\n  }\n  const mean1 = sum1 / N;\n  const mean2 = sum2 / N;\n  const var1 = sum1Sq / N - mean1 * mean1;\n  const var2 = sum2Sq / N - mean2 * mean2;\n  const cov12 = sum12 / N - mean1 * mean2;\n  const numerator = (2 * mean1 * mean2 + c1) * (2 * cov12 + c2);\n  const denominator = (mean1 * mean1 + mean2 * mean2 + c1) * (var1 + var2 + c2);\n  return numerator / denominator;\n}\n"]}

package/dist/cjs/lib/v3/verifier/index.d.ts

@@ -1,6 +1,6 @@
 /**
  * Public re-exports for the verifier subsystem.
  */
-export type { AgentEvidence, AgentEvidenceModality, CriterionScore, EvaluationResult, FirstPointOfFailure, ProbeEvidence, Rubric, RubricCriterion, TaskSpec, TaskValidity, ToolOutput, Trajectory, TrajectoryStatus, TrajectoryStep, TrajectoryUsage, Verifier, VerifierFinding, VerifierRawSteps, } from "./types.js";
+export type { AgentEvidence, AgentEvidenceModality, CanonicalEvidence, CanonicalScreenshot, CanonicalTextEvidence, CriterionScore, EvaluationResult, ErrorTaxonomyCategory, ErrorTaxonomySubCategory, EvidenceLoadOptions, EvidenceLoadResult, FirstPointOfFailure, ParseFailureStepNumbersOptions, ProbeEvidence, Rubric, RubricCriterion, RubricVerifierOptions, TaskSpec, TaskValidity, ToolOutput, Trajectory, TrajectoryStatus, TrajectoryStep, TrajectoryUsage, Verifier, VerifierConfig, VerifierFinding, VerifierRawSteps, } from "./types.js";
 export { buildAgentEvidenceFromStepFinished, collectInlineImagePayloads, mergeAgentEvidence, redactInlineImagePayloads, REDACTED_INLINE_IMAGE, } from "./evidenceNormalization.js";
 export { loadTrajectoryFromDisk, nextResultFilename, normalizeRubric, shouldPersistTrajectory, writeTrajectoryDir, } from "./trajectory.js";

package/dist/cjs/lib/v3/verifier/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../../lib/v3/verifier/index.ts"],"names":[],"mappings":";;;AAuBA,uEAMoC;AALlC,8IAAA,kCAAkC,OAAA;AAClC,sIAAA,0BAA0B,OAAA;AAC1B,8HAAA,kBAAkB,OAAA;AAClB,qIAAA,yBAAyB,OAAA;AACzB,iIAAA,qBAAqB,OAAA;AAEvB,iDAMyB;AALvB,uHAAA,sBAAsB,OAAA;AACtB,mHAAA,kBAAkB,OAAA;AAClB,gHAAA,eAAe,OAAA;AACf,wHAAA,uBAAuB,OAAA;AACvB,mHAAA,kBAAkB,OAAA","sourcesContent":["/**\n * Public re-exports for the verifier subsystem.\n */\nexport type {\n  AgentEvidence,\n  AgentEvidenceModality,\n  CriterionScore,\n  EvaluationResult,\n  FirstPointOfFailure,\n  ProbeEvidence,\n  Rubric,\n  RubricCriterion,\n  TaskSpec,\n  TaskValidity,\n  ToolOutput,\n  Trajectory,\n  TrajectoryStatus,\n  TrajectoryStep,\n  TrajectoryUsage,\n  Verifier,\n  VerifierFinding,\n  VerifierRawSteps,\n} from \"./types.js\";\nexport {\n  buildAgentEvidenceFromStepFinished,\n  collectInlineImagePayloads,\n  mergeAgentEvidence,\n  redactInlineImagePayloads,\n  REDACTED_INLINE_IMAGE,\n} from \"./evidenceNormalization.js\";\nexport {\n  loadTrajectoryFromDisk,\n  nextResultFilename,\n  normalizeRubric,\n  shouldPersistTrajectory,\n  writeTrajectoryDir,\n} from \"./trajectory.js\";\n"]}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../../lib/v3/verifier/index.ts"],"names":[],"mappings":";;;AAiCA,uEAMoC;AALlC,8IAAA,kCAAkC,OAAA;AAClC,sIAAA,0BAA0B,OAAA;AAC1B,8HAAA,kBAAkB,OAAA;AAClB,qIAAA,yBAAyB,OAAA;AACzB,iIAAA,qBAAqB,OAAA;AAEvB,iDAMyB;AALvB,uHAAA,sBAAsB,OAAA;AACtB,mHAAA,kBAAkB,OAAA;AAClB,gHAAA,eAAe,OAAA;AACf,wHAAA,uBAAuB,OAAA;AACvB,mHAAA,kBAAkB,OAAA","sourcesContent":["/**\n * Public re-exports for the verifier subsystem.\n */\nexport type {\n  AgentEvidence,\n  AgentEvidenceModality,\n  CanonicalEvidence,\n  CanonicalScreenshot,\n  CanonicalTextEvidence,\n  CriterionScore,\n  EvaluationResult,\n  ErrorTaxonomyCategory,\n  ErrorTaxonomySubCategory,\n  EvidenceLoadOptions,\n  EvidenceLoadResult,\n  FirstPointOfFailure,\n  ParseFailureStepNumbersOptions,\n  ProbeEvidence,\n  Rubric,\n  RubricCriterion,\n  RubricVerifierOptions,\n  TaskSpec,\n  TaskValidity,\n  ToolOutput,\n  Trajectory,\n  TrajectoryStatus,\n  TrajectoryStep,\n  TrajectoryUsage,\n  Verifier,\n  VerifierConfig,\n  VerifierFinding,\n  VerifierRawSteps,\n} from \"./types.js\";\nexport {\n  buildAgentEvidenceFromStepFinished,\n  collectInlineImagePayloads,\n  mergeAgentEvidence,\n  redactInlineImagePayloads,\n  REDACTED_INLINE_IMAGE,\n} from \"./evidenceNormalization.js\";\nexport {\n  loadTrajectoryFromDisk,\n  nextResultFilename,\n  normalizeRubric,\n  shouldPersistTrajectory,\n  writeTrajectoryDir,\n} from \"./trajectory.js\";\n"]}

package/dist/cjs/lib/v3/verifier/prompts/batchedRelevance.d.ts

@@ -0,0 +1,21 @@
+/**
+ * Batched evidence-relevance prompt — replacement for the per-screenshot
+ * MM_SCREENSHOT_CRITERION_RELEVANCE_PROMPT.
+ *
+ * One call grades the relevance of B evidence points (mixed image + text)
+ * against all N rubric criteria. Reduces Step 2 cost from M calls to
+ * ⌈M / B⌉ calls. Each evidence point in the batch is labelled with an
+ * `evidence_idx` (0..B-1) the model must echo back so we can join the
+ * scores to the right evidence.
+ *
+ * Variables:
+ *   - task_definition    — task instruction string
+ *   - init_url_context   — optional "Starting URL: ..." appendix
+ *   - rubric_criteria    — numbered list of criteria ("\n{idx}. **{name}**\n   Description: {desc}\n")
+ *   - evidence_manifest  — textual list describing each evidence point in the
+ *                          batch: kind (image/text), source (probe/agent),
+ *                          step index, brief preview. Lets the model line up
+ *                          the inline images/text with the `evidence_idx`
+ *                          it's expected to score.
+ */
+export declare const MM_BATCHED_RELEVANCE_PROMPT = "Task: $task_definition$init_url_context\n\nYou are analyzing a batch of evidence points (screenshots and text snippets) from an agent's trajectory to determine which rubric criteria each evidence point helps evaluate.\n\n**Rubric Criteria:**\n$rubric_criteria\n\n**Evidence Points in This Batch:**\n$evidence_manifest\n\nThe evidence is presented to you in order: each image / text block in this message corresponds to one entry in the manifest above, identified by its `evidence_idx`.\n\n**Your Task:**\nFor EACH evidence point in the batch, assign a relevance score from 0\u201310 against EACH criterion.\n\n**Scoring Guidelines:**\n- **10**: Evidence directly shows critical proof for this criterion (e.g., shows the exact item, cart contents, confirmation page, the filled form field).\n- **7-9**: Evidence shows important contextual information for this criterion (search results, applied filters, navigation state).\n- **4-6**: Evidence shows partial / related information for this criterion.\n- **1-3**: Evidence shows minimal relevance to this criterion.\n- **0**: Evidence is completely irrelevant to this criterion.\n\n**Important:**\n- An evidence point can be highly relevant to multiple criteria.\n- Focus on what is VISIBLE in the screenshot or PRESENT in the text snippet, not what the agent claimed.\n- Text-form evidence (e.g., accessibility tree snippets, agent text, JSON tool outputs) is especially relevant for criteria asking \"is this field filled with X?\", \"does the page contain Y?\", or \"did the tool return X?\" \u2014 score it accordingly.\n\n**Output Format:**\nOutput a JSON object with an `items` list. One entry per evidence point in the batch:\n\n{{\n  \"items\": [\n    {{\n      \"evidence_idx\": 0,\n      \"scores\": [\n        {{ \"criterion_idx\": 0, \"score\": 7 }},\n        {{ \"criterion_idx\": 1, \"score\": 2 }}\n      ]\n    }},\n    ...one entry per evidence point...\n  ]\n}}\n\nYou MUST include an entry for every `evidence_idx` listed in the manifest, and every criterion_idx in each `scores` list.\n\nDO NOT OUTPUT ANYTHING OTHER THAN JSON.\n";

package/dist/cjs/lib/v3/verifier/prompts/batchedRelevance.js

@@ -0,0 +1,71 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MM_BATCHED_RELEVANCE_PROMPT = void 0;
+/**
+ * Batched evidence-relevance prompt — replacement for the per-screenshot
+ * MM_SCREENSHOT_CRITERION_RELEVANCE_PROMPT.
+ *
+ * One call grades the relevance of B evidence points (mixed image + text)
+ * against all N rubric criteria. Reduces Step 2 cost from M calls to
+ * ⌈M / B⌉ calls. Each evidence point in the batch is labelled with an
+ * `evidence_idx` (0..B-1) the model must echo back so we can join the
+ * scores to the right evidence.
+ *
+ * Variables:
+ *   - task_definition    — task instruction string
+ *   - init_url_context   — optional "Starting URL: ..." appendix
+ *   - rubric_criteria    — numbered list of criteria ("\n{idx}. **{name}**\n   Description: {desc}\n")
+ *   - evidence_manifest  — textual list describing each evidence point in the
+ *                          batch: kind (image/text), source (probe/agent),
+ *                          step index, brief preview. Lets the model line up
+ *                          the inline images/text with the `evidence_idx`
+ *                          it's expected to score.
+ */
+exports.MM_BATCHED_RELEVANCE_PROMPT = `Task: $task_definition$init_url_context
+
+You are analyzing a batch of evidence points (screenshots and text snippets) from an agent's trajectory to determine which rubric criteria each evidence point helps evaluate.
+
+**Rubric Criteria:**
+$rubric_criteria
+
+**Evidence Points in This Batch:**
+$evidence_manifest
+
+The evidence is presented to you in order: each image / text block in this message corresponds to one entry in the manifest above, identified by its \`evidence_idx\`.
+
+**Your Task:**
+For EACH evidence point in the batch, assign a relevance score from 0–10 against EACH criterion.
+
+**Scoring Guidelines:**
+- **10**: Evidence directly shows critical proof for this criterion (e.g., shows the exact item, cart contents, confirmation page, the filled form field).
+- **7-9**: Evidence shows important contextual information for this criterion (search results, applied filters, navigation state).
+- **4-6**: Evidence shows partial / related information for this criterion.
+- **1-3**: Evidence shows minimal relevance to this criterion.
+- **0**: Evidence is completely irrelevant to this criterion.
+
+**Important:**
+- An evidence point can be highly relevant to multiple criteria.
+- Focus on what is VISIBLE in the screenshot or PRESENT in the text snippet, not what the agent claimed.
+- Text-form evidence (e.g., accessibility tree snippets, agent text, JSON tool outputs) is especially relevant for criteria asking "is this field filled with X?", "does the page contain Y?", or "did the tool return X?" — score it accordingly.
+
+**Output Format:**
+Output a JSON object with an \`items\` list. One entry per evidence point in the batch:
+
+{{
+  "items": [
+    {{
+      "evidence_idx": 0,
+      "scores": [
+        {{ "criterion_idx": 0, "score": 7 }},
+        {{ "criterion_idx": 1, "score": 2 }}
+      ]
+    }},
+    ...one entry per evidence point...
+  ]
+}}
+
+You MUST include an entry for every \`evidence_idx\` listed in the manifest, and every criterion_idx in each \`scores\` list.
+
+DO NOT OUTPUT ANYTHING OTHER THAN JSON.
+`;
+//# sourceMappingURL=batchedRelevance.js.map

package/dist/cjs/lib/v3/verifier/prompts/batchedRelevance.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"batchedRelevance.js","sourceRoot":"","sources":["../../../../../../lib/v3/verifier/prompts/batchedRelevance.ts"],"names":[],"mappings":";;;AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AACU,QAAA,2BAA2B,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA8C1C,CAAC","sourcesContent":["/**\n * Batched evidence-relevance prompt — replacement for the per-screenshot\n * MM_SCREENSHOT_CRITERION_RELEVANCE_PROMPT.\n *\n * One call grades the relevance of B evidence points (mixed image + text)\n * against all N rubric criteria. Reduces Step 2 cost from M calls to\n * ⌈M / B⌉ calls. Each evidence point in the batch is labelled with an\n * `evidence_idx` (0..B-1) the model must echo back so we can join the\n * scores to the right evidence.\n *\n * Variables:\n *   - task_definition    — task instruction string\n *   - init_url_context   — optional \"Starting URL: ...\" appendix\n *   - rubric_criteria    — numbered list of criteria (\"\\n{idx}. **{name}**\\n   Description: {desc}\\n\")\n *   - evidence_manifest  — textual list describing each evidence point in the\n *                          batch: kind (image/text), source (probe/agent),\n *                          step index, brief preview. Lets the model line up\n *                          the inline images/text with the `evidence_idx`\n *                          it's expected to score.\n */\nexport const MM_BATCHED_RELEVANCE_PROMPT = `Task: $task_definition$init_url_context\n\nYou are analyzing a batch of evidence points (screenshots and text snippets) from an agent's trajectory to determine which rubric criteria each evidence point helps evaluate.\n\n**Rubric Criteria:**\n$rubric_criteria\n\n**Evidence Points in This Batch:**\n$evidence_manifest\n\nThe evidence is presented to you in order: each image / text block in this message corresponds to one entry in the manifest above, identified by its \\`evidence_idx\\`.\n\n**Your Task:**\nFor EACH evidence point in the batch, assign a relevance score from 0–10 against EACH criterion.\n\n**Scoring Guidelines:**\n- **10**: Evidence directly shows critical proof for this criterion (e.g., shows the exact item, cart contents, confirmation page, the filled form field).\n- **7-9**: Evidence shows important contextual information for this criterion (search results, applied filters, navigation state).\n- **4-6**: Evidence shows partial / related information for this criterion.\n- **1-3**: Evidence shows minimal relevance to this criterion.\n- **0**: Evidence is completely irrelevant to this criterion.\n\n**Important:**\n- An evidence point can be highly relevant to multiple criteria.\n- Focus on what is VISIBLE in the screenshot or PRESENT in the text snippet, not what the agent claimed.\n- Text-form evidence (e.g., accessibility tree snippets, agent text, JSON tool outputs) is especially relevant for criteria asking \"is this field filled with X?\", \"does the page contain Y?\", or \"did the tool return X?\" — score it accordingly.\n\n**Output Format:**\nOutput a JSON object with an \\`items\\` list. One entry per evidence point in the batch:\n\n{{\n  \"items\": [\n    {{\n      \"evidence_idx\": 0,\n      \"scores\": [\n        {{ \"criterion_idx\": 0, \"score\": 7 }},\n        {{ \"criterion_idx\": 1, \"score\": 2 }}\n      ]\n    }},\n    ...one entry per evidence point...\n  ]\n}}\n\nYou MUST include an entry for every \\`evidence_idx\\` listed in the manifest, and every criterion_idx in each \\`scores\\` list.\n\nDO NOT OUTPUT ANYTHING OTHER THAN JSON.\n`;\n"]}

package/dist/cjs/lib/v3/verifier/prompts/firstPointOfFailure.d.ts

@@ -0,0 +1,3 @@
+import type { ParseFailureStepNumbersOptions } from "../types.js";
+export declare const FIRST_POINT_OF_FAILURE_PROMPT: string;
+export declare function parseFailureStepNumbers(raw: string, opts?: ParseFailureStepNumbersOptions): number[];