cli-jaw 2.17.41 → 2.17.42

package/dist/src/browser/grounding-candidate.js

@@ -0,0 +1,272 @@
+/**
+ * src/browser/grounding-candidate.ts — the shape a grounding answer takes, and
+ * how it is recovered from model output.
+ *
+ * Two things were wrong with the previous extractor, and both are structural
+ * rather than cosmetic.
+ *
+ * It matched with `/\{[^{}]*"found"...\}/`. The `[^{}]*` class cannot cross a
+ * nested object, so a well-formed `{"found":true,"bbox":{...},"point":{...}}`
+ * could never match at all — the richer schema was unreachable no matter what
+ * the model returned. It also required canonical key order and rejected
+ * negative numbers.
+ *
+ * And nothing bounded the coordinate. A hallucinated `{"x":99999}` was
+ * converted and clicked.
+ *
+ * On bbox: it is carried for verification, cropping, and size sanity — NOT as
+ * an accuracy mechanism. A controlled comparison (GUI-Actor, same training
+ * recipe) scores bbox output below point output, so asking a model for a box
+ * does not make its answer better. The box earns its place by letting us
+ * reconcile against element geometry and re-crop for a second look.
+ */
+/** Below this, a candidate must be verified before it is acted on. */
+export const LOW_CONFIDENCE_THRESHOLD = 0.75;
+/** A point-only answer carries no extent, so it starts as a coin flip. */
+const POINT_ONLY_CONFIDENCE = 0.5;
+const BBOX_CONFIDENCE = 0.8;
+/**
+ * Beyond this, a coordinate is not a plausible screen position. Finiteness is
+ * too weak a gate: `1e308` is finite, and a bbox that size yields a center of
+ * `5e307` which then flows into a click.
+ */
+export const MAX_PLAUSIBLE_COORDINATE = 100_000;
+function plausible(n) {
+    return Math.abs(n) <= MAX_PLAUSIBLE_COORDINATE;
+}
+function isRecord(value) {
+    return Boolean(value && typeof value === 'object' && !Array.isArray(value));
+}
+function finiteNumber(value) {
+    return typeof value === 'number' && Number.isFinite(value) ? value : null;
+}
+function normalizePoint(value) {
+    if (!isRecord(value))
+        return null;
+    const x = finiteNumber(value['x']);
+    const y = finiteNumber(value['y']);
+    if (x === null || y === null)
+        return null;
+    return plausible(x) && plausible(y) ? { x, y } : null;
+}
+function normalizeBBox(value) {
+    if (!isRecord(value))
+        return null;
+    const x = finiteNumber(value['x']);
+    const y = finiteNumber(value['y']);
+    const width = finiteNumber(value['width']);
+    const height = finiteNumber(value['height']);
+    if (x === null || y === null || width === null || height === null)
+        return null;
+    // A zero-area box carries no extent, which is the only thing a box is for.
+    if (width <= 0 || height <= 0)
+        return null;
+    // A box larger than any screen is a confused answer, not a big target.
+    if (![x, y, width, height].every(plausible))
+        return null;
+    return { x, y, width, height };
+}
+function centerOfBBox(bbox) {
+    return { x: Math.round(bbox.x + bbox.width / 2), y: Math.round(bbox.y + bbox.height / 2) };
+}
+function clampConfidence(value, fallback) {
+    const n = finiteNumber(value);
+    return n === null ? fallback : Math.max(0, Math.min(1, n));
+}
+/**
+ * Accepts the current shape and the legacy `{found,x,y}` one.
+ *
+ * A legacy answer is normalized rather than rejected, but it is marked:
+ * `kind: 'coordinate'`, `point_only` in riskFlags, and a confidence that fails
+ * the gate. It says where to click without saying how big the thing is, and
+ * that difference should survive into the decision.
+ */
+export function normalizeCandidate(raw) {
+    if (!isRecord(raw))
+        return null;
+    if (typeof raw['found'] !== 'boolean')
+        return null;
+    if (!raw['found']) {
+        return {
+            schemaVersion: 'grounding-candidate-v1',
+            found: false,
+            kind: 'not_found',
+            bbox: null,
+            point: { x: 0, y: 0 },
+            confidence: 0,
+            description: typeof raw['description'] === 'string' ? raw['description'] : 'not found',
+            reason: typeof raw['reason'] === 'string' ? raw['reason'] : 'target_not_found',
+            riskFlags: [],
+        };
+    }
+    const bbox = normalizeBBox(raw['bbox']);
+    const explicitPoint = normalizePoint(raw['point']);
+    const legacyPoint = normalizePoint(raw); // top-level {x, y}
+    const point = explicitPoint ?? legacyPoint ?? (bbox ? centerOfBBox(bbox) : null);
+    if (!point)
+        return null;
+    // Two stated locations that disagree is evidence of a confused answer.
+    // Prefer the richer field, but say so rather than resolving it silently.
+    const disagrees = Boolean(explicitPoint && legacyPoint &&
+        (explicitPoint.x !== legacyPoint.x || explicitPoint.y !== legacyPoint.y));
+    const pointOnly = !bbox;
+    const candidate = {
+        schemaVersion: 'grounding-candidate-v1',
+        found: true,
+        kind: bbox ? 'grounding_bbox' : 'coordinate',
+        bbox,
+        point,
+        confidence: clampConfidence(raw['confidence'], pointOnly ? POINT_ONLY_CONFIDENCE : BBOX_CONFIDENCE),
+        riskFlags: [
+            ...(pointOnly ? ['point_only'] : []),
+            ...(disagrees ? ['conflicting_point'] : []),
+        ],
+    };
+    if (typeof raw['description'] === 'string')
+        candidate.description = raw['description'];
+    if (typeof raw['reason'] === 'string')
+        candidate.reason = raw['reason'];
+    return candidate;
+}
+/**
+ * Scan out balanced JSON objects, tracking string state and escapes so a brace
+ * inside a string literal does not open or close a span.
+ *
+ * Emits nested objects as well as depth-0 ones, innermost first within a span.
+ * Models routinely wrap the answer — `{"result":{...}}`, `{"candidates":[...]}`
+ * — and a depth-0-only scan silently loses those, which the regex it replaced
+ * happened to handle.
+ *
+ * A stray quote is the hard case. Model text quotes page content ("the 15\"
+ * monitor"), and stdout truncation can cut inside a string literal, either of
+ * which leaves an odd quote count that inverts string tracking for everything
+ * after it. Tracking from the start of the text would then hide the real
+ * answer entirely, so the scan is also run from each subsequent `{` as a
+ * fresh origin, and any span that parses is kept.
+ */
+export function extractJsonObjects(text) {
+    const found = new Set();
+    const out = [];
+    for (const span of scanFrom(text, 0)) {
+        if (!found.has(span)) {
+            found.add(span);
+            out.push(span);
+        }
+    }
+    // Re-scan from each opening brace. A prior stray quote can only desync the
+    // tracker to the LEFT of a given origin, so restarting recovers the spans
+    // it swallowed. Bounded to keep a pathological input from going quadratic.
+    let origins = 0;
+    for (let i = 0; i < text.length && origins < MAX_RESCAN_ORIGINS; i++) {
+        if (text[i] !== '{')
+            continue;
+        origins += 1;
+        for (const span of scanFrom(text, i)) {
+            if (!found.has(span)) {
+                found.add(span);
+                out.push(span);
+            }
+        }
+    }
+    return out;
+}
+/** Restart budget for the stray-quote recovery above. */
+const MAX_RESCAN_ORIGINS = 64;
+function scanFrom(text, from) {
+    const objects = [];
+    const starts = [];
+    let inString = false;
+    let escaped = false;
+    for (let i = from; i < text.length; i++) {
+        const ch = text[i];
+        if (escaped) {
+            escaped = false;
+            continue;
+        }
+        if (inString && ch === '\\') {
+            escaped = true;
+            continue;
+        }
+        if (ch === '"') {
+            inString = !inString;
+            continue;
+        }
+        if (inString)
+            continue;
+        if (ch === '{') {
+            starts.push(i);
+        }
+        else if (ch === '}') {
+            const start = starts.pop();
+            if (start === undefined)
+                continue; // stray closer
+            objects.push(text.slice(start, i + 1));
+        }
+    }
+    return objects;
+}
+/**
+ * Recover a candidate from arbitrary model text.
+ *
+ * Prefers a located answer over a not-found one, and only then prefers the
+ * later of two equally useful candidates.
+ *
+ * Ordering alone is not enough here. The prompt itself contains a literal
+ * `{"found":false,...}` template, so any echo of the instructions parses as a
+ * perfectly valid not-found answer — and taking simply the last one turns a
+ * successful lookup into "target not found". Preferring `found: true` is what
+ * actually resolves that, not the scan direction.
+ *
+ * A malformed span is skipped rather than aborting the scan: a stray quote in
+ * quoted page text can desynchronize string tracking for the rest of the
+ * input, so a later object may be the only parseable one.
+ */
+export function parseCandidate(text) {
+    const objects = extractJsonObjects(text);
+    let fallback = null;
+    for (let i = objects.length - 1; i >= 0; i--) {
+        let parsed;
+        try {
+            parsed = JSON.parse(objects[i]);
+        }
+        catch {
+            continue;
+        }
+        const candidate = normalizeCandidate(parsed);
+        if (!candidate)
+            continue;
+        if (candidate.found)
+            return candidate;
+        fallback ??= candidate;
+    }
+    return fallback;
+}
+/**
+ * Reject a point that cannot be on screen.
+ *
+ * The old path converted and clicked whatever came back, so a hallucinated
+ * coordinate produced a click at an arbitrary place. Bounds are checked in the
+ * candidate's own frame — image pixels when a clip was captured, otherwise the
+ * viewport scaled by device pixel ratio.
+ */
+export function validateCandidate(candidate, frame) {
+    if (!candidate.found)
+        return candidate;
+    const { x, y } = candidate.point;
+    const inside = x >= 0 && y >= 0 && x <= frame.width && y <= frame.height;
+    if (inside)
+        return candidate;
+    return {
+        ...candidate,
+        found: false,
+        kind: 'not_found',
+        confidence: 0,
+        reason: `point (${x}, ${y}) is outside the ${frame.width}x${frame.height} capture`,
+        riskFlags: [...candidate.riskFlags, 'out_of_bounds'],
+    };
+}
+/** A candidate that must be verified before it is acted on. */
+export function isLowConfidence(candidate, threshold = LOW_CONFIDENCE_THRESHOLD) {
+    return candidate.confidence < threshold || candidate.riskFlags.includes('point_only');
+}
+//# sourceMappingURL=grounding-candidate.js.map

package/dist/src/browser/grounding-candidate.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"grounding-candidate.js","sourceRoot":"","sources":["../../../src/browser/grounding-candidate.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAsBH,sEAAsE;AACtE,MAAM,CAAC,MAAM,wBAAwB,GAAG,IAAI,CAAC;AAE7C,0EAA0E;AAC1E,MAAM,qBAAqB,GAAG,GAAG,CAAC;AAClC,MAAM,eAAe,GAAG,GAAG,CAAC;AAE5B;;;;GAIG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAG,OAAO,CAAC;AAEhD,SAAS,SAAS,CAAC,CAAS;IACxB,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,wBAAwB,CAAC;AACnD,CAAC;AAED,SAAS,QAAQ,CAAC,KAAc;IAC5B,OAAO,OAAO,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;AAChF,CAAC;AAED,SAAS,YAAY,CAAC,KAAc;IAChC,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC;AAC9E,CAAC;AAED,SAAS,cAAc,CAAC,KAAc;IAClC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAClC,MAAM,CAAC,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;IACnC,MAAM,CAAC,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;IACnC,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,IAAI;QAAE,OAAO,IAAI,CAAC;IAC1C,OAAO,SAAS,CAAC,CAAC,CAAC,IAAI,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;AAC1D,CAAC;AAED,SAAS,aAAa,CAAC,KAAc;IACjC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAClC,MAAM,CAAC,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;IACnC,MAAM,CAAC,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;IACnC,MAAM,KAAK,GAAG,YAAY,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;IAC3C,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC7C,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,KAAK,KAAK,IAAI,IAAI,MAAM,KAAK,IAAI;QAAE,OAAO,IAAI,CAAC;IAC/E,2EAA2E;IAC3E,IAAI,KAAK,IAAI,CAAC,IAAI,MAAM,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC;IAC3C,uEAAuE;IACvE,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC;QAAE,OAAO,IAAI,CAAC;IACzD,OAAO,EAAE,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC;AACnC,CAAC;AAED,SAAS,YAAY,CAAC,IAAU;IAC5B,OAAO,EAAE,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,EAAE,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC;AAC/F,CAAC;AAED,SAAS,eAAe,CAAC,KAAc,EAAE,QAAgB;IACrD,MAAM,CAAC,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;IAC9B,OAAO,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;AAC/D,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,kBAAkB,CAAC,GAAY;IAC3C,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC;QAAE,OAAO,IAAI,CAAC;IAChC,IAAI,OAAO,GAAG,CAAC,OAAO,CAAC,KAAK,SAAS;QAAE,OAAO,IAAI,CAAC;IAEnD,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;QAChB,OAAO;YACH,aAAa,EAAE,wBAAwB;YACvC,KAAK,EAAE,KAAK;YACZ,IAAI,EAAE,WAAW;YACjB,IAAI,EAAE,IAAI;YACV,KAAK,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE;YACrB,UAAU,EAAE,CAAC;YACb,WAAW,EAAE,OAAO,GAAG,CAAC,aAAa,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,WAAW;YACtF,MAAM,EAAE,OAAO,GAAG,CAAC,QAAQ,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,kBAAkB;YAC9E,SAAS,EAAE,EAAE;SAChB,CAAC;IACN,CAAC;IAED,MAAM,IAAI,GAAG,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC;IACxC,MAAM,aAAa,GAAG,cAAc,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC;IACnD,MAAM,WAAW,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,mBAAmB;IAC5D,MAAM,KAAK,GAAG,aAAa,IAAI,WAAW,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACjF,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAC;IAExB,uEAAuE;IACvE,yEAAyE;IACzE,MAAM,SAAS,GAAG,OAAO,CACrB,aAAa,IAAI,WAAW;QAC5B,CAAC,aAAa,CAAC,CAAC,KAAK,WAAW,CAAC,CAAC,IAAI,aAAa,CAAC,CAAC,KAAK,WAAW,CAAC,CAAC,CAAC,CAC3E,CAAC;IAEF,MAAM,SAAS,GAAG,CAAC,IAAI,CAAC;IACxB,MAAM,SAAS,GAAuB;QAClC,aAAa,EAAE,wBAAwB;QACvC,KAAK,EAAE,IAAI;QACX,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,YAAY;QAC5C,IAAI;QACJ,KAAK;QACL,UAAU,EAAE,eAAe,CAAC,GAAG,CAAC,YAAY,CAAC,EAAE,SAAS,CAAC,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,eAAe,CAAC;QACnG,SAAS,EAAE;YACP,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YACpC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;SAC9C;KACJ,CAAC;IACF,IAAI,OAAO,GAAG,CAAC,aAAa,CAAC,KAAK,QAAQ;QAAE,SAAS,CAAC,WAAW,GAAG,GAAG,CAAC,aAAa,CAAC,CAAC;IACvF,IAAI,OAAO,GAAG,CAAC,QAAQ,CAAC,KAAK,QAAQ;QAAE,SAAS,CAAC,MAAM,GAAG,GAAG,CAAC,QAAQ,CAAC,CAAC;IACxE,OAAO,SAAS,CAAC;AACrB,CAAC;AAED;;;;;;;;;;;;;;;GAeG;AACH,MAAM,UAAU,kBAAkB,CAAC,IAAY;IAC3C,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;IAChC,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,KAAK,MAAM,IAAI,IAAI,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC;QACnC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAAC,CAAC;IAC9D,CAAC;IACD,2EAA2E;IAC3E,0EAA0E;IAC1E,2EAA2E;IAC3E,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,IAAI,OAAO,GAAG,kBAAkB,EAAE,CAAC,EAAE,EAAE,CAAC;QACnE,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,GAAG;YAAE,SAAS;QAC9B,OAAO,IAAI,CAAC,CAAC;QACb,KAAK,MAAM,IAAI,IAAI,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC;YACnC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAAC,CAAC;QAC9D,CAAC;IACL,CAAC;IACD,OAAO,GAAG,CAAC;AACf,CAAC;AAED,yDAAyD;AACzD,MAAM,kBAAkB,GAAG,EAAE,CAAC;AAE9B,SAAS,QAAQ,CAAC,IAAY,EAAE,IAAY;IACxC,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,IAAI,QAAQ,GAAG,KAAK,CAAC;IACrB,IAAI,OAAO,GAAG,KAAK,CAAC;IAEpB,KAAK,IAAI,CAAC,GAAG,IAAI,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,MAAM,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;QACnB,IAAI,OAAO,EAAE,CAAC;YAAC,OAAO,GAAG,KAAK,CAAC;YAAC,SAAS;QAAC,CAAC;QAC3C,IAAI,QAAQ,IAAI,EAAE,KAAK,IAAI,EAAE,CAAC;YAAC,OAAO,GAAG,IAAI,CAAC;YAAC,SAAS;QAAC,CAAC;QAC1D,IAAI,EAAE,KAAK,GAAG,EAAE,CAAC;YAAC,QAAQ,GAAG,CAAC,QAAQ,CAAC;YAAC,SAAS;QAAC,CAAC;QACnD,IAAI,QAAQ;YAAE,SAAS;QACvB,IAAI,EAAE,KAAK,GAAG,EAAE,CAAC;YACb,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACnB,CAAC;aAAM,IAAI,EAAE,KAAK,GAAG,EAAE,CAAC;YACpB,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,EAAE,CAAC;YAC3B,IAAI,KAAK,KAAK,SAAS;gBAAE,SAAS,CAAC,eAAe;YAClD,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QAC3C,CAAC;IACL,CAAC;IACD,OAAO,OAAO,CAAC;AACnB,CAAC;AAED;;;;;;;;;;;;;;;GAeG;AACH,MAAM,UAAU,cAAc,CAAC,IAAY;IACvC,MAAM,OAAO,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC;IACzC,IAAI,QAAQ,GAA8B,IAAI,CAAC;IAC/C,KAAK,IAAI,CAAC,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC3C,IAAI,MAAe,CAAC;QACpB,IAAI,CAAC;YAAC,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAW,CAAC,CAAC;QAAC,CAAC;QAAC,MAAM,CAAC;YAAC,SAAS;QAAC,CAAC;QACtE,MAAM,SAAS,GAAG,kBAAkB,CAAC,MAAM,CAAC,CAAC;QAC7C,IAAI,CAAC,SAAS;YAAE,SAAS;QACzB,IAAI,SAAS,CAAC,KAAK;YAAE,OAAO,SAAS,CAAC;QACtC,QAAQ,KAAK,SAAS,CAAC;IAC3B,CAAC;IACD,OAAO,QAAQ,CAAC;AACpB,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,iBAAiB,CAC7B,SAA6B,EAC7B,KAAwC;IAExC,IAAI,CAAC,SAAS,CAAC,KAAK;QAAE,OAAO,SAAS,CAAC;IACvC,MAAM,EAAE,CAAC,EAAE,CAAC,EAAE,GAAG,SAAS,CAAC,KAAK,CAAC;IACjC,MAAM,MAAM,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC,KAAK,IAAI,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC;IACzE,IAAI,MAAM;QAAE,OAAO,SAAS,CAAC;IAC7B,OAAO;QACH,GAAG,SAAS;QACZ,KAAK,EAAE,KAAK;QACZ,IAAI,EAAE,WAAW;QACjB,UAAU,EAAE,CAAC;QACb,MAAM,EAAE,UAAU,CAAC,KAAK,CAAC,oBAAoB,KAAK,CAAC,KAAK,IAAI,KAAK,CAAC,MAAM,UAAU;QAClF,SAAS,EAAE,CAAC,GAAG,SAAS,CAAC,SAAS,EAAE,eAAe,CAAC;KACvD,CAAC;AACN,CAAC;AAED,+DAA+D;AAC/D,MAAM,UAAU,eAAe,CAC3B,SAA6B,EAC7B,SAAS,GAAG,wBAAwB;IAEpC,OAAO,SAAS,CAAC,UAAU,GAAG,SAAS,IAAI,SAAS,CAAC,SAAS,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;AAC1F,CAAC"}

package/dist/src/browser/grounding-eval.js

@@ -0,0 +1,247 @@
+/**
+ * src/browser/grounding-eval.ts — scoring for the grounding harness.
+ *
+ * Every phase in this work shipped on unit evidence with the same admitted
+ * gap: the pipeline seam needs a live browser and a model spawn, so nothing
+ * measured whether grounding actually works. This is the scoring half of the
+ * thing that measures it. The runner is separate and lives in scripts/, so the
+ * arithmetic here stays testable without Chrome.
+ *
+ * The measurement that matters is NOT "did the call return success". A click
+ * that lands on the wrong element also returns success, which is the failure
+ * this whole stack was opened to address. So a case declares the element it
+ * expects, and a run is scored against what was actually clicked.
+ *
+ * Abstention is a first-class outcome, not a failure. Refusing an ambiguous
+ * target, an occluded one, or a stale observation is the pipeline working.
+ * Folding those into "failed" would make every guard look like a regression
+ * and reward a system that clicks confidently regardless.
+ */
+/**
+ * Percentile by nearest-rank on a sorted sample.
+ *
+ * Deliberately not interpolating: with a handful of cases an interpolated p95
+ * reports a duration that never happened, which reads as precision the sample
+ * does not have.
+ */
+export function percentile(sortedMs, p) {
+    if (sortedMs.length === 0)
+        return 0;
+    const rank = Math.ceil((p / 100) * sortedMs.length);
+    const index = Math.min(Math.max(rank, 1), sortedMs.length) - 1;
+    return sortedMs[index];
+}
+export function scoreRun(results) {
+    const counts = { verified: 0, misclicked: 0, abstained: 0, groundingFailed: 0, errored: 0 };
+    const durations = [];
+    const group = () => ({ scored: 0, verified: 0, misclicked: 0, abstained: 0, groundingFailed: 0, verifiedRate: 0, misclickRate: 0 });
+    const click = group();
+    const refusal = group();
+    const errors = new Set();
+    for (const r of results) {
+        // `expected: 'abstain'` marks a case whose correct answer is a refusal.
+        const bucket = r.expected === 'abstain' ? refusal : click;
+        switch (r.outcome.kind) {
+            case 'verified':
+                counts.verified += 1;
+                bucket.verified += 1;
+                bucket.scored += 1;
+                break;
+            case 'misclick':
+                counts.misclicked += 1;
+                bucket.misclicked += 1;
+                bucket.scored += 1;
+                break;
+            case 'abstained':
+                counts.abstained += 1;
+                bucket.abstained += 1;
+                bucket.scored += 1;
+                break;
+            // A grounding failure stays in the denominator. It is a defect in
+            // the thing being measured, so excluding it would let a pipeline
+            // that grounds nothing report a clean sheet.
+            case 'grounding-failure':
+                counts.groundingFailed += 1;
+                bucket.groundingFailed += 1;
+                bucket.scored += 1;
+                break;
+            case 'errored':
+                counts.errored += 1;
+                errors.add(r.outcome.error);
+                break;
+        }
+        // An errored case measures the harness, not the pipeline, so its
+        // duration would skew the latency picture.
+        if (r.outcome.kind !== 'errored')
+            durations.push(r.outcome.ms);
+    }
+    // Errors are excluded from the denominator. A case the harness could not
+    // run is not evidence that grounding failed, and counting it as one would
+    // let a broken fixture look like a regression.
+    const scored = counts.verified + counts.misclicked + counts.abstained + counts.groundingFailed;
+    durations.sort((a, b) => a - b);
+    for (const g of [click, refusal]) {
+        g.verifiedRate = g.scored === 0 ? 0 : g.verified / g.scored;
+        g.misclickRate = g.scored === 0 ? 0 : g.misclicked / g.scored;
+    }
+    return {
+        total: results.length,
+        scored,
+        ...counts,
+        // All three headline rates are scoped to click cases. Blending them
+        // with refusal cases produced numbers that could not be read.
+        verifiedRate: click.verifiedRate,
+        misclickRate: click.misclickRate,
+        abstentionRate: click.scored === 0 ? 0 : click.abstained / click.scored,
+        latency: durations.length
+            ? { p50: percentile(durations, 50), p95: percentile(durations, 95), max: durations[durations.length - 1] }
+            : null,
+        click,
+        refusal,
+        errors: [...errors],
+    };
+}
+/**
+ * Decide a case outcome from what the pipeline returned.
+ *
+ * The refusal codes are named rather than pattern-matched on prose, so a
+ * reworded message cannot silently reclassify a refusal as a failure.
+ */
+export const ABSTENTION_CODES = new Set([
+    'COMPUTER_TARGET_AMBIGUOUS',
+    'COMPUTER_TARGET_COVERED',
+    'COMPUTER_VERIFY_DISAGREED',
+    'COMPUTER_OBSERVATION_EXPIRED',
+    'COMPUTER_OBSERVATION_STALE',
+    // Declining because the geometry was only partly captured, so a single
+    // containing element could not be concluded safely. A refusal, not a
+    // failure: the pipeline knew what it did not know.
+    'COMPUTER_GEOMETRY_TRUNCATED',
+]);
+/**
+ * The pipeline looked and did not find it.
+ *
+ * Separate from the abstention codes above because it is a different claim:
+ * those say "I found something and declined to act", this says "there was
+ * nothing to act on". A reader chasing a missing button needs to know which.
+ */
+export const NOT_FOUND_CODE = 'COMPUTER_TARGET_NOT_FOUND';
+/**
+ * The call could not be completed for reasons that have nothing to do with
+ * the page: a capture that could not be measured, a viewport that could not be
+ * read, a scale factor that was absent.
+ *
+ * These used to arrive with no code at all, which made them indistinguishable
+ * from a refusal — an infrastructure failure scored as commendable restraint,
+ * and printed to a human as a missing element.
+ */
+export const INFRASTRUCTURE_CODES = new Set([
+    'COMPUTER_CAPTURE_UNMEASURABLE',
+    'COMPUTER_VIEWPORT_UNAVAILABLE',
+    'COMPUTER_CAPTURE_NO_DPR',
+]);
+/**
+ * The pipeline produced an answer that was wrong on its face.
+ *
+ * `COMPUTER_CANDIDATE_OUT_OF_BOUNDS` fires when the model returns a point
+ * outside the frame it was shown. That is a defect in the thing being
+ * measured, not in the environment measuring it — grouping it with the
+ * capture failures would take a real grounding defect out of the denominator
+ * and blame the harness for it.
+ */
+export const GROUNDING_FAILURE_CODES = new Set([
+    'COMPUTER_CANDIDATE_OUT_OF_BOUNDS',
+]);
+export function classifyFailure(code) {
+    // An absent code and an unrecognised one are different situations. The
+    // first says the responder does not speak this protocol — an older server,
+    // most likely, since this runner talks to a separately built one over
+    // HTTP. The second says it speaks a newer dialect. Collapsing them would
+    // let a version skew silently reinstate the mis-scoring this fixes.
+    if (!code)
+        return 'uncoded';
+    if (ABSTENTION_CODES.has(code))
+        return 'abstention';
+    if (code === NOT_FOUND_CODE)
+        return 'not-found';
+    if (GROUNDING_FAILURE_CODES.has(code))
+        return 'grounding-failure';
+    if (INFRASTRUCTURE_CODES.has(code))
+        return 'infrastructure';
+    return 'unknown';
+}
+export function classify(response, expected, ms, clickedRef) {
+    if (response.success !== true) {
+        const kind = classifyFailure(response.code);
+        if (kind === 'abstention') {
+            return { kind: 'abstained', ms, reason: response.code };
+        }
+        if (kind === 'infrastructure') {
+            // The harness could not ask the question. That is not evidence
+            // about grounding in either direction, and scoring it as
+            // restraint made a broken capture look like good judgement.
+            return { kind: 'errored', ms, error: response.code };
+        }
+        if (kind === 'grounding-failure') {
+            return { kind: 'grounding-failure', ms, reason: response.code };
+        }
+        if (kind === 'not-found') {
+            // Expected-aware, because the same code means opposite things.
+            // On a case that should have declined, not finding the target IS
+            // the right answer. On a case where the element is present and
+            // described, failing to find it is a grounding failure — filing it
+            // as an abstention credits restraint that was never exercised and
+            // keeps it out of the misclick denominator, so a pipeline that
+            // grounds nothing reports a clean sheet.
+            return expected === 'abstain'
+                ? { kind: 'abstained', ms, reason: response.code }
+                : { kind: 'grounding-failure', ms, reason: response.code };
+        }
+        if (kind === 'uncoded' && response.reason) {
+            // A `success:false` body carrying prose and no code. Every current
+            // source path names its failures, so this means the responder is
+            // older than the naming — which is exactly the case where the
+            // prose could be describing a capture failure. Scoring it as an
+            // abstention is the mistake this function was rewritten to stop,
+            // so it is surfaced as unscoreable rather than credited.
+            return { kind: 'errored', ms, error: `uncoded failure: ${response.reason}` };
+        }
+        // An unrecognised code is forward compatibility, not breakage: this
+        // build cannot judge a dialect it does not know, and hard-failing
+        // would make the harness brittle against its own future.
+        if (response.reason)
+            return { kind: 'abstained', ms, reason: response.reason };
+        return { kind: 'errored', ms, error: 'no reason given' };
+    }
+    const got = clickedRef ?? response.ref;
+    // Success without an identifiable target cannot be scored. Calling it
+    // verified would be the exact assumption this harness exists to test.
+    if (!got)
+        return { kind: 'errored', ms, error: 'clicked but no element identity was reported' };
+    return got === expected ? { kind: 'verified', ms } : { kind: 'misclick', ms, got };
+}
+/** One line per rate, plus the caveat the numbers need to be read with. */
+export function formatReport(report) {
+    const pct = (n) => `${(n * 100).toFixed(1)}%`;
+    const lines = [
+        `cases           ${report.total} (${report.scored} scored, ${report.errored} errored)`,
+        `verified        ${report.verified} (${pct(report.verifiedRate)})`,
+        `misclicked      ${report.misclicked} (${pct(report.misclickRate)})`,
+        `abstained       ${report.abstained} (${pct(report.abstentionRate)} of cases that should have clicked)`,
+    ];
+    if (report.groundingFailed > 0) {
+        lines.push(`not grounded    ${report.groundingFailed} (the target was there and the pipeline could not find it)`);
+    }
+    if (report.latency) {
+        lines.push(`latency         p50 ${report.latency.p50}ms  p95 ${report.latency.p95}ms  max ${report.latency.max}ms`);
+    }
+    lines.push('', `when clicking   ${report.click.verified}/${report.click.scored} right, ${report.click.misclicked} wrong`, `when refusing   ${report.refusal.verified}/${report.refusal.scored} correctly declined, ${report.refusal.misclicked} clicked anyway`);
+    if (report.errors.length > 0) {
+        lines.push('', 'could not be scored:');
+        for (const e of report.errors)
+            lines.push(`  ${e}`);
+    }
+    lines.push('', 'An abstention is the pipeline declining, not failing. Read the misclick', 'rate first: a wrong click is worse than no click.');
+    return lines.join('\n');
+}
+//# sourceMappingURL=grounding-eval.js.map

package/dist/src/browser/grounding-eval.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"grounding-eval.js","sourceRoot":"","sources":["../../../src/browser/grounding-eval.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AA2FH;;;;;;GAMG;AACH,MAAM,UAAU,UAAU,CAAC,QAAkB,EAAE,CAAS;IACpD,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IACpC,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC;IACpD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IAC/D,OAAO,QAAQ,CAAC,KAAK,CAAW,CAAC;AACrC,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,OAAqB;IAC1C,MAAM,MAAM,GAAG,EAAE,QAAQ,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,eAAe,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;IAC5F,MAAM,SAAS,GAAa,EAAE,CAAC;IAC/B,MAAM,KAAK,GAAG,GAAgB,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,eAAe,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,CAAC,CAAC;IACjJ,MAAM,KAAK,GAAG,KAAK,EAAE,CAAC;IACtB,MAAM,OAAO,GAAG,KAAK,EAAE,CAAC;IACxB,MAAM,MAAM,GAAG,IAAI,GAAG,EAAU,CAAC;IAEjC,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACtB,wEAAwE;QACxE,MAAM,MAAM,GAAG,CAAC,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC;QAC1D,QAAQ,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;YACrB,KAAK,UAAU;gBAAE,MAAM,CAAC,QAAQ,IAAI,CAAC,CAAC;gBAAC,MAAM,CAAC,QAAQ,IAAI,CAAC,CAAC;gBAAC,MAAM,CAAC,MAAM,IAAI,CAAC,CAAC;gBAAC,MAAM;YACvF,KAAK,UAAU;gBAAE,MAAM,CAAC,UAAU,IAAI,CAAC,CAAC;gBAAC,MAAM,CAAC,UAAU,IAAI,CAAC,CAAC;gBAAC,MAAM,CAAC,MAAM,IAAI,CAAC,CAAC;gBAAC,MAAM;YAC3F,KAAK,WAAW;gBAAE,MAAM,CAAC,SAAS,IAAI,CAAC,CAAC;gBAAC,MAAM,CAAC,SAAS,IAAI,CAAC,CAAC;gBAAC,MAAM,CAAC,MAAM,IAAI,CAAC,CAAC;gBAAC,MAAM;YAC1F,kEAAkE;YAClE,iEAAiE;YACjE,6CAA6C;YAC7C,KAAK,mBAAmB;gBAAE,MAAM,CAAC,eAAe,IAAI,CAAC,CAAC;gBAAC,MAAM,CAAC,eAAe,IAAI,CAAC,CAAC;gBAAC,MAAM,CAAC,MAAM,IAAI,CAAC,CAAC;gBAAC,MAAM;YAC9G,KAAK,SAAS;gBAAE,MAAM,CAAC,OAAO,IAAI,CAAC,CAAC;gBAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;gBAAC,MAAM;QAC5E,CAAC;QACD,iEAAiE;QACjE,2CAA2C;QAC3C,IAAI,CAAC,CAAC,OAAO,CAAC,IAAI,KAAK,SAAS;YAAE,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IACnE,CAAC;IAED,yEAAyE;IACzE,0EAA0E;IAC1E,+CAA+C;IAC/C,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,GAAG,MAAM,CAAC,UAAU,GAAG,MAAM,CAAC,SAAS,GAAG,MAAM,CAAC,eAAe,CAAC;IAC/F,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAChC,KAAK,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,OAAO,CAAC,EAAE,CAAC;QAC/B,CAAC,CAAC,YAAY,GAAG,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,MAAM,CAAC;QAC5D,CAAC,CAAC,YAAY,GAAG,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,MAAM,CAAC;IAClE,CAAC;IAED,OAAO;QACH,KAAK,EAAE,OAAO,CAAC,MAAM;QACrB,MAAM;QACN,GAAG,MAAM;QACT,oEAAoE;QACpE,8DAA8D;QAC9D,YAAY,EAAE,KAAK,CAAC,YAAY;QAChC,YAAY,EAAE,KAAK,CAAC,YAAY;QAChC,cAAc,EAAE,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,SAAS,GAAG,KAAK,CAAC,MAAM;QACvE,OAAO,EAAE,SAAS,CAAC,MAAM;YACrB,CAAC,CAAC,EAAE,GAAG,EAAE,UAAU,CAAC,SAAS,EAAE,EAAE,CAAC,EAAE,GAAG,EAAE,UAAU,CAAC,SAAS,EAAE,EAAE,CAAC,EAAE,GAAG,EAAE,SAAS,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAW,EAAE;YACpH,CAAC,CAAC,IAAI;QACV,KAAK;QACL,OAAO;QACP,MAAM,EAAE,CAAC,GAAG,MAAM,CAAC;KACtB,CAAC;AACN,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAC;IACpC,2BAA2B;IAC3B,yBAAyB;IACzB,2BAA2B;IAC3B,8BAA8B;IAC9B,4BAA4B;IAC5B,uEAAuE;IACvE,qEAAqE;IACrE,mDAAmD;IACnD,6BAA6B;CAChC,CAAC,CAAC;AAEH;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,cAAc,GAAG,2BAA2B,CAAC;AAE1D;;;;;;;;GAQG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAG,IAAI,GAAG,CAAC;IACxC,+BAA+B;IAC/B,+BAA+B;IAC/B,yBAAyB;CAC5B,CAAC,CAAC;AAEH;;;;;;;;GAQG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAG,IAAI,GAAG,CAAC;IAC3C,kCAAkC;CACrC,CAAC,CAAC;AAaH,MAAM,UAAU,eAAe,CAAC,IAA+B;IAC3D,uEAAuE;IACvE,2EAA2E;IAC3E,sEAAsE;IACtE,yEAAyE;IACzE,oEAAoE;IACpE,IAAI,CAAC,IAAI;QAAE,OAAO,SAAS,CAAC;IAC5B,IAAI,gBAAgB,CAAC,GAAG,CAAC,IAAI,CAAC;QAAE,OAAO,YAAY,CAAC;IACpD,IAAI,IAAI,KAAK,cAAc;QAAE,OAAO,WAAW,CAAC;IAChD,IAAI,uBAAuB,CAAC,GAAG,CAAC,IAAI,CAAC;QAAE,OAAO,mBAAmB,CAAC;IAClE,IAAI,oBAAoB,CAAC,GAAG,CAAC,IAAI,CAAC;QAAE,OAAO,gBAAgB,CAAC;IAC5D,OAAO,SAAS,CAAC;AACrB,CAAC;AAED,MAAM,UAAU,QAAQ,CACpB,QAA6E,EAC7E,QAAgB,EAChB,EAAU,EACV,UAAmB;IAEnB,IAAI,QAAQ,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;QAC5B,MAAM,IAAI,GAAG,eAAe,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAC5C,IAAI,IAAI,KAAK,YAAY,EAAE,CAAC;YACxB,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,EAAE,EAAE,MAAM,EAAE,QAAQ,CAAC,IAAc,EAAE,CAAC;QACtE,CAAC;QACD,IAAI,IAAI,KAAK,gBAAgB,EAAE,CAAC;YAC5B,+DAA+D;YAC/D,yDAAyD;YACzD,4DAA4D;YAC5D,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,IAAc,EAAE,CAAC;QACnE,CAAC;QACD,IAAI,IAAI,KAAK,mBAAmB,EAAE,CAAC;YAC/B,OAAO,EAAE,IAAI,EAAE,mBAAmB,EAAE,EAAE,EAAE,MAAM,EAAE,QAAQ,CAAC,IAAc,EAAE,CAAC;QAC9E,CAAC;QACD,IAAI,IAAI,KAAK,WAAW,EAAE,CAAC;YACvB,+DAA+D;YAC/D,iEAAiE;YACjE,+DAA+D;YAC/D,mEAAmE;YACnE,kEAAkE;YAClE,+DAA+D;YAC/D,yCAAyC;YACzC,OAAO,QAAQ,KAAK,SAAS;gBACzB,CAAC,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,EAAE,EAAE,MAAM,EAAE,QAAQ,CAAC,IAAc,EAAE;gBAC5D,CAAC,CAAC,EAAE,IAAI,EAAE,mBAAmB,EAAE,EAAE,EAAE,MAAM,EAAE,QAAQ,CAAC,IAAc,EAAE,CAAC;QAC7E,CAAC;QACD,IAAI,IAAI,KAAK,SAAS,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC;YACxC,mEAAmE;YACnE,iEAAiE;YACjE,8DAA8D;YAC9D,gEAAgE;YAChE,iEAAiE;YACjE,yDAAyD;YACzD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,EAAE,EAAE,KAAK,EAAE,oBAAoB,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC;QACjF,CAAC;QACD,oEAAoE;QACpE,kEAAkE;QAClE,yDAAyD;QACzD,IAAI,QAAQ,CAAC,MAAM;YAAE,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,EAAE,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,CAAC;QAC/E,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,EAAE,EAAE,KAAK,EAAE,iBAAiB,EAAE,CAAC;IAC7D,CAAC;IAED,MAAM,GAAG,GAAG,UAAU,IAAI,QAAQ,CAAC,GAAG,CAAC;IACvC,sEAAsE;IACtE,sEAAsE;IACtE,IAAI,CAAC,GAAG;QAAE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,EAAE,EAAE,KAAK,EAAE,8CAA8C,EAAE,CAAC;IAChG,OAAO,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,EAAE,EAAE,GAAG,EAAE,CAAC;AACvF,CAAC;AAED,2EAA2E;AAC3E,MAAM,UAAU,YAAY,CAAC,MAAkB;IAC3C,MAAM,GAAG,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;IACtD,MAAM,KAAK,GAAG;QACV,mBAAmB,MAAM,CAAC,KAAK,KAAK,MAAM,CAAC,MAAM,YAAY,MAAM,CAAC,OAAO,WAAW;QACtF,mBAAmB,MAAM,CAAC,QAAQ,KAAK,GAAG,CAAC,MAAM,CAAC,YAAY,CAAC,GAAG;QAClE,mBAAmB,MAAM,CAAC,UAAU,KAAK,GAAG,CAAC,MAAM,CAAC,YAAY,CAAC,GAAG;QACpE,mBAAmB,MAAM,CAAC,SAAS,KAAK,GAAG,CAAC,MAAM,CAAC,cAAc,CAAC,qCAAqC;KAC1G,CAAC;IACF,IAAI,MAAM,CAAC,eAAe,GAAG,CAAC,EAAE,CAAC;QAC7B,KAAK,CAAC,IAAI,CAAC,mBAAmB,MAAM,CAAC,eAAe,4DAA4D,CAAC,CAAC;IACtH,CAAC;IACD,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;QACjB,KAAK,CAAC,IAAI,CAAC,uBAAuB,MAAM,CAAC,OAAO,CAAC,GAAG,WAAW,MAAM,CAAC,OAAO,CAAC,GAAG,WAAW,MAAM,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC,CAAC;IACxH,CAAC;IACD,KAAK,CAAC,IAAI,CACN,EAAE,EACF,mBAAmB,MAAM,CAAC,KAAK,CAAC,QAAQ,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,WAAW,MAAM,CAAC,KAAK,CAAC,UAAU,QAAQ,EACzG,mBAAmB,MAAM,CAAC,OAAO,CAAC,QAAQ,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,wBAAwB,MAAM,CAAC,OAAO,CAAC,UAAU,iBAAiB,CACxI,CAAC;IACF,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC3B,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,sBAAsB,CAAC,CAAC;QACvC,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM;YAAE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IACxD,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,yEAAyE,EAAE,mDAAmD,CAAC,CAAC;IAC/I,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC5B,CAAC"}

package/dist/src/browser/image-size.js

@@ -0,0 +1,63 @@
+/**
+ * src/browser/image-size.ts — read pixel dimensions from a capture file.
+ *
+ * Kept out of actions.ts so it can be tested without the CDP layer.
+ *
+ * This exists because the bounds check needs the frame the model actually saw.
+ * The requested clip is not that frame: Playwright trims a clip to the viewport
+ * before capture, and the route accepts an array clip whose `.width` is
+ * undefined — and comparing a number against `undefined` is always false, so
+ * such a bound looks present while checking nothing.
+ */
+import fs from 'fs';
+/**
+ * Returns null when the size cannot be read. Callers must fail closed on null:
+ * a plausible-looking default would be worse than admitting it is unknown.
+ */
+export function imageSize(filepath) {
+    let head;
+    try {
+        const fd = fs.openSync(filepath, 'r');
+        try {
+            const buffer = Buffer.alloc(65536);
+            const read = fs.readSync(fd, buffer, 0, buffer.length, 0);
+            head = buffer.subarray(0, read);
+        }
+        finally {
+            fs.closeSync(fd);
+        }
+    }
+    catch {
+        return null;
+    }
+    // PNG: 8-byte signature, then an IHDR chunk whose data begins at byte 16.
+    if (head.length >= 24 && head.readUInt32BE(0) === 0x89504e47) {
+        return { width: head.readUInt32BE(16), height: head.readUInt32BE(20) };
+    }
+    // JPEG: walk the segment chain to a start-of-frame marker.
+    if (head.length >= 4 && head[0] === 0xff && head[1] === 0xd8) {
+        let i = 2;
+        while (i + 9 < head.length) {
+            if (head[i] !== 0xff) {
+                i += 1;
+                continue;
+            }
+            const marker = head[i + 1];
+            // Standalone markers carry no length field.
+            if (marker === 0xd8 || marker === 0x01 || (marker >= 0xd0 && marker <= 0xd7)) {
+                i += 2;
+                continue;
+            }
+            const length = head.readUInt16BE(i + 2);
+            if (length < 2)
+                return null; // malformed chain
+            // SOF0..SOF15, excluding DHT (c4), JPG (c8) and DAC (cc).
+            if (marker >= 0xc0 && marker <= 0xcf && marker !== 0xc4 && marker !== 0xc8 && marker !== 0xcc) {
+                return { height: head.readUInt16BE(i + 5), width: head.readUInt16BE(i + 7) };
+            }
+            i += 2 + length;
+        }
+    }
+    return null;
+}
+//# sourceMappingURL=image-size.js.map

package/dist/src/browser/image-size.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"image-size.js","sourceRoot":"","sources":["../../../src/browser/image-size.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AACH,OAAO,EAAE,MAAM,IAAI,CAAC;AAEpB;;;GAGG;AACH,MAAM,UAAU,SAAS,CAAC,QAAgB;IACtC,IAAI,IAAY,CAAC;IACjB,IAAI,CAAC;QACD,MAAM,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;QACtC,IAAI,CAAC;YACD,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YACnC,MAAM,IAAI,GAAG,EAAE,CAAC,QAAQ,CAAC,EAAE,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YAC1D,IAAI,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;QACpC,CAAC;gBAAS,CAAC;YACP,EAAE,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QACrB,CAAC;IACL,CAAC;IAAC,MAAM,CAAC;QAAC,OAAO,IAAI,CAAC;IAAC,CAAC;IAExB,0EAA0E;IAC1E,IAAI,IAAI,CAAC,MAAM,IAAI,EAAE,IAAI,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,KAAK,UAAU,EAAE,CAAC;QAC3D,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,YAAY,CAAC,EAAE,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,YAAY,CAAC,EAAE,CAAC,EAAE,CAAC;IAC3E,CAAC;IAED,2DAA2D;IAC3D,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QAC3D,IAAI,CAAC,GAAG,CAAC,CAAC;QACV,OAAO,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YACzB,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;gBAAC,CAAC,IAAI,CAAC,CAAC;gBAAC,SAAS;YAAC,CAAC;YAC3C,MAAM,MAAM,GAAG,IAAI,CAAC,CAAC,GAAG,CAAC,CAAW,CAAC;YACrC,4CAA4C;YAC5C,IAAI,MAAM,KAAK,IAAI,IAAI,MAAM,KAAK,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,IAAI,MAAM,IAAI,IAAI,CAAC,EAAE,CAAC;gBAAC,CAAC,IAAI,CAAC,CAAC;gBAAC,SAAS;YAAC,CAAC;YACnG,MAAM,MAAM,GAAG,IAAI,CAAC,YAAY,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YACxC,IAAI,MAAM,GAAG,CAAC;gBAAE,OAAO,IAAI,CAAC,CAAC,kBAAkB;YAC/C,0DAA0D;YAC1D,IAAI,MAAM,IAAI,IAAI,IAAI,MAAM,IAAI,IAAI,IAAI,MAAM,KAAK,IAAI,IAAI,MAAM,KAAK,IAAI,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;gBAC5F,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;YACjF,CAAC;YACD,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC;QACpB,CAAC;IACL,CAAC;IACD,OAAO,IAAI,CAAC;AAChB,CAAC"}

package/dist/src/browser/index.js

@@ -5,6 +5,7 @@ export { cleanupIdleTabs, isPinned, parseTabDuration, pinTab, selectTabsForClean
 export * from './primitives.js';
 export { snapshot, screenshot, click, type, press, hover, navigate, evaluate, getPageText, mouseClick, getDom, waitForSelector, waitForText, reload, resize, scroll, select, drag, mouseMove, mouseDown, mouseUp, getConsole, getNetwork, } from './actions.js';
 export { visionClick, extractCoordinates } from './vision.js';
+export { sanitizeTarget, appendBounded, MAX_TARGET_LENGTH, MAX_CODEX_STDOUT_BYTES } from './vision-input.js';
 export * as adaptiveFetch from './adaptive-fetch/index.js';
 export * as webAi from './web-ai/index.js';
 //# sourceMappingURL=index.js.map

package/dist/src/browser/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/browser/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACH,YAAY,EAAE,UAAU,EAAE,aAAa,EAAE,aAAa,EACtD,QAAQ,EAAE,gBAAgB,EAAE,YAAY,EACxC,uBAAuB,EAAE,oBAAoB,EAAE,mBAAmB,EAClE,2BAA2B,EAC3B,aAAa,EAAE,YAAY,EAAE,SAAS,EACtC,SAAS,EAAE,QAAQ,EAAE,iBAAiB,EAAE,qBAAqB,EAC7D,uBAAuB,EAAE,sBAAsB,EAC/C,aAAa,EAAE,iBAAiB,EAAE,cAAc,GACnD,MAAM,iBAAiB,CAAC;AAGzB,OAAO,EAAE,qBAAqB,EAAE,MAAM,0BAA0B,CAAC;AAEjE,OAAO,EAAE,4BAA4B,EAAE,4BAA4B,EAAE,MAAM,sBAAsB,CAAC;AAElG,OAAO,EACH,eAAe,EAAE,QAAQ,EAAE,gBAAgB,EAAE,MAAM,EAAE,oBAAoB,EAAE,QAAQ,GACtF,MAAM,oBAAoB,CAAC;AAE5B,cAAc,iBAAiB,CAAC;AAEhC,OAAO,EACH,QAAQ,EAAE,UAAU,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EACxC,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,WAAW,EACtC,UAAU,EAAE,MAAM,EAAE,eAAe,EAAE,WAAW,EAChD,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EACpC,SAAS,EAAE,SAAS,EAAE,OAAO,EAAE,UAAU,EAAE,UAAU,GACxD,MAAM,cAAc,CAAC;AAEtB,OAAO,EAAE,WAAW,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAC9D,OAAO,KAAK,aAAa,MAAM,2BAA2B,CAAC;AAC3D,OAAO,KAAK,KAAK,MAAM,mBAAmB,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/browser/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACH,YAAY,EAAE,UAAU,EAAE,aAAa,EAAE,aAAa,EACtD,QAAQ,EAAE,gBAAgB,EAAE,YAAY,EACxC,uBAAuB,EAAE,oBAAoB,EAAE,mBAAmB,EAClE,2BAA2B,EAC3B,aAAa,EAAE,YAAY,EAAE,SAAS,EACtC,SAAS,EAAE,QAAQ,EAAE,iBAAiB,EAAE,qBAAqB,EAC7D,uBAAuB,EAAE,sBAAsB,EAC/C,aAAa,EAAE,iBAAiB,EAAE,cAAc,GACnD,MAAM,iBAAiB,CAAC;AAGzB,OAAO,EAAE,qBAAqB,EAAE,MAAM,0BAA0B,CAAC;AAEjE,OAAO,EAAE,4BAA4B,EAAE,4BAA4B,EAAE,MAAM,sBAAsB,CAAC;AAElG,OAAO,EACH,eAAe,EAAE,QAAQ,EAAE,gBAAgB,EAAE,MAAM,EAAE,oBAAoB,EAAE,QAAQ,GACtF,MAAM,oBAAoB,CAAC;AAE5B,cAAc,iBAAiB,CAAC;AAEhC,OAAO,EACH,QAAQ,EAAE,UAAU,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EACxC,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,WAAW,EACtC,UAAU,EAAE,MAAM,EAAE,eAAe,EAAE,WAAW,EAChD,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EACpC,SAAS,EAAE,SAAS,EAAE,OAAO,EAAE,UAAU,EAAE,UAAU,GACxD,MAAM,cAAc,CAAC;AAEtB,OAAO,EAAE,WAAW,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAC9D,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,iBAAiB,EAAE,sBAAsB,EAAE,MAAM,mBAAmB,CAAC;AAC7G,OAAO,KAAK,aAAa,MAAM,2BAA2B,CAAC;AAC3D,OAAO,KAAK,KAAK,MAAM,mBAAmB,CAAC"}