@mseep/clawdcursor 1.5.5

package/dist/core/sense/reactive-check.js

@@ -0,0 +1,48 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SETTLE_INTERVAL_MS = exports.SETTLE_BUDGET_MS = void 0;
+exports.reactiveCheck = reactiveCheck;
+const assertions_1 = require("../verify/assertions");
+/**
+ * How long a failing hard check re-polls before declaring a DEVIATION. UIs
+ * are asynchronous: a recipient chip resolves via a directory lookup
+ * (0.5–3s), window titles update lazily after a save. Declaring DEVIATION
+ * off a single immediate check told the model to RETRY actions that had
+ * actually taken — a duplicate-send risk on Send/Submit (audit 2026-06-10,
+ * finding D1).
+ */
+exports.SETTLE_BUDGET_MS = 2_000;
+exports.SETTLE_INTERVAL_MS = 250;
+async function reactiveCheck(input) {
+    const hasExpect = input.expect !== undefined && input.expect !== null;
+    if (hasExpect) {
+        const parsed = (0, assertions_1.parseAssertions)(input.expect);
+        if ('error' in parsed) {
+            return { success: false, text: `${input.toolText}\nexpect rejected: ${parsed.error}` };
+        }
+        // Poll until the assertions pass or the settle budget runs out — the UI
+        // is allowed to take a moment to manifest the effect before we call it
+        // a DEVIATION (which tells the model to retry a possibly-taken action).
+        const budget = input.settleMs ?? exports.SETTLE_BUDGET_MS;
+        const deadline = Date.now() + budget;
+        let report = await (0, assertions_1.checkAssertions)(parsed.assertions, { adapter: input.adapter, ocrText: input.ocrText });
+        while (!report.ok && Date.now() < deadline) {
+            await new Promise(r => setTimeout(r, exports.SETTLE_INTERVAL_MS));
+            report = await (0, assertions_1.checkAssertions)(parsed.assertions, { adapter: input.adapter, ocrText: input.ocrText });
+        }
+        if (report.ok) {
+            return { success: input.toolSuccess, text: `${input.toolText} — verified ${report.passed} check(s)` };
+        }
+        return {
+            success: false,
+            text: `${input.toolText}\nDEVIATION: ${report.failed}/${report.outcomes.length} expected check(s) failed (still failing after a ${Math.round(budget / 1000)}s settle window) — the action did not achieve its effect; adapt (re-find, retry, or a different approach) before continuing:\n${(0, assertions_1.renderReport)(report)}`,
+        };
+    }
+    // No expect: tolerant soft net — only for a SUCCESSFUL consequential action
+    // that produced no observable change. Never fails the action.
+    if (input.changesScreen && input.toolSuccess && !input.observedChange) {
+        return { success: true, text: `${input.toolText}\n⚠ no observable change — verify it took (pass \`expect\`) or try another approach.` };
+    }
+    return null;
+}
+//# sourceMappingURL=reactive-check.js.map

package/dist/core/sense/reactive-check.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"reactive-check.js","sourceRoot":"","sources":["../../../src/core/sense/reactive-check.ts"],"names":[],"mappings":";;;AAyCA,sCAkCC;AAnED,qDAAsF;AAmBtF;;;;;;;GAOG;AACU,QAAA,gBAAgB,GAAG,KAAK,CAAC;AACzB,QAAA,kBAAkB,GAAG,GAAG,CAAC;AAK/B,KAAK,UAAU,aAAa,CAAC,KAAoB;IACtD,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,KAAK,SAAS,IAAI,KAAK,CAAC,MAAM,KAAK,IAAI,CAAC;IAEtE,IAAI,SAAS,EAAE,CAAC;QACd,MAAM,MAAM,GAAG,IAAA,4BAAe,EAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAC7C,IAAI,OAAO,IAAI,MAAM,EAAE,CAAC;YACtB,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,KAAK,CAAC,QAAQ,sBAAsB,MAAM,CAAC,KAAK,EAAE,EAAE,CAAC;QACzF,CAAC;QACD,wEAAwE;QACxE,uEAAuE;QACvE,wEAAwE;QACxE,MAAM,MAAM,GAAG,KAAK,CAAC,QAAQ,IAAI,wBAAgB,CAAC;QAClD,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC;QACrC,IAAI,MAAM,GAAG,MAAM,IAAA,4BAAe,EAAC,MAAM,CAAC,UAAU,EAAE,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QAC1G,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,QAAQ,EAAE,CAAC;YAC3C,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,0BAAkB,CAAC,CAAC,CAAC;YAC1D,MAAM,GAAG,MAAM,IAAA,4BAAe,EAAC,MAAM,CAAC,UAAU,EAAE,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QACxG,CAAC;QACD,IAAI,MAAM,CAAC,EAAE,EAAE,CAAC;YACd,OAAO,EAAE,OAAO,EAAE,KAAK,CAAC,WAAW,EAAE,IAAI,EAAE,GAAG,KAAK,CAAC,QAAQ,eAAe,MAAM,CAAC,MAAM,WAAW,EAAE,CAAC;QACxG,CAAC;QACD,OAAO;YACL,OAAO,EAAE,KAAK;YACd,IAAI,EAAE,GAAG,KAAK,CAAC,QAAQ,gBAAgB,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,oDAAoD,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,iIAAiI,IAAA,yBAAY,EAAC,MAAM,CAAC,EAAE;SACnT,CAAC;IACJ,CAAC;IAED,4EAA4E;IAC5E,8DAA8D;IAC9D,IAAI,KAAK,CAAC,aAAa,IAAI,KAAK,CAAC,WAAW,IAAI,CAAC,KAAK,CAAC,cAAc,EAAE,CAAC;QACtE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,KAAK,CAAC,QAAQ,sFAAsF,EAAE,CAAC;IAC1I,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC"}

package/dist/core/sense/snapshot.d.ts

@@ -0,0 +1,19 @@
+/**
+ * Snapshot builder — assembles a structured Snapshot from the PlatformAdapter.
+ *
+ * Blind-first scope: accessibility tree only. When a11y can describe the
+ * screen, the text-agent reads this and picks an action. When a11y can't
+ * (empty tree, canvas app, custom rendering), the text-agent emits
+ * `cannot_read` and the vision-agent takes over with real pixels.
+ *
+ * OCR fusion on top of a11y (merging OCR hits with a11y bounds into a single
+ * element list) is a cost-saving enhancement we can layer in later without
+ * changing the `Snapshot` contract.
+ */
+import type { PlatformAdapter } from '../../platform/types';
+import type { Snapshot } from './types';
+/**
+ * Capture a structured snapshot. a11y-only for v0.8.1; OCR merge is a
+ * future enhancement.
+ */
+export declare function captureSnapshot(adapter: PlatformAdapter): Promise<Snapshot>;

package/dist/core/sense/snapshot.js

@@ -0,0 +1,100 @@
+"use strict";
+/**
+ * Snapshot builder — assembles a structured Snapshot from the PlatformAdapter.
+ *
+ * Blind-first scope: accessibility tree only. When a11y can describe the
+ * screen, the text-agent reads this and picks an action. When a11y can't
+ * (empty tree, canvas app, custom rendering), the text-agent emits
+ * `cannot_read` and the vision-agent takes over with real pixels.
+ *
+ * OCR fusion on top of a11y (merging OCR hits with a11y bounds into a single
+ * element list) is a cost-saving enhancement we can layer in later without
+ * changing the `Snapshot` contract.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.captureSnapshot = captureSnapshot;
+const fingerprint_1 = require("./fingerprint");
+function normPlatform(p) {
+    return p === 'darwin' ? 'macos' : p === 'win32' ? 'windows' : 'linux';
+}
+const SECURE_CONTROL_TYPES = new Set([
+    // Windows UIA (post "ControlType." strip) and macOS AX (post "AX" strip).
+    'edit', 'passwordbox', 'securefield',
+    'securetextfield', 'passwordfield',
+]);
+function looksSecure(controlType, name, subrole) {
+    // AX subrole is the authoritative macOS signal (role is the generic
+    // "TextField"; the secureness is in subrole=AXSecureTextField). Strip any AX
+    // prefix before the set test so "SecureTextField" matches (audit 2026-06-11, M2).
+    if (subrole && /securetextfield/i.test(subrole))
+        return true;
+    if (!controlType && !name)
+        return false;
+    const tLower = (controlType ?? '').replace(/^ax/i, '').toLowerCase();
+    const nLower = (name ?? '').toLowerCase();
+    if (SECURE_CONTROL_TYPES.has(tLower))
+        return true;
+    if (/\b(password|passcode|pin|secret|token|api\s*key|credit\s*card|cvv|ssn)\b/.test(nLower))
+        return true;
+    return false;
+}
+/**
+ * Capture a structured snapshot. a11y-only for v0.8.1; OCR merge is a
+ * future enhancement.
+ */
+async function captureSnapshot(adapter) {
+    const capturedAt = Date.now();
+    const platform = normPlatform(adapter.platform);
+    const sources = [];
+    let elements = [];
+    let activeWindow = undefined;
+    try {
+        const aw = await adapter.getActiveWindow();
+        if (aw) {
+            activeWindow = {
+                processId: aw.processId,
+                processName: aw.processName,
+                title: aw.title,
+                bounds: aw.bounds,
+            };
+        }
+    }
+    catch { /* active window optional */ }
+    try {
+        const tree = await adapter.getUiTree(activeWindow?.processId);
+        if (tree && tree.length > 0) {
+            sources.push('a11y');
+            elements = tree
+                .filter(el => el.bounds.width > 0 && el.bounds.height > 0)
+                .map(el => {
+                // Trust the adapter's explicit secure flag first (macOS AX subrole),
+                // then fall back to control-type/name/subrole heuristics.
+                const secure = el.secure === true || looksSecure(el.controlType, el.name, el.subrole);
+                return {
+                    name: el.name ?? '',
+                    role: el.controlType,
+                    x: el.bounds.x,
+                    y: el.bounds.y,
+                    width: el.bounds.width,
+                    height: el.bounds.height,
+                    source: 'a11y',
+                    interactive: el.enabled !== false,
+                    value: secure ? undefined : el.value,
+                    secure,
+                    processId: activeWindow?.processId,
+                };
+            });
+        }
+    }
+    catch { /* a11y unavailable — snapshot stays empty */ }
+    const fp = (0, fingerprint_1.fingerprint)(elements, activeWindow?.title);
+    return {
+        platform,
+        activeWindow,
+        elements,
+        fingerprint: fp,
+        capturedAt,
+        sources,
+    };
+}
+//# sourceMappingURL=snapshot.js.map

package/dist/core/sense/snapshot.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"snapshot.js","sourceRoot":"","sources":["../../../src/core/sense/snapshot.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;GAWG;;AAiCH,0CAyDC;AAtFD,+CAA4C;AAE5C,SAAS,YAAY,CAAC,CAA+B;IACnD,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,KAAK,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC;AACxE,CAAC;AAED,MAAM,oBAAoB,GAAG,IAAI,GAAG,CAAC;IACnC,0EAA0E;IAC1E,MAAM,EAAE,aAAa,EAAE,aAAa;IACpC,iBAAiB,EAAE,eAAe;CACnC,CAAC,CAAC;AAEH,SAAS,WAAW,CAAC,WAAoB,EAAE,IAAa,EAAE,OAAgB;IACxE,oEAAoE;IACpE,6EAA6E;IAC7E,kFAAkF;IAClF,IAAI,OAAO,IAAI,kBAAkB,CAAC,IAAI,CAAC,OAAO,CAAC;QAAE,OAAO,IAAI,CAAC;IAC7D,IAAI,CAAC,WAAW,IAAI,CAAC,IAAI;QAAE,OAAO,KAAK,CAAC;IACxC,MAAM,MAAM,GAAG,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;IACrE,MAAM,MAAM,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;IAC1C,IAAI,oBAAoB,CAAC,GAAG,CAAC,MAAM,CAAC;QAAE,OAAO,IAAI,CAAC;IAClD,IAAI,0EAA0E,CAAC,IAAI,CAAC,MAAM,CAAC;QAAE,OAAO,IAAI,CAAC;IACzG,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;GAGG;AACI,KAAK,UAAU,eAAe,CAAC,OAAwB;IAC5D,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC9B,MAAM,QAAQ,GAAG,YAAY,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAChD,MAAM,OAAO,GAAkC,EAAE,CAAC;IAElD,IAAI,QAAQ,GAAsB,EAAE,CAAC;IACrC,IAAI,YAAY,GAAyC,SAAS,CAAC;IAEnE,IAAI,CAAC;QACH,MAAM,EAAE,GAAG,MAAM,OAAO,CAAC,eAAe,EAAE,CAAC;QAC3C,IAAI,EAAE,EAAE,CAAC;YACP,YAAY,GAAG;gBACb,SAAS,EAAE,EAAE,CAAC,SAAS;gBACvB,WAAW,EAAE,EAAE,CAAC,WAAW;gBAC3B,KAAK,EAAE,EAAE,CAAC,KAAK;gBACf,MAAM,EAAE,EAAE,CAAC,MAAM;aAClB,CAAC;QACJ,CAAC;IACH,CAAC;IAAC,MAAM,CAAC,CAAC,4BAA4B,CAAC,CAAC;IAExC,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,SAAS,CAAC,YAAY,EAAE,SAAS,CAAC,CAAC;QAC9D,IAAI,IAAI,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5B,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACrB,QAAQ,GAAG,IAAI;iBACZ,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,MAAM,CAAC,KAAK,GAAG,CAAC,IAAI,EAAE,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC;iBACzD,GAAG,CAAC,EAAE,CAAC,EAAE;gBACR,qEAAqE;gBACrE,0DAA0D;gBAC1D,MAAM,MAAM,GAAG,EAAE,CAAC,MAAM,KAAK,IAAI,IAAI,WAAW,CAAC,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC;gBACtF,OAAO;oBACL,IAAI,EAAE,EAAE,CAAC,IAAI,IAAI,EAAE;oBACnB,IAAI,EAAE,EAAE,CAAC,WAAW;oBACpB,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC;oBACd,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC;oBACd,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK;oBACtB,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM;oBACxB,MAAM,EAAE,MAAe;oBACvB,WAAW,EAAE,EAAE,CAAC,OAAO,KAAK,KAAK;oBACjC,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK;oBACpC,MAAM;oBACN,SAAS,EAAE,YAAY,EAAE,SAAS;iBACnC,CAAC;YACJ,CAAC,CAAC,CAAC;QACP,CAAC;IACH,CAAC;IAAC,MAAM,CAAC,CAAC,6CAA6C,CAAC,CAAC;IAEzD,MAAM,EAAE,GAAG,IAAA,yBAAW,EAAC,QAAQ,EAAE,YAAY,EAAE,KAAK,CAAC,CAAC;IAEtD,OAAO;QACL,QAAQ;QACR,YAAY;QACZ,QAAQ;QACR,WAAW,EAAE,EAAE;QACf,UAAU;QACV,OAAO;KACR,CAAC;AACJ,CAAC"}

package/dist/core/sense/types.d.ts

@@ -0,0 +1,66 @@
+/**
+ * Sense-layer shared types — Snapshot, SnapshotElement, Platform.
+ *
+ * These types are used by the snapshot builder, fingerprint, rank, and the
+ * agent-loop prompt renderer. Extracted here so they survive pipeline deletion.
+ */
+export type Platform = 'windows' | 'macos' | 'linux';
+/**
+ * A single element in the merged perception snapshot.
+ * Coordinates are in real screen pixels (after DPI scaling).
+ */
+export interface SnapshotElement {
+    /** Human-readable label (a11y name or OCR text). */
+    name: string;
+    /** Accessibility role / control type when known. */
+    role?: string;
+    /** Top-left screen coords (after DPI scaling). */
+    x: number;
+    y: number;
+    width: number;
+    height: number;
+    /** Source of this element — preserved so agents can prefer a11y when available. */
+    source: 'a11y' | 'ocr' | 'cdp';
+    /** A11y-specific automation ID, if present. */
+    automationId?: string;
+    /** Whether the element accepts input (button, link, input). */
+    interactive?: boolean;
+    /** Whether the field is a password/secure field — redacted in `value`. */
+    secure?: boolean;
+    /** Current text value for inputs (redacted if `secure`). */
+    value?: string;
+    /** Process ID of the owning window. */
+    processId?: number;
+}
+/**
+ * A merged perception snapshot — one call, parallel OCR + a11y + optional CDP.
+ * Modeled on the per-turn perception snapshot; extended with fingerprint for
+ * stagnation detection.
+ */
+export interface Snapshot {
+    /** Source platform. */
+    platform: Platform;
+    /** Active window when the snapshot was taken. */
+    activeWindow?: {
+        processId: number;
+        processName: string;
+        title: string;
+        bounds: {
+            x: number;
+            y: number;
+            width: number;
+            height: number;
+        };
+    };
+    /** All elements merged from a11y + OCR + CDP, de-duped by spatial overlap. */
+    elements: SnapshotElement[];
+    /**
+     * Stable fingerprint of the snapshot — same UI produces same string, used by
+     * the agent loop to detect "nothing changed, stop retrying the same action".
+     */
+    fingerprint: string;
+    /** Timestamp for staleness checks. */
+    capturedAt: number;
+    /** Which sources successfully contributed; empty sources fell back silently. */
+    sources: Array<'a11y' | 'ocr' | 'cdp'>;
+}

package/dist/core/sense/types.js

@@ -0,0 +1,9 @@
+"use strict";
+/**
+ * Sense-layer shared types — Snapshot, SnapshotElement, Platform.
+ *
+ * These types are used by the snapshot builder, fingerprint, rank, and the
+ * agent-loop prompt renderer. Extracted here so they survive pipeline deletion.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+//# sourceMappingURL=types.js.map

package/dist/core/sense/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../src/core/sense/types.ts"],"names":[],"mappings":";AAAA;;;;;GAKG"}

package/dist/core/sense/ui-map-anchors.d.ts

@@ -0,0 +1,7 @@
+import type { UIElement, UIMap } from './ui-map-types';
+/**
+ * Two cross-turn anchors. `prevAnchors` is accepted for re-identification
+ * continuity (matched by role + normalized_text); with only two anchors this
+ * stays cheap and needs no element database.
+ */
+export declare function computeAnchors(elements: UIElement[], _prevAnchors: UIMap['anchors'] | undefined): UIMap['anchors'];

package/dist/core/sense/ui-map-anchors.js

@@ -0,0 +1,24 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.computeAnchors = computeAnchors;
+const PRIMARY_VERBS = ['send', 'save', 'submit', 'ok', 'continue', 'next',
+    'confirm', 'post', 'publish', 'done', 'apply'];
+const toRef = (e) => ({
+    id: e.id, role: e.role, normalized_text: e.normalized_text
+});
+/**
+ * Two cross-turn anchors. `prevAnchors` is accepted for re-identification
+ * continuity (matched by role + normalized_text); with only two anchors this
+ * stays cheap and needs no element database.
+ */
+function computeAnchors(elements, _prevAnchors) {
+    const focusedEl = elements.find(e => e.state?.focused);
+    const candidates = elements
+        .filter(e => e.clickable && PRIMARY_VERBS.includes(e.normalized_text ?? ''))
+        .sort((a, b) => b.confidence - a.confidence);
+    return {
+        focused: focusedEl ? toRef(focusedEl) : undefined,
+        primary_action_candidate: candidates[0] ? toRef(candidates[0]) : undefined,
+    };
+}
+//# sourceMappingURL=ui-map-anchors.js.map

package/dist/core/sense/ui-map-anchors.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ui-map-anchors.js","sourceRoot":"","sources":["../../../src/core/sense/ui-map-anchors.ts"],"names":[],"mappings":";;AAaA,wCAYC;AAvBD,MAAM,aAAa,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM;IACvE,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;AAEjD,MAAM,KAAK,GAAG,CAAC,CAAY,EAAc,EAAE,CAAC,CAAC;IAC3C,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,eAAe,EAAE,CAAC,CAAC,eAAe;CAAE,CAAC,CAAC;AAEhE;;;;GAIG;AACH,SAAgB,cAAc,CAC5B,QAAqB,EACrB,YAA0C;IAE1C,MAAM,SAAS,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IACvD,MAAM,UAAU,GAAG,QAAQ;SACxB,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,IAAI,aAAa,CAAC,QAAQ,CAAC,CAAC,CAAC,eAAe,IAAI,EAAE,CAAC,CAAC;SAC3E,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC;IAC/C,OAAO;QACL,OAAO,EAAE,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS;QACjD,wBAAwB,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS;KAC3E,CAAC;AACJ,CAAC"}

package/dist/core/sense/ui-map-elements.d.ts

@@ -0,0 +1,5 @@
+import type { SnapshotElement } from './types';
+import type { OcrElement } from '../../platform/ocr-engine';
+import type { UIElement } from './ui-map-types';
+export declare function a11yToUI(se: SnapshotElement, id: string): UIElement;
+export declare function ocrToUI(oe: OcrElement, id: string): UIElement;

package/dist/core/sense/ui-map-elements.js

@@ -0,0 +1,33 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.a11yToUI = a11yToUI;
+exports.ocrToUI = ocrToUI;
+const ui_map_normalize_1 = require("./ui-map-normalize");
+// Confidence bases (see spec §3). a11y is structured/trustworthy; OCR is scaled
+// by the recognizer's own score AND damped so a lone OCR hit never outranks a
+// corroborated a11y element. 1-char OCR fragments are damped hard (stray-"O").
+const A11Y_BASE = 0.85;
+const OCR_SCALE = 0.6;
+function a11yToUI(se, id) {
+    const role = (0, ui_map_normalize_1.normalizeRole)(se.role);
+    const caps = (0, ui_map_normalize_1.inferCapabilities)({ role, source: 'a11y', enabled: se.interactive });
+    return {
+        id, role, text: se.name, normalized_text: (0, ui_map_normalize_1.normText)(se.name),
+        bounds: [se.x, se.y, se.width, se.height],
+        confidence: A11Y_BASE, sources: ['a11y'], ...caps,
+        state: { enabled: se.interactive !== false, value: se.secure ? undefined : se.value },
+    };
+}
+function ocrToUI(oe, id) {
+    const role = 'text';
+    const caps = (0, ui_map_normalize_1.inferCapabilities)({ role, source: 'ocr', enabled: true });
+    // Single-char fragments are OCR noise — damp hard so they never win a match.
+    const lenDamp = oe.text.trim().length <= 1 ? 0.4 : 1;
+    const confidence = oe.confidence * OCR_SCALE * lenDamp;
+    return {
+        id, role, text: oe.text, normalized_text: (0, ui_map_normalize_1.normText)(oe.text),
+        bounds: [oe.x, oe.y, oe.width, oe.height],
+        confidence, sources: ['ocr'], ...caps,
+    };
+}
+//# sourceMappingURL=ui-map-elements.js.map

package/dist/core/sense/ui-map-elements.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ui-map-elements.js","sourceRoot":"","sources":["../../../src/core/sense/ui-map-elements.ts"],"names":[],"mappings":";;AAWA,4BASC;AAED,0BAWC;AA9BD,yDAAgF;AAEhF,gFAAgF;AAChF,8EAA8E;AAC9E,+EAA+E;AAC/E,MAAM,SAAS,GAAG,IAAI,CAAC;AACvB,MAAM,SAAS,GAAG,GAAG,CAAC;AAEtB,SAAgB,QAAQ,CAAC,EAAmB,EAAE,EAAU;IACtD,MAAM,IAAI,GAAG,IAAA,gCAAa,EAAC,EAAE,CAAC,IAAI,CAAC,CAAC;IACpC,MAAM,IAAI,GAAG,IAAA,oCAAiB,EAAC,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,EAAE,CAAC,WAAW,EAAE,CAAC,CAAC;IAClF,OAAO;QACL,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,CAAC,IAAI,EAAE,eAAe,EAAE,IAAA,2BAAQ,EAAC,EAAE,CAAC,IAAI,CAAC;QAC3D,MAAM,EAAE,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC;QACzC,UAAU,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC,MAAM,CAAC,EAAE,GAAG,IAAI;QACjD,KAAK,EAAE,EAAE,OAAO,EAAE,EAAE,CAAC,WAAW,KAAK,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,EAAE;KACtF,CAAC;AACJ,CAAC;AAED,SAAgB,OAAO,CAAC,EAAc,EAAE,EAAU;IAChD,MAAM,IAAI,GAAG,MAAe,CAAC;IAC7B,MAAM,IAAI,GAAG,IAAA,oCAAiB,EAAC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;IACvE,6EAA6E;IAC7E,MAAM,OAAO,GAAG,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACrD,MAAM,UAAU,GAAG,EAAE,CAAC,UAAU,GAAG,SAAS,GAAG,OAAO,CAAC;IACvD,OAAO;QACL,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,CAAC,IAAI,EAAE,eAAe,EAAE,IAAA,2BAAQ,EAAC,EAAE,CAAC,IAAI,CAAC;QAC3D,MAAM,EAAE,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC;QACzC,UAAU,EAAE,OAAO,EAAE,CAAC,KAAK,CAAC,EAAE,GAAG,IAAI;KACtC,CAAC;AACJ,CAAC"}

package/dist/core/sense/ui-map-find.d.ts

@@ -0,0 +1,56 @@
+/**
+ * Layer B — pure, deterministic semantic finders over a compiled UIMap.
+ * Turn an intent/purpose into the best el_NN. No LLM, no platform calls,
+ * no process.platform branching. See the Layer B design spec.
+ *
+ * BOUNDARY: ranks only elements the UIMap marks clickable/editable (a11y-exposed
+ * interactive elements, where the Part-2 ref-act chain is valid). OCR text is a
+ * label SOURCE only (Task 3), not an independent actionable candidate.
+ */
+import type { UIElement, Role } from './ui-map-types';
+export declare const MATCH_THRESHOLD = 0.4;
+export declare const AMBIGUITY_MARGIN = 0.05;
+export declare const MAX_CANDIDATES = 5;
+export interface FindCandidate {
+    element_id: string;
+    label: string;
+    role: Role;
+    score: number;
+    confidence: number;
+}
+export type FindResult = {
+    status: 'ok';
+    snapshot_id: string;
+    best: FindCandidate;
+    candidates: FindCandidate[];
+} | {
+    status: 'ambiguous';
+    snapshot_id: string;
+    reason: string;
+    candidates: FindCandidate[];
+} | {
+    status: 'none';
+    snapshot_id: string;
+    reason: string;
+    candidates: FindCandidate[];
+};
+export interface ExpandedTerms {
+    /** The normalized intent literal and its tokens — exact match → score 1.0. */
+    literals: Set<string>;
+    /** Synonyms drawn from the table — synonym match → score 0.9. */
+    synonyms: Set<string>;
+}
+/** Expand an intent into literal + synonym sets. Keeps exact vs synonym distinct
+ *  so that scoreLabel can assign different raw scores to each tier. */
+export declare function expandTerms(intent: string, table: Record<string, string[]>): ExpandedTerms;
+/** Raw 0..1 match of a label against the term sets (no confidence yet).
+ *  Literal exact → 1.0; synonym exact / literal substring → 0.9; token overlap → ≤0.5. */
+export declare function scoreLabel(label: string, { literals, synonyms }: ExpandedTerms, intentTokens: string[]): number;
+/** Shared finder core. `labelOf` lets fields override with geometric association. */
+export declare function runFinder(elements: UIElement[], snapshotId: string, intent: string, table: Record<string, string[]>, isCandidate: (e: UIElement) => boolean, labelOf: (e: UIElement, all: UIElement[]) => string): FindResult;
+export declare function findActionButton(elements: UIElement[], snapshotId: string, intent: string): FindResult;
+/** Find the descriptive label for an (unnamed) field by geometry: nearest text
+ *  element to the LEFT (same row) preferred, else ABOVE (same column). a11y OR
+ *  OCR sourced. Returns '' when none is within range. Pure geometry. */
+export declare function associateLabel(field: UIElement, all: UIElement[]): string;
+export declare function findInputField(elements: UIElement[], snapshotId: string, purpose: string): FindResult;

package/dist/core/sense/ui-map-find.js

@@ -0,0 +1,153 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MAX_CANDIDATES = exports.AMBIGUITY_MARGIN = exports.MATCH_THRESHOLD = void 0;
+exports.expandTerms = expandTerms;
+exports.scoreLabel = scoreLabel;
+exports.runFinder = runFinder;
+exports.findActionButton = findActionButton;
+exports.associateLabel = associateLabel;
+exports.findInputField = findInputField;
+const ui_map_normalize_1 = require("./ui-map-normalize");
+exports.MATCH_THRESHOLD = 0.4; // min score×confidence to count as a match
+exports.AMBIGUITY_MARGIN = 0.05; // top-two closeness that forces 'ambiguous'
+exports.MAX_CANDIDATES = 5;
+/** Curated intent → synonym sets. Unknown intents fall back to literal/token match. */
+const ACTION_SYNONYMS = {
+    submit: ['submit', 'send', 'ok', 'confirm', 'save', 'continue', 'next', 'post', 'publish', 'apply', 'done', 'go'],
+    cancel: ['cancel', 'close', 'dismiss', 'back', 'no'],
+    delete: ['delete', 'remove', 'trash', 'discard'],
+    search: ['search', 'find', 'go', 'query'],
+    login: ['login', 'log in', 'sign in', 'signin'],
+    open: ['open', 'launch', 'view'],
+    add: ['add', 'new', 'create', 'plus'],
+};
+/** Expand an intent into literal + synonym sets. Keeps exact vs synonym distinct
+ *  so that scoreLabel can assign different raw scores to each tier. */
+function expandTerms(intent, table) {
+    const lit = (0, ui_map_normalize_1.normText)(intent);
+    const literals = new Set([lit]);
+    lit.split(' ').filter(Boolean).forEach(t => literals.add(t));
+    literals.delete('');
+    const synonyms = new Set();
+    for (const [key, syns] of Object.entries(table)) {
+        if (key === lit || syns.includes(lit)) {
+            syns.forEach(s => {
+                if (!literals.has(s))
+                    synonyms.add(s);
+            });
+        }
+    }
+    return { literals, synonyms };
+}
+/** Raw 0..1 match of a label against the term sets (no confidence yet).
+ *  Literal exact → 1.0; synonym exact / literal substring → 0.9; token overlap → ≤0.5. */
+function scoreLabel(label, { literals, synonyms }, intentTokens) {
+    const L = (0, ui_map_normalize_1.normText)(label);
+    if (!L)
+        return 0;
+    // Exact literal match (highest tier)
+    if (literals.has(L))
+        return 1.0;
+    const lTokens = L.split(' ').filter(Boolean);
+    const lSet = new Set(lTokens);
+    // Synonym exact match or literal term as whole word / substring in label
+    for (const t of synonyms) {
+        if (lSet.has(t))
+            return 0.9;
+        if (t.length > 2 && L.includes(t))
+            return 0.9;
+    }
+    for (const t of literals) {
+        if (lSet.has(t))
+            return 0.9; // intent token is a whole word in the label
+        if (t.length > 2 && L.includes(t))
+            return 0.9; // intent token is a substring of the label
+    }
+    const overlap = intentTokens.filter(t => lSet.has(t)).length;
+    return intentTokens.length > 0 ? 0.5 * (overlap / intentTokens.length) : 0;
+}
+/** Shared finder core. `labelOf` lets fields override with geometric association. */
+function runFinder(elements, snapshotId, intent, table, isCandidate, labelOf) {
+    const terms = expandTerms(intent, table);
+    const intentTokens = (0, ui_map_normalize_1.normText)(intent).split(' ').filter(Boolean);
+    const scored = [];
+    for (const e of elements) {
+        if (!isCandidate(e))
+            continue;
+        const label = labelOf(e, elements);
+        const raw = scoreLabel(label, terms, intentTokens);
+        if (raw <= 0)
+            continue;
+        scored.push({ element_id: e.id, label, role: e.role, score: raw * e.confidence, confidence: e.confidence });
+    }
+    scored.sort((a, b) => b.score - a.score);
+    const candidates = scored.slice(0, exports.MAX_CANDIDATES);
+    const best = scored[0];
+    if (!best || best.score < exports.MATCH_THRESHOLD) {
+        return { status: 'none', snapshot_id: snapshotId, reason: 'no candidate cleared the match threshold', candidates };
+    }
+    const second = scored[1];
+    if (second && best.score - second.score < exports.AMBIGUITY_MARGIN) {
+        return { status: 'ambiguous', snapshot_id: snapshotId, reason: `top candidates within ${exports.AMBIGUITY_MARGIN} score margin`, candidates };
+    }
+    return { status: 'ok', snapshot_id: snapshotId, best, candidates };
+}
+function findActionButton(elements, snapshotId, intent) {
+    return runFinder(elements, snapshotId, intent, ACTION_SYNONYMS, e => e.clickable === true, e => e.normalized_text ?? '');
+}
+const MAX_LABEL_GAP_X = 240; // px: a left label's right edge to the field's left edge
+const MAX_LABEL_GAP_Y = 64; // px: an above label's bottom edge to the field's top edge
+const LABEL_ROLES = new Set(['text', 'link', 'unknown']);
+/** Find the descriptive label for an (unnamed) field by geometry: nearest text
+ *  element to the LEFT (same row) preferred, else ABOVE (same column). a11y OR
+ *  OCR sourced. Returns '' when none is within range. Pure geometry. */
+function associateLabel(field, all) {
+    const [fx, fy, fw, fh] = field.bounds;
+    let left = null;
+    let above = null;
+    for (const e of all) {
+        if (e.id === field.id || !LABEL_ROLES.has(e.role))
+            continue;
+        const label = e.normalized_text ?? '';
+        if (!label)
+            continue;
+        const [ex, ey, ew, eh] = e.bounds;
+        const eRight = ex + ew;
+        const eBottom = ey + eh;
+        const eCenterY = ey + eh / 2;
+        const eCenterX = ex + ew / 2;
+        // LEFT, same row: label's vertical center within the field's y-band, right edge just left of the field.
+        const sameRow = eCenterY >= fy && eCenterY <= fy + fh;
+        if (sameRow && eRight <= fx + 4) {
+            const gap = fx - eRight;
+            if (gap >= 0 && gap <= MAX_LABEL_GAP_X && (!left || gap < left.gap))
+                left = { gap, label };
+        }
+        // ABOVE, same column: label horizontally overlaps the field, bottom just above the field top.
+        const sameCol = eCenterX >= fx && eCenterX <= fx + fw;
+        if (sameCol && eBottom <= fy + 4) {
+            const gap = fy - eBottom;
+            if (gap >= 0 && gap <= MAX_LABEL_GAP_Y && (!above || gap < above.gap))
+                above = { gap, label };
+        }
+    }
+    return (left ?? above)?.label ?? '';
+}
+/** Curated field-purpose → synonym sets. */
+const FIELD_SYNONYMS = {
+    recipient: ['to', 'recipient', 'email', 'address', 'send to'],
+    cc: ['cc', 'carbon copy'],
+    subject: ['subject', 'title', 're'],
+    body: ['body', 'message', 'content', 'compose', 'note'],
+    search: ['search', 'query', 'find', 'filter'],
+    password: ['password', 'pass', 'pwd'],
+    username: ['username', 'user', 'login', 'email'],
+    name: ['name', 'full name'],
+};
+function findInputField(elements, snapshotId, purpose) {
+    return runFinder(elements, snapshotId, purpose, FIELD_SYNONYMS, e => e.editable === true, (e, all) => {
+        const own = e.normalized_text ?? '';
+        return own !== '' ? own : associateLabel(e, all);
+    });
+}
+//# sourceMappingURL=ui-map-find.js.map

package/dist/core/sense/ui-map-find.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ui-map-find.js","sourceRoot":"","sources":["../../../src/core/sense/ui-map-find.ts"],"names":[],"mappings":";;;AA0CA,kCAeC;AAID,gCAkBC;AAGD,8BA6BC;AAED,4CAIC;AAUD,wCA2BC;AAcD,wCAOC;AArKD,yDAA8C;AAEjC,QAAA,eAAe,GAAG,GAAG,CAAC,CAAG,2CAA2C;AACpE,QAAA,gBAAgB,GAAG,IAAI,CAAC,CAAC,4CAA4C;AACrE,QAAA,cAAc,GAAG,CAAC,CAAC;AAQhC,uFAAuF;AACvF,MAAM,eAAe,GAA6B;IAChD,MAAM,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC;IACjH,MAAM,EAAE,CAAC,QAAQ,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,CAAC;IACpD,MAAM,EAAE,CAAC,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,SAAS,CAAC;IAChD,MAAM,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,CAAC;IACzC,KAAK,EAAE,CAAC,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,CAAC;IAC/C,IAAI,EAAE,CAAC,MAAM,EAAE,QAAQ,EAAE,MAAM,CAAC;IAChC,GAAG,EAAE,CAAC,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,CAAC;CACtC,CAAC;AASF;uEACuE;AACvE,SAAgB,WAAW,CAAC,MAAc,EAAE,KAA+B;IACzE,MAAM,GAAG,GAAG,IAAA,2BAAQ,EAAC,MAAM,CAAC,CAAC;IAC7B,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAS,CAAC,GAAG,CAAC,CAAC,CAAC;IACxC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7D,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IAEpB,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAC;IACnC,KAAK,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QAChD,IAAI,GAAG,KAAK,GAAG,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YACtC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE;gBACf,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;oBAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACxC,CAAC,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IACD,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC;AAChC,CAAC;AAED;0FAC0F;AAC1F,SAAgB,UAAU,CAAC,KAAa,EAAE,EAAE,QAAQ,EAAE,QAAQ,EAAiB,EAAE,YAAsB;IACrG,MAAM,CAAC,GAAG,IAAA,2BAAQ,EAAC,KAAK,CAAC,CAAC;IAC1B,IAAI,CAAC,CAAC;QAAE,OAAO,CAAC,CAAC;IACjB,qCAAqC;IACrC,IAAI,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;QAAE,OAAO,GAAG,CAAC;IAChC,MAAM,OAAO,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAC7C,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC;IAC9B,yEAAyE;IACzE,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;QACzB,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;YAAE,OAAO,GAAG,CAAC;QAC5B,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;YAAE,OAAO,GAAG,CAAC;IAChD,CAAC;IACD,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;QACzB,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;YAAE,OAAO,GAAG,CAAC,CAA0B,4CAA4C;QAClG,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;YAAE,OAAO,GAAG,CAAC,CAAQ,2CAA2C;IACnG,CAAC;IACD,MAAM,OAAO,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;IAC7D,OAAO,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,OAAO,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAC7E,CAAC;AAED,qFAAqF;AACrF,SAAgB,SAAS,CACvB,QAAqB,EACrB,UAAkB,EAClB,MAAc,EACd,KAA+B,EAC/B,WAAsC,EACtC,OAAmD;IAEnD,MAAM,KAAK,GAAG,WAAW,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;IACzC,MAAM,YAAY,GAAG,IAAA,2BAAQ,EAAC,MAAM,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IACjE,MAAM,MAAM,GAAoB,EAAE,CAAC;IACnC,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;QACzB,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC;YAAE,SAAS;QAC9B,MAAM,KAAK,GAAG,OAAO,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;QACnC,MAAM,GAAG,GAAG,UAAU,CAAC,KAAK,EAAE,KAAK,EAAE,YAAY,CAAC,CAAC;QACnD,IAAI,GAAG,IAAI,CAAC;YAAE,SAAS;QACvB,MAAM,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,GAAG,GAAG,CAAC,CAAC,UAAU,EAAE,UAAU,EAAE,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC;IAC9G,CAAC;IACD,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;IACzC,MAAM,UAAU,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,sBAAc,CAAC,CAAC;IACnD,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;IACvB,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,KAAK,GAAG,uBAAe,EAAE,CAAC;QAC1C,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,EAAE,0CAA0C,EAAE,UAAU,EAAE,CAAC;IACrH,CAAC;IACD,MAAM,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;IACzB,IAAI,MAAM,IAAI,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,GAAG,wBAAgB,EAAE,CAAC;QAC3D,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,EAAE,yBAAyB,wBAAgB,eAAe,EAAE,UAAU,EAAE,CAAC;IACxI,CAAC;IACD,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,WAAW,EAAE,UAAU,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC;AACrE,CAAC;AAED,SAAgB,gBAAgB,CAAC,QAAqB,EAAE,UAAkB,EAAE,MAAc;IACxF,OAAO,SAAS,CAAC,QAAQ,EAAE,UAAU,EAAE,MAAM,EAAE,eAAe,EAC5D,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,IAAI,EACzB,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,eAAe,IAAI,EAAE,CAAC,CAAC;AAClC,CAAC;AAED,MAAM,eAAe,GAAG,GAAG,CAAC,CAAE,yDAAyD;AACvF,MAAM,eAAe,GAAG,EAAE,CAAC,CAAG,2DAA2D;AAEzF,MAAM,WAAW,GAAG,IAAI,GAAG,CAAO,CAAC,MAAM,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC;AAE/D;;wEAEwE;AACxE,SAAgB,cAAc,CAAC,KAAgB,EAAE,GAAgB;IAC/D,MAAM,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC;IACtC,IAAI,IAAI,GAA0C,IAAI,CAAC;IACvD,IAAI,KAAK,GAA0C,IAAI,CAAC;IACxD,KAAK,MAAM,CAAC,IAAI,GAAG,EAAE,CAAC;QACpB,IAAI,CAAC,CAAC,EAAE,KAAK,KAAK,CAAC,EAAE,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC;YAAE,SAAS;QAC5D,MAAM,KAAK,GAAG,CAAC,CAAC,eAAe,IAAI,EAAE,CAAC;QACtC,IAAI,CAAC,KAAK;YAAE,SAAS;QACrB,MAAM,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC;QAClC,MAAM,MAAM,GAAG,EAAE,GAAG,EAAE,CAAC;QACvB,MAAM,OAAO,GAAG,EAAE,GAAG,EAAE,CAAC;QACxB,MAAM,QAAQ,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;QAC7B,MAAM,QAAQ,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;QAC7B,wGAAwG;QACxG,MAAM,OAAO,GAAG,QAAQ,IAAI,EAAE,IAAI,QAAQ,IAAI,EAAE,GAAG,EAAE,CAAC;QACtD,IAAI,OAAO,IAAI,MAAM,IAAI,EAAE,GAAG,CAAC,EAAE,CAAC;YAChC,MAAM,GAAG,GAAG,EAAE,GAAG,MAAM,CAAC;YACxB,IAAI,GAAG,IAAI,CAAC,IAAI,GAAG,IAAI,eAAe,IAAI,CAAC,CAAC,IAAI,IAAI,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC;gBAAE,IAAI,GAAG,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC;QAC7F,CAAC;QACD,8FAA8F;QAC9F,MAAM,OAAO,GAAG,QAAQ,IAAI,EAAE,IAAI,QAAQ,IAAI,EAAE,GAAG,EAAE,CAAC;QACtD,IAAI,OAAO,IAAI,OAAO,IAAI,EAAE,GAAG,CAAC,EAAE,CAAC;YACjC,MAAM,GAAG,GAAG,EAAE,GAAG,OAAO,CAAC;YACzB,IAAI,GAAG,IAAI,CAAC,IAAI,GAAG,IAAI,eAAe,IAAI,CAAC,CAAC,KAAK,IAAI,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC;gBAAE,KAAK,GAAG,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC;QAChG,CAAC;IACH,CAAC;IACD,OAAO,CAAC,IAAI,IAAI,KAAK,CAAC,EAAE,KAAK,IAAI,EAAE,CAAC;AACtC,CAAC;AAED,4CAA4C;AAC5C,MAAM,cAAc,GAA6B;IAC/C,SAAS,EAAE,CAAC,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,SAAS,EAAE,SAAS,CAAC;IAC7D,EAAE,EAAE,CAAC,IAAI,EAAE,aAAa,CAAC;IACzB,OAAO,EAAE,CAAC,SAAS,EAAE,OAAO,EAAE,IAAI,CAAC;IACnC,IAAI,EAAE,CAAC,MAAM,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,CAAC;IACvD,MAAM,EAAE,CAAC,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC;IAC7C,QAAQ,EAAE,CAAC,UAAU,EAAE,MAAM,EAAE,KAAK,CAAC;IACrC,QAAQ,EAAE,CAAC,UAAU,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,CAAC;IAChD,IAAI,EAAE,CAAC,MAAM,EAAE,WAAW,CAAC;CAC5B,CAAC;AAEF,SAAgB,cAAc,CAAC,QAAqB,EAAE,UAAkB,EAAE,OAAe;IACvF,OAAO,SAAS,CAAC,QAAQ,EAAE,UAAU,EAAE,OAAO,EAAE,cAAc,EAC5D,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,IAAI,EACxB,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE;QACT,MAAM,GAAG,GAAG,CAAC,CAAC,eAAe,IAAI,EAAE,CAAC;QACpC,OAAO,GAAG,KAAK,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IACnD,CAAC,CAAC,CAAC;AACP,CAAC"}

package/dist/core/sense/ui-map-fuse.d.ts

@@ -0,0 +1,4 @@
+import type { UIElement } from './ui-map-types';
+/** Merge same-place + same-text elements across sources; raise confidence per
+ *  corroborating source; keep the stronger role/capabilities. */
+export declare function fuse(elements: UIElement[]): UIElement[];

package/dist/core/sense/ui-map-fuse.js

@@ -0,0 +1,44 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.fuse = fuse;
+const ui_map_geom_1 = require("./ui-map-geom");
+const AGREEMENT_BONUS = 0.15;
+const OVERLAP_MIN = 0.5; // IoU threshold to treat two boxes as the same element
+const ROLE_PRIORITY = ['button', 'input', 'link', 'checkbox', 'tab',
+    'listitem', 'list', 'image', 'text', 'unknown'];
+function betterRole(a, b) {
+    return ROLE_PRIORITY.indexOf(a) <= ROLE_PRIORITY.indexOf(b) ? a : b;
+}
+/** Merge same-place + same-text elements across sources; raise confidence per
+ *  corroborating source; keep the stronger role/capabilities. */
+function fuse(elements) {
+    const out = [];
+    // Running max of the ORIGINAL (un-bonused) confidence per merged cluster, so
+    // the agreement bonus is applied once against the base — idempotent for 3+
+    // sources (a11y+ocr+vision must give base+2·bonus, not a compounded value).
+    const baseConf = new Map();
+    for (const el of elements) {
+        const match = out.find(o => o.normalized_text === el.normalized_text &&
+            o.normalized_text !== '' &&
+            o.normalized_text !== undefined &&
+            (0, ui_map_geom_1.iou)(o.bounds, el.bounds) >= OVERLAP_MIN);
+        if (!match) {
+            const copy = { ...el, sources: [...el.sources] };
+            out.push(copy);
+            baseConf.set(copy, el.confidence);
+            continue;
+        }
+        const merged = Array.from(new Set([...match.sources, ...el.sources]));
+        match.sources = merged;
+        match.role = betterRole(match.role, el.role);
+        match.clickable = match.clickable || el.clickable;
+        match.editable = match.editable || el.editable;
+        match.actionable = match.actionable || el.actionable;
+        const base = Math.max(baseConf.get(match) ?? match.confidence, el.confidence);
+        baseConf.set(match, base);
+        match.confidence = Math.min(1, base + AGREEMENT_BONUS * (merged.length - 1));
+        match.state = { ...el.state, ...match.state };
+    }
+    return out;
+}
+//# sourceMappingURL=ui-map-fuse.js.map

package/dist/core/sense/ui-map-fuse.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ui-map-fuse.js","sourceRoot":"","sources":["../../../src/core/sense/ui-map-fuse.ts"],"names":[],"mappings":";;AAcA,oBA8BC;AA3CD,+CAAoC;AAEpC,MAAM,eAAe,GAAG,IAAI,CAAC;AAC7B,MAAM,WAAW,GAAG,GAAG,CAAC,CAAQ,uDAAuD;AACvF,MAAM,aAAa,GAAW,CAAC,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,KAAK;IACzE,UAAU,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;AAElD,SAAS,UAAU,CAAC,CAAO,EAAE,CAAO;IAClC,OAAO,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AACtE,CAAC;AAED;iEACiE;AACjE,SAAgB,IAAI,CAAC,QAAqB;IACxC,MAAM,GAAG,GAAgB,EAAE,CAAC;IAC5B,6EAA6E;IAC7E,2EAA2E;IAC3E,4EAA4E;IAC5E,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAqB,CAAC;IAC9C,KAAK,MAAM,EAAE,IAAI,QAAQ,EAAE,CAAC;QAC1B,MAAM,KAAK,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CACzB,CAAC,CAAC,eAAe,KAAK,EAAE,CAAC,eAAe;YACxC,CAAC,CAAC,eAAe,KAAK,EAAE;YACxB,CAAC,CAAC,eAAe,KAAK,SAAS;YAC/B,IAAA,iBAAG,EAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,WAAW,CAAC,CAAC;QAC3C,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,MAAM,IAAI,GAAG,EAAE,GAAG,EAAE,EAAE,OAAO,EAAE,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC;YACjD,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACf,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC;YAClC,SAAS;QACX,CAAC;QACD,MAAM,MAAM,GAAa,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC,GAAG,KAAK,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAChF,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC;QACvB,KAAK,CAAC,IAAI,GAAG,UAAU,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC;QAC7C,KAAK,CAAC,SAAS,GAAG,KAAK,CAAC,SAAS,IAAI,EAAE,CAAC,SAAS,CAAC;QAClD,KAAK,CAAC,QAAQ,GAAG,KAAK,CAAC,QAAQ,IAAI,EAAE,CAAC,QAAQ,CAAC;QAC/C,KAAK,CAAC,UAAU,GAAG,KAAK,CAAC,UAAU,IAAI,EAAE,CAAC,UAAU,CAAC;QACrD,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,UAAU,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC;QAC9E,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;QAC1B,KAAK,CAAC,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,GAAG,eAAe,GAAG,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;QAC7E,KAAK,CAAC,KAAK,GAAG,EAAE,GAAG,EAAE,CAAC,KAAK,EAAE,GAAG,KAAK,CAAC,KAAK,EAAE,CAAC;IAChD,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC"}

package/dist/core/sense/ui-map-geom.d.ts

@@ -0,0 +1,3 @@
+import type { Bounds } from './ui-map-types';
+/** Intersection-over-union of two [x,y,w,h] screen-space boxes. 0 when disjoint. */
+export declare function iou(a: Bounds, b: Bounds): number;