mobile-device-mcp 0.1.0

package/dist/ai/element-search.js

@@ -0,0 +1,387 @@
+// ============================================================
+// Local Element Search — Fast UI tree search without AI calls
+//
+// Searches UIElement[] by text, content description, resource ID,
+// and class name. Returns an ElementMatch compatible with the AI
+// findElement response so callers can use it as a drop-in fast
+// path before falling back to the AI.
+// ============================================================
+// ----------------------------------------------------------
+// Number-word mapping for natural language queries
+// ----------------------------------------------------------
+const WORD_TO_DIGIT = {
+    zero: "0",
+    one: "1",
+    two: "2",
+    three: "3",
+    four: "4",
+    five: "5",
+    six: "6",
+    seven: "7",
+    eight: "8",
+    nine: "9",
+    ten: "10",
+};
+const DIGIT_TO_WORD = {};
+for (const [word, digit] of Object.entries(WORD_TO_DIGIT)) {
+    DIGIT_TO_WORD[digit] = word;
+}
+// Common UI label aliases — maps user-friendly terms to what Android actually uses
+const ALIASES = {
+    ac: ["clear", "clr", "all_clear"],
+    "all clear": ["clear", "clr", "ac"],
+    clear: ["clr", "ac", "all_clear"],
+    delete: ["del", "backspace"],
+    backspace: ["del", "delete"],
+    back: ["navigate_up", "back"],
+    close: ["dismiss", "close", "cancel"],
+    cancel: ["dismiss", "close", "cancel"],
+    ok: ["confirm", "accept", "done"],
+    confirm: ["ok", "accept", "done"],
+    search: ["search", "find", "query"],
+    settings: ["settings", "preferences", "gear"],
+    more: ["more_options", "overflow", "menu"],
+    share: ["share", "send"],
+    add: ["add", "plus", "new", "create"],
+    subtract: ["minus", "subtract"],
+    minus: ["subtract", "minus"],
+    divide: ["division", "divide"],
+    enter: ["return", "submit", "done"],
+    submit: ["enter", "return", "done"],
+};
+// Words that provide type hints but are not part of the search text
+const TYPE_HINT_WORDS = new Set([
+    "button",
+    "field",
+    "input",
+    "text",
+    "label",
+    "icon",
+    "image",
+    "checkbox",
+    "switch",
+    "toggle",
+    "link",
+    "tab",
+    "menu",
+    "item",
+    "option",
+    "card",
+    "toolbar",
+]);
+// Filler words to strip from queries
+const FILLER_WORDS = new Set([
+    "the",
+    "a",
+    "an",
+    "that",
+    "this",
+    "which",
+    "with",
+    "for",
+    "on",
+    "at",
+    "in",
+    "of",
+    "to",
+    "says",
+    "saying",
+    "labeled",
+    "labelled",
+    "called",
+    "named",
+    "number",
+    "digit",
+]);
+function parseQuery(query) {
+    const raw = query.toLowerCase().trim();
+    const words = raw.split(/\s+/);
+    const searchTokens = [];
+    const typeHints = [];
+    const numericVariants = [];
+    const aliasVariants = [];
+    for (const word of words) {
+        if (FILLER_WORDS.has(word))
+            continue;
+        if (TYPE_HINT_WORDS.has(word)) {
+            typeHints.push(word);
+            continue;
+        }
+        // Convert number words to digits and vice versa
+        if (WORD_TO_DIGIT[word]) {
+            const digit = WORD_TO_DIGIT[word];
+            searchTokens.push(digit);
+            numericVariants.push(word);
+        }
+        else if (DIGIT_TO_WORD[word]) {
+            searchTokens.push(word);
+            numericVariants.push(DIGIT_TO_WORD[word]);
+        }
+        else {
+            searchTokens.push(word);
+        }
+        // Expand aliases for all non-filler words
+        if (ALIASES[word]) {
+            aliasVariants.push(...ALIASES[word]);
+        }
+    }
+    // Also check multi-word aliases (e.g., "all clear")
+    const joined = words.filter(w => !FILLER_WORDS.has(w)).join(" ");
+    if (ALIASES[joined]) {
+        aliasVariants.push(...ALIASES[joined]);
+    }
+    // If all words were fillers/hints, use the full raw query minus fillers
+    if (searchTokens.length === 0) {
+        // Use type hints as search tokens as fallback
+        searchTokens.push(...typeHints);
+    }
+    return { raw, searchTokens, typeHints, numericVariants, aliasVariants };
+}
+/**
+ * Map simplified class name to AnalyzedElement type.
+ */
+function classNameToType(className) {
+    const simple = className.toLowerCase();
+    if (simple.includes("button"))
+        return "button";
+    if (simple.includes("edittext") || simple.includes("textfield"))
+        return "text_field";
+    if (simple.includes("checkbox"))
+        return "checkbox";
+    if (simple.includes("switch") || simple.includes("toggle"))
+        return "switch";
+    if (simple.includes("image"))
+        return "image";
+    if (simple.includes("tab"))
+        return "tab";
+    if (simple.includes("textview") || simple.includes("text"))
+        return "text";
+    return "other";
+}
+/**
+ * Map class name to a suggested action.
+ */
+function suggestedActionForClass(className) {
+    const simple = className.toLowerCase();
+    if (simple.includes("edittext") || simple.includes("textfield"))
+        return "type";
+    if (simple.includes("scroll"))
+        return "scroll";
+    return "tap";
+}
+/**
+ * Flatten the element tree, collecting all elements with their children recursively.
+ */
+function flattenElements(elements) {
+    const result = [];
+    function walk(els) {
+        for (const el of els) {
+            result.push(el);
+            if (el.children) {
+                walk(el.children);
+            }
+        }
+    }
+    walk(elements);
+    return result;
+}
+/**
+ * Get the stripped resource ID (after the last '/').
+ */
+function stripResourceId(resourceId) {
+    const slash = resourceId.lastIndexOf("/");
+    return slash >= 0 ? resourceId.slice(slash + 1) : resourceId;
+}
+/**
+ * Score a single element against the parsed query.
+ * Returns a score from 0.0 to 1.0 and a match reason.
+ */
+function scoreElement(el, pq) {
+    let bestScore = 0;
+    let bestReason = "";
+    const elText = el.text.toLowerCase();
+    const elDesc = el.contentDescription.toLowerCase();
+    const elId = stripResourceId(el.resourceId).toLowerCase();
+    const elClass = el.className.toLowerCase();
+    // All search tokens joined
+    const searchString = pq.searchTokens.join(" ");
+    // Also build variant strings that include numeric word forms and aliases
+    const allVariants = [...pq.searchTokens, ...pq.numericVariants, ...pq.aliasVariants];
+    // ---- Strategy 1: Exact text match (highest confidence) ----
+    if (elText && elText === searchString) {
+        bestScore = 1.0;
+        bestReason = `Exact text match: "${el.text}"`;
+    }
+    // ---- Strategy 2: Text contains all search tokens ----
+    if (bestScore < 0.95 && elText) {
+        const tokensMatched = pq.searchTokens.filter((t) => elText.includes(t)).length;
+        if (tokensMatched === pq.searchTokens.length && pq.searchTokens.length > 0) {
+            // All tokens found in text
+            const ratio = searchString.length / Math.max(elText.length, 1);
+            const score = 0.85 + 0.1 * Math.min(ratio, 1);
+            if (score > bestScore) {
+                bestScore = score;
+                bestReason = `Text contains all tokens: "${el.text}"`;
+            }
+        }
+        else if (tokensMatched > 0) {
+            const score = 0.5 * (tokensMatched / pq.searchTokens.length);
+            if (score > bestScore) {
+                bestScore = score;
+                bestReason = `Text partial match: "${el.text}"`;
+            }
+        }
+    }
+    // ---- Strategy 2b: Text matches numeric variants ----
+    if (bestScore < 0.9 && elText && pq.numericVariants.length > 0) {
+        const variantsMatched = allVariants.filter((v) => elText.includes(v)).length;
+        if (variantsMatched > 0) {
+            const score = 0.8 * (variantsMatched / allVariants.length);
+            if (score > bestScore) {
+                bestScore = score;
+                bestReason = `Text matches numeric variant: "${el.text}"`;
+            }
+        }
+    }
+    // ---- Strategy 3: Content description match ----
+    if (bestScore < 0.9 && elDesc) {
+        // Exact match on search string
+        if (elDesc === searchString) {
+            const score = 0.95;
+            if (score > bestScore) {
+                bestScore = score;
+                bestReason = `Exact content description match: "${el.contentDescription}"`;
+            }
+        }
+        // Exact match on an alias variant (e.g., "AC" → desc="clear")
+        if (bestScore < 0.9 && pq.aliasVariants.some((a) => elDesc === a)) {
+            const score = 0.9;
+            if (score > bestScore) {
+                bestScore = score;
+                bestReason = `Alias match on content description: "${el.contentDescription}"`;
+            }
+        }
+        // Partial token/variant match
+        if (bestScore < 0.85) {
+            const tokensMatched = allVariants.filter((t) => elDesc.includes(t)).length;
+            if (tokensMatched > 0) {
+                // Use max of search tokens and alias variants separately to avoid dilution
+                const searchMatched = pq.searchTokens.filter((t) => elDesc.includes(t)).length;
+                const aliasMatched = pq.aliasVariants.filter((t) => elDesc.includes(t)).length;
+                const bestRatio = Math.max(pq.searchTokens.length > 0 ? searchMatched / pq.searchTokens.length : 0, pq.aliasVariants.length > 0 ? aliasMatched / pq.aliasVariants.length : 0);
+                const score = 0.7 * bestRatio;
+                if (score > bestScore) {
+                    bestScore = score;
+                    bestReason = `Content description match: "${el.contentDescription}"`;
+                }
+            }
+        }
+    }
+    // ---- Strategy 3b: Alias matches on text ----
+    if (bestScore < 0.9 && elText && pq.aliasVariants.length > 0) {
+        if (pq.aliasVariants.some((a) => elText === a)) {
+            const score = 0.9;
+            if (score > bestScore) {
+                bestScore = score;
+                bestReason = `Alias match on text: "${el.text}"`;
+            }
+        }
+    }
+    // ---- Strategy 4: Resource ID match ----
+    if (bestScore < 0.85 && elId) {
+        // Resource IDs use snake_case, so split on underscores
+        const idParts = elId.split(/[_\-]/);
+        // Exact alias match on resource ID
+        if (pq.aliasVariants.some((a) => idParts.includes(a))) {
+            const score = 0.8;
+            if (score > bestScore) {
+                bestScore = score;
+                bestReason = `Alias match on resource ID: "${el.resourceId}"`;
+            }
+        }
+        // Partial token match
+        if (bestScore < 0.75) {
+            const tokensMatched = pq.searchTokens.filter((t) => idParts.some((p) => p === t || p.includes(t))).length;
+            if (tokensMatched > 0) {
+                const score = 0.6 * (tokensMatched / pq.searchTokens.length);
+                if (score > bestScore) {
+                    bestScore = score;
+                    bestReason = `Resource ID match: "${el.resourceId}"`;
+                }
+            }
+        }
+    }
+    // ---- Strategy 5: Class name hint match ----
+    if (pq.typeHints.length > 0 && bestScore > 0) {
+        // Boost score if the type hint matches the element class
+        const classMatches = pq.typeHints.some((hint) => elClass.includes(hint));
+        if (classMatches) {
+            bestScore = Math.min(bestScore + 0.1, 1.0);
+            bestReason += " (type hint match)";
+        }
+    }
+    // ---- Penalty: non-interactive elements get a small penalty ----
+    if (!el.clickable && !el.focusable && !el.scrollable) {
+        bestScore *= 0.9;
+    }
+    // ---- Penalty: empty/invisible elements ----
+    if (el.bounds.left === el.bounds.right || el.bounds.top === el.bounds.bottom) {
+        bestScore *= 0.1;
+    }
+    return { element: el, score: bestScore, matchReason: bestReason };
+}
+// ----------------------------------------------------------
+// Public API
+// ----------------------------------------------------------
+/**
+ * Search UI elements locally by text matching.
+ * This is INSTANT and FREE — no AI API call needed.
+ * Falls back to AI only when local search fails (returns found: false).
+ *
+ * @param elements - The UI element tree from the device
+ * @param query - Natural language query like "the number 7 button"
+ * @returns An ElementMatch result compatible with AI findElement output
+ */
+export function searchElementsLocally(elements, query) {
+    if (!elements || elements.length === 0) {
+        return { found: false, confidence: 0 };
+    }
+    const pq = parseQuery(query);
+    // If we couldn't extract any meaningful tokens, bail out to AI
+    if (pq.searchTokens.length === 0) {
+        return { found: false, confidence: 0 };
+    }
+    // Flatten the tree and score every element
+    const flat = flattenElements(elements);
+    const scored = flat
+        .map((el) => scoreElement(el, pq))
+        .filter((s) => s.score > 0)
+        .sort((a, b) => b.score - a.score);
+    if (scored.length === 0) {
+        return { found: false, confidence: 0 };
+    }
+    const best = scored[0];
+    // Convert UIElement to AnalyzedElement
+    function toAnalyzed(se) {
+        const el = se.element;
+        return {
+            description: se.matchReason,
+            type: classNameToType(el.className),
+            text: el.text || el.contentDescription,
+            bounds: { ...el.bounds },
+            suggestedAction: suggestedActionForClass(el.className),
+            confidence: se.score,
+        };
+    }
+    const alternatives = scored
+        .slice(1, 4) // up to 3 runners-up
+        .filter((s) => s.score > 0.2)
+        .map(toAnalyzed);
+    return {
+        found: best.score > 0.7,
+        element: toAnalyzed(best),
+        confidence: best.score,
+        alternatives,
+    };
+}
+//# sourceMappingURL=element-search.js.map

package/dist/ai/element-search.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"element-search.js","sourceRoot":"","sources":["../../src/ai/element-search.ts"],"names":[],"mappings":"AAAA,+DAA+D;AAC/D,8DAA8D;AAC9D,EAAE;AACF,kEAAkE;AAClE,iEAAiE;AACjE,+DAA+D;AAC/D,sCAAsC;AACtC,+DAA+D;AAI/D,6DAA6D;AAC7D,mDAAmD;AACnD,6DAA6D;AAE7D,MAAM,aAAa,GAA2B;IAC5C,IAAI,EAAE,GAAG;IACT,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,KAAK,EAAE,GAAG;IACV,IAAI,EAAE,GAAG;IACT,IAAI,EAAE,GAAG;IACT,GAAG,EAAE,GAAG;IACR,KAAK,EAAE,GAAG;IACV,KAAK,EAAE,GAAG;IACV,IAAI,EAAE,GAAG;IACT,GAAG,EAAE,IAAI;CACV,CAAC;AAEF,MAAM,aAAa,GAA2B,EAAE,CAAC;AACjD,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,aAAa,CAAC,EAAE,CAAC;IAC1D,aAAa,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC;AAC9B,CAAC;AAED,mFAAmF;AACnF,MAAM,OAAO,GAA6B;IACxC,EAAE,EAAE,CAAC,OAAO,EAAE,KAAK,EAAE,WAAW,CAAC;IACjC,WAAW,EAAE,CAAC,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC;IACnC,KAAK,EAAE,CAAC,KAAK,EAAE,IAAI,EAAE,WAAW,CAAC;IACjC,MAAM,EAAE,CAAC,KAAK,EAAE,WAAW,CAAC;IAC5B,SAAS,EAAE,CAAC,KAAK,EAAE,QAAQ,CAAC;IAC5B,IAAI,EAAE,CAAC,aAAa,EAAE,MAAM,CAAC;IAC7B,KAAK,EAAE,CAAC,SAAS,EAAE,OAAO,EAAE,QAAQ,CAAC;IACrC,MAAM,EAAE,CAAC,SAAS,EAAE,OAAO,EAAE,QAAQ,CAAC;IACtC,EAAE,EAAE,CAAC,SAAS,EAAE,QAAQ,EAAE,MAAM,CAAC;IACjC,OAAO,EAAE,CAAC,IAAI,EAAE,QAAQ,EAAE,MAAM,CAAC;IACjC,MAAM,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC;IACnC,QAAQ,EAAE,CAAC,UAAU,EAAE,aAAa,EAAE,MAAM,CAAC;IAC7C,IAAI,EAAE,CAAC,cAAc,EAAE,UAAU,EAAE,MAAM,CAAC;IAC1C,KAAK,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC;IACxB,GAAG,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,CAAC;IACrC,QAAQ,EAAE,CAAC,OAAO,EAAE,UAAU,CAAC;IAC/B,KAAK,EAAE,CAAC,UAAU,EAAE,OAAO,CAAC;IAC5B,MAAM,EAAE,CAAC,UAAU,EAAE,QAAQ,CAAC;IAC9B,KAAK,EAAE,CAAC,QAAQ,EAAE,QAAQ,EAAE,MAAM,CAAC;IACnC,MAAM,EAAE,CAAC,OAAO,EAAE,QAAQ,EAAE,MAAM,CAAC;CACpC,CAAC;AAEF,oEAAoE;AACpE,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC;IAC9B,QAAQ;IACR,OAAO;IACP,OAAO;IACP,MAAM;IACN,OAAO;IACP,MAAM;IACN,OAAO;IACP,UAAU;IACV,QAAQ;IACR,QAAQ;IACR,MAAM;IACN,KAAK;IACL,MAAM;IACN,MAAM;IACN,QAAQ;IACR,MAAM;IACN,SAAS;CACV,CAAC,CAAC;AAEH,qCAAqC;AACrC,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC;IAC3B,KAAK;IACL,GAAG;IACH,IAAI;IACJ,MAAM;IACN,MAAM;IACN,OAAO;IACP,MAAM;IACN,KAAK;IACL,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,MAAM;IACN,QAAQ;IACR,SAAS;IACT,UAAU;IACV,QAAQ;IACR,OAAO;IACP,QAAQ;IACR,OAAO;CACR,CAAC,CAAC;AAmBH,SAAS,UAAU,CAAC,KAAa;IAC/B,MAAM,GAAG,GAAG,KAAK,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,CAAC;IACvC,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAE/B,MAAM,YAAY,GAAa,EAAE,CAAC;IAClC,MAAM,SAAS,GAAa,EAAE,CAAC;IAC/B,MAAM,eAAe,GAAa,EAAE,CAAC;IACrC,MAAM,aAAa,GAAa,EAAE,CAAC;IAEnC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC;YAAE,SAAS;QAErC,IAAI,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YAC9B,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACrB,SAAS;QACX,CAAC;QAED,gDAAgD;QAChD,IAAI,aAAa,CAAC,IAAI,CAAC,EAAE,CAAC;YACxB,MAAM,KAAK,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC;YAClC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACzB,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7B,CAAC;aAAM,IAAI,aAAa,CAAC,IAAI,CAAC,EAAE,CAAC;YAC/B,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACxB,eAAe,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC;QAC5C,CAAC;aAAM,CAAC;YACN,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1B,CAAC;QAED,0CAA0C;QAC1C,IAAI,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;YAClB,aAAa,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;QACvC,CAAC;IACH,CAAC;IAED,oDAAoD;IACpD,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACjE,IAAI,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QACpB,aAAa,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;IACzC,CAAC;IAED,wEAAwE;IACxE,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC9B,8CAA8C;QAC9C,YAAY,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC;IAClC,CAAC;IAED,OAAO,EAAE,GAAG,EAAE,YAAY,EAAE,SAAS,EAAE,eAAe,EAAE,aAAa,EAAE,CAAC;AAC1E,CAAC;AAYD;;GAEG;AACH,SAAS,eAAe,CAAC,SAAiB;IACxC,MAAM,MAAM,GAAG,SAAS,CAAC,WAAW,EAAE,CAAC;IACvC,IAAI,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC;QAAE,OAAO,QAAQ,CAAC;IAC/C,IAAI,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC;QAAE,OAAO,YAAY,CAAC;IACrF,IAAI,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC;QAAE,OAAO,UAAU,CAAC;IACnD,IAAI,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC;QAAE,OAAO,QAAQ,CAAC;IAC5E,IAAI,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC;QAAE,OAAO,OAAO,CAAC;IAC7C,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IACzC,IAAI,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC;QAAE,OAAO,MAAM,CAAC;IAC1E,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;GAEG;AACH,SAAS,uBAAuB,CAAC,SAAiB;IAChD,MAAM,MAAM,GAAG,SAAS,CAAC,WAAW,EAAE,CAAC;IACvC,IAAI,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC;QAAE,OAAO,MAAM,CAAC;IAC/E,IAAI,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC;QAAE,OAAO,QAAQ,CAAC;IAC/C,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;GAEG;AACH,SAAS,eAAe,CAAC,QAAqB;IAC5C,MAAM,MAAM,GAAgB,EAAE,CAAC;IAC/B,SAAS,IAAI,CAAC,GAAgB;QAC5B,KAAK,MAAM,EAAE,IAAI,GAAG,EAAE,CAAC;YACrB,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAChB,IAAI,EAAE,CAAC,QAAQ,EAAE,CAAC;gBAChB,IAAI,CAAC,EAAE,CAAC,QAAQ,CAAC,CAAC;YACpB,CAAC;QACH,CAAC;IACH,CAAC;IACD,IAAI,CAAC,QAAQ,CAAC,CAAC;IACf,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;GAEG;AACH,SAAS,eAAe,CAAC,UAAkB;IACzC,MAAM,KAAK,GAAG,UAAU,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IAC1C,OAAO,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC;AAC/D,CAAC;AAED;;;GAGG;AACH,SAAS,YAAY,CAAC,EAAa,EAAE,EAAe;IAClD,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,UAAU,GAAG,EAAE,CAAC;IAEpB,MAAM,MAAM,GAAG,EAAE,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;IACrC,MAAM,MAAM,GAAG,EAAE,CAAC,kBAAkB,CAAC,WAAW,EAAE,CAAC;IACnD,MAAM,IAAI,GAAG,eAAe,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,WAAW,EAAE,CAAC;IAC1D,MAAM,OAAO,GAAG,EAAE,CAAC,SAAS,CAAC,WAAW,EAAE,CAAC;IAE3C,2BAA2B;IAC3B,MAAM,YAAY,GAAG,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAE/C,yEAAyE;IACzE,MAAM,WAAW,GAAG,CAAC,GAAG,EAAE,CAAC,YAAY,EAAE,GAAG,EAAE,CAAC,eAAe,EAAE,GAAG,EAAE,CAAC,aAAa,CAAC,CAAC;IAErF,8DAA8D;IAC9D,IAAI,MAAM,IAAI,MAAM,KAAK,YAAY,EAAE,CAAC;QACtC,SAAS,GAAG,GAAG,CAAC;QAChB,UAAU,GAAG,sBAAsB,EAAE,CAAC,IAAI,GAAG,CAAC;IAChD,CAAC;IAED,wDAAwD;IACxD,IAAI,SAAS,GAAG,IAAI,IAAI,MAAM,EAAE,CAAC;QAC/B,MAAM,aAAa,GAAG,EAAE,CAAC,YAAY,CAAC,MAAM,CAC1C,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAC1B,CAAC,MAAM,CAAC;QACT,IAAI,aAAa,KAAK,EAAE,CAAC,YAAY,CAAC,MAAM,IAAI,EAAE,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC3E,2BAA2B;YAC3B,MAAM,KAAK,GAAG,YAAY,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YAC/D,MAAM,KAAK,GAAG,IAAI,GAAG,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;YAC9C,IAAI,KAAK,GAAG,SAAS,EAAE,CAAC;gBACtB,SAAS,GAAG,KAAK,CAAC;gBAClB,UAAU,GAAG,8BAA8B,EAAE,CAAC,IAAI,GAAG,CAAC;YACxD,CAAC;QACH,CAAC;aAAM,IAAI,aAAa,GAAG,CAAC,EAAE,CAAC;YAC7B,MAAM,KAAK,GAAG,GAAG,GAAG,CAAC,aAAa,GAAG,EAAE,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;YAC7D,IAAI,KAAK,GAAG,SAAS,EAAE,CAAC;gBACtB,SAAS,GAAG,KAAK,CAAC;gBAClB,UAAU,GAAG,wBAAwB,EAAE,CAAC,IAAI,GAAG,CAAC;YAClD,CAAC;QACH,CAAC;IACH,CAAC;IAED,uDAAuD;IACvD,IAAI,SAAS,GAAG,GAAG,IAAI,MAAM,IAAI,EAAE,CAAC,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC/D,MAAM,eAAe,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;QAC7E,IAAI,eAAe,GAAG,CAAC,EAAE,CAAC;YACxB,MAAM,KAAK,GAAG,GAAG,GAAG,CAAC,eAAe,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC;YAC3D,IAAI,KAAK,GAAG,SAAS,EAAE,CAAC;gBACtB,SAAS,GAAG,KAAK,CAAC;gBAClB,UAAU,GAAG,kCAAkC,EAAE,CAAC,IAAI,GAAG,CAAC;YAC5D,CAAC;QACH,CAAC;IACH,CAAC;IAED,kDAAkD;IAClD,IAAI,SAAS,GAAG,GAAG,IAAI,MAAM,EAAE,CAAC;QAC9B,+BAA+B;QAC/B,IAAI,MAAM,KAAK,YAAY,EAAE,CAAC;YAC5B,MAAM,KAAK,GAAG,IAAI,CAAC;YACnB,IAAI,KAAK,GAAG,SAAS,EAAE,CAAC;gBACtB,SAAS,GAAG,KAAK,CAAC;gBAClB,UAAU,GAAG,qCAAqC,EAAE,CAAC,kBAAkB,GAAG,CAAC;YAC7E,CAAC;QACH,CAAC;QACD,8DAA8D;QAC9D,IAAI,SAAS,GAAG,GAAG,IAAI,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,KAAK,CAAC,CAAC,EAAE,CAAC;YAClE,MAAM,KAAK,GAAG,GAAG,CAAC;YAClB,IAAI,KAAK,GAAG,SAAS,EAAE,CAAC;gBACtB,SAAS,GAAG,KAAK,CAAC;gBAClB,UAAU,GAAG,wCAAwC,EAAE,CAAC,kBAAkB,GAAG,CAAC;YAChF,CAAC;QACH,CAAC;QACD,8BAA8B;QAC9B,IAAI,SAAS,GAAG,IAAI,EAAE,CAAC;YACrB,MAAM,aAAa,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;YAC3E,IAAI,aAAa,GAAG,CAAC,EAAE,CAAC;gBACtB,2EAA2E;gBAC3E,MAAM,aAAa,GAAG,EAAE,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;gBAC/E,MAAM,YAAY,GAAG,EAAE,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;gBAC/E,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CACxB,EAAE,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,aAAa,GAAG,EAAE,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EACvE,EAAE,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,GAAG,EAAE,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CACzE,CAAC;gBACF,MAAM,KAAK,GAAG,GAAG,GAAG,SAAS,CAAC;gBAC9B,IAAI,KAAK,GAAG,SAAS,EAAE,CAAC;oBACtB,SAAS,GAAG,KAAK,CAAC;oBAClB,UAAU,GAAG,+BAA+B,EAAE,CAAC,kBAAkB,GAAG,CAAC;gBACvE,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,+CAA+C;IAC/C,IAAI,SAAS,GAAG,GAAG,IAAI,MAAM,IAAI,EAAE,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC7D,IAAI,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,KAAK,CAAC,CAAC,EAAE,CAAC;YAC/C,MAAM,KAAK,GAAG,GAAG,CAAC;YAClB,IAAI,KAAK,GAAG,SAAS,EAAE,CAAC;gBACtB,SAAS,GAAG,KAAK,CAAC;gBAClB,UAAU,GAAG,yBAAyB,EAAE,CAAC,IAAI,GAAG,CAAC;YACnD,CAAC;QACH,CAAC;IACH,CAAC;IAED,0CAA0C;IAC1C,IAAI,SAAS,GAAG,IAAI,IAAI,IAAI,EAAE,CAAC;QAC7B,uDAAuD;QACvD,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QACpC,mCAAmC;QACnC,IAAI,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YACtD,MAAM,KAAK,GAAG,GAAG,CAAC;YAClB,IAAI,KAAK,GAAG,SAAS,EAAE,CAAC;gBACtB,SAAS,GAAG,KAAK,CAAC;gBAClB,UAAU,GAAG,gCAAgC,EAAE,CAAC,UAAU,GAAG,CAAC;YAChE,CAAC;QACH,CAAC;QACD,sBAAsB;QACtB,IAAI,SAAS,GAAG,IAAI,EAAE,CAAC;YACrB,MAAM,aAAa,GAAG,EAAE,CAAC,YAAY,CAAC,MAAM,CAC1C,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CACrD,CAAC,MAAM,CAAC;YACT,IAAI,aAAa,GAAG,CAAC,EAAE,CAAC;gBACtB,MAAM,KAAK,GAAG,GAAG,GAAG,CAAC,aAAa,GAAG,EAAE,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;gBAC7D,IAAI,KAAK,GAAG,SAAS,EAAE,CAAC;oBACtB,SAAS,GAAG,KAAK,CAAC;oBAClB,UAAU,GAAG,uBAAuB,EAAE,CAAC,UAAU,GAAG,CAAC;gBACvD,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,8CAA8C;IAC9C,IAAI,EAAE,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;QAC7C,yDAAyD;QACzD,MAAM,YAAY,GAAG,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;QACzE,IAAI,YAAY,EAAE,CAAC;YACjB,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,GAAG,GAAG,EAAE,GAAG,CAAC,CAAC;YAC3C,UAAU,IAAI,oBAAoB,CAAC;QACrC,CAAC;IACH,CAAC;IAED,kEAAkE;IAClE,IAAI,CAAC,EAAE,CAAC,SAAS,IAAI,CAAC,EAAE,CAAC,SAAS,IAAI,CAAC,EAAE,CAAC,UAAU,EAAE,CAAC;QACrD,SAAS,IAAI,GAAG,CAAC;IACnB,CAAC;IAED,8CAA8C;IAC9C,IAAI,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC,MAAM,CAAC,GAAG,KAAK,EAAE,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QAC7E,SAAS,IAAI,GAAG,CAAC;IACnB,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,WAAW,EAAE,UAAU,EAAE,CAAC;AACpE,CAAC;AAED,6DAA6D;AAC7D,aAAa;AACb,6DAA6D;AAE7D;;;;;;;;GAQG;AACH,MAAM,UAAU,qBAAqB,CACnC,QAAqB,EACrB,KAAa;IAEb,IAAI,CAAC,QAAQ,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvC,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC,EAAE,CAAC;IACzC,CAAC;IAED,MAAM,EAAE,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;IAE7B,+DAA+D;IAC/D,IAAI,EAAE,CAAC,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACjC,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC,EAAE,CAAC;IACzC,CAAC;IAED,2CAA2C;IAC3C,MAAM,IAAI,GAAG,eAAe,CAAC,QAAQ,CAAC,CAAC;IACvC,MAAM,MAAM,GAAG,IAAI;SAChB,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,YAAY,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;SACjC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC;SAC1B,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;IAErC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC,EAAE,CAAC;IACzC,CAAC;IAED,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;IAEvB,uCAAuC;IACvC,SAAS,UAAU,CAAC,EAAiB;QACnC,MAAM,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC;QACtB,OAAO;YACL,WAAW,EAAE,EAAE,CAAC,WAAW;YAC3B,IAAI,EAAE,eAAe,CAAC,EAAE,CAAC,SAAS,CAAC;YACnC,IAAI,EAAE,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC,kBAAkB;YACtC,MAAM,EAAE,EAAE,GAAG,EAAE,CAAC,MAAM,EAAE;YACxB,eAAe,EAAE,uBAAuB,CAAC,EAAE,CAAC,SAAS,CAAC;YACtD,UAAU,EAAE,EAAE,CAAC,KAAK;SACrB,CAAC;IACJ,CAAC;IAED,MAAM,YAAY,GAAG,MAAM;SACxB,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,qBAAqB;SACjC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,GAAG,CAAC;SAC5B,GAAG,CAAC,UAAU,CAAC,CAAC;IAEnB,OAAO;QACL,KAAK,EAAE,IAAI,CAAC,KAAK,GAAG,GAAG;QACvB,OAAO,EAAE,UAAU,CAAC,IAAI,CAAC;QACzB,UAAU,EAAE,IAAI,CAAC,KAAK;QACtB,YAAY;KACb,CAAC;AACJ,CAAC"}

package/dist/ai/prompts.d.ts

@@ -0,0 +1,27 @@
+import type { UIElement } from '../types.js';
+export interface UIElementSummary {
+    idx: number;
+    text: string;
+    desc: string;
+    type: string;
+    id: string;
+    cx: number;
+    cy: number;
+    actions: string[];
+}
+export declare const PROMPTS: {
+    readonly ANALYZE_SCREEN: "You are a mobile app screenshot analyzer. Return ONLY valid JSON, no markdown or explanation.\n\nCoordinate system: origin (0,0) at top-left, x increases right, y increases down.\n\nJSON schema:\n{\"description\":\"string\",\"appName\":\"string\",\"screenType\":\"string\",\"elements\":[{\"description\":\"string\",\"type\":\"string\",\"text\":\"string\",\"bounds\":{\"left\":0,\"top\":0,\"right\":0,\"bottom\":0,\"centerX\":0,\"centerY\":0},\"suggestedAction\":\"string\",\"confidence\":0.0}],\"visibleText\":[\"string\"],\"suggestions\":[\"string\"]}\n\nscreenType: one of \"login\",\"settings\",\"list\",\"detail\",\"dialog\",\"home\",\"search\",\"menu\",\"form\",\"media\",\"navigation\",\"error\",\"loading\",\"onboarding\",\"other\".\ntype: one of \"button\",\"text_field\",\"checkbox\",\"switch\",\"icon\",\"image\",\"link\",\"tab\",\"text\",\"toolbar\",\"menu_item\",\"list_item\",\"card\",\"other\".\nsuggestedAction: one of \"tap\",\"type\",\"scroll\",\"long_press\",\"swipe\",\"none\".\nconfidence: 0.0 to 1.0.\n\nIdentify all interactive elements with precise bounding boxes. Include all visible text strings.";
+    readonly FIND_ELEMENT: "You are a UI element locator for mobile app screenshots. Return ONLY valid JSON, no markdown or explanation.\n\nCoordinate system: origin (0,0) at top-left, x increases right, y increases down.\n\nJSON schema:\n{\"found\":true,\"element\":{\"description\":\"string\",\"type\":\"string\",\"text\":\"string\",\"bounds\":{\"left\":0,\"top\":0,\"right\":0,\"bottom\":0,\"centerX\":0,\"centerY\":0},\"suggestedAction\":\"string\",\"confidence\":0.0},\"confidence\":0.0,\"alternatives\":[]}\n\nIf no match, set found=false, element=null, confidence=0, and list any partial matches in alternatives.\nconfidence: 0.0 to 1.0. Return the single best match as element, up to 3 runners-up in alternatives.";
+    readonly SUGGEST_ACTIONS: "You are a mobile app automation planner. Given the current screen and a goal, plan a sequence of actions. Return ONLY valid JSON, no markdown or explanation.\n\nCoordinate system: origin (0,0) at top-left, x increases right, y increases down.\n\nJSON schema:\n{\"goal\":\"string\",\"steps\":[{\"step\":1,\"action\":\"tap\",\"target\":\"string\",\"coordinates\":{\"x\":0,\"y\":0},\"text\":null,\"key\":null,\"swipe\":null,\"description\":\"string\"}]}\n\naction: one of \"tap\",\"type\",\"swipe\",\"press_key\",\"wait\",\"long_press\".\nFor \"type\": set text to the string to enter.\nFor \"press_key\": set key (e.g. \"KEYCODE_BACK\",\"KEYCODE_HOME\",\"KEYCODE_ENTER\").\nFor \"swipe\": set swipe to {\"startX\":0,\"startY\":0,\"endX\":0,\"endY\":0}.\nFor \"wait\": coordinates may be null.\nProvide precise coordinates for tap/long_press targets. Keep steps minimal.";
+    readonly VISUAL_DIFF: "You are a visual diff analyzer for mobile app screenshots. You will receive two images: the first is \"before\", the second is \"after\". Return ONLY valid JSON, no markdown or explanation.\n\nJSON schema:\n{\"hasChanges\":true,\"summary\":\"string\",\"changes\":[{\"description\":\"string\",\"region\":\"string\",\"type\":\"added|removed|changed\"}]}\n\nregion: one of \"top-left\",\"top-center\",\"top-right\",\"center-left\",\"center\",\"center-right\",\"bottom-left\",\"bottom-center\",\"bottom-right\".\ntype: one of \"added\",\"removed\",\"changed\".\nIf screens are identical, set hasChanges=false, summary=\"No changes detected\", changes=[].";
+    readonly EXTRACT_TEXT: "You are an OCR engine for mobile app screenshots. Extract every readable text element. Return ONLY valid JSON, no markdown or explanation.\n\nJSON schema:\n{\"texts\":[\"string\"]}\n\nRead in order: top to bottom, left to right. Include all labels, buttons, headers, body text, hints, placeholders, status bar text, and navigation items. Preserve original casing and punctuation.";
+    readonly VERIFY_SCREEN: "You are a screen state verifier for mobile app testing. Check whether a given assertion holds true for the screenshot. Return ONLY valid JSON, no markdown or explanation.\n\nJSON schema:\n{\"verified\":true,\"confidence\":0.0,\"details\":\"string\",\"evidence\":[\"string\"]}\n\nverified: true if assertion holds, false otherwise.\nconfidence: 0.0 to 1.0.\ndetails: brief explanation of your judgment.\nevidence: list of specific visual observations supporting your conclusion.";
+};
+export declare function summarizeUIElements(elements: UIElement[]): UIElementSummary[];
+export declare function buildAnalyzeScreenPrompt(uiElements?: UIElementSummary[]): string;
+export declare function buildFindElementPrompt(query: string, uiElements?: UIElementSummary[]): string;
+export declare function buildSuggestActionsPrompt(goal: string, currentApp?: string, uiElements?: UIElementSummary[]): string;
+export declare function buildVisualDiffPrompt(): string;
+export declare function buildExtractTextPrompt(): string;
+export declare function buildVerifyScreenPrompt(assertion: string, uiElements?: UIElementSummary[]): string;
+//# sourceMappingURL=prompts.d.ts.map

package/dist/ai/prompts.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"prompts.d.ts","sourceRoot":"","sources":["../../src/ai/prompts.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAM7C,MAAM,WAAW,gBAAgB;IAC/B,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,EAAE,EAAE,MAAM,CAAC;IACX,EAAE,EAAE,MAAM,CAAC;IACX,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,MAAM,EAAE,CAAC;CACnB;AAsED,eAAO,MAAM,OAAO;;;;;;;CAOV,CAAC;AAMX,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,SAAS,EAAE,GAAG,gBAAgB,EAAE,CAI7E;AA4DD,wBAAgB,wBAAwB,CAAC,UAAU,CAAC,EAAE,gBAAgB,EAAE,GAAG,MAAM,CAEhF;AAED,wBAAgB,sBAAsB,CAAC,KAAK,EAAE,MAAM,EAAE,UAAU,CAAC,EAAE,gBAAgB,EAAE,GAAG,MAAM,CAE7F;AAED,wBAAgB,yBAAyB,CACvC,IAAI,EAAE,MAAM,EACZ,UAAU,CAAC,EAAE,MAAM,EACnB,UAAU,CAAC,EAAE,gBAAgB,EAAE,GAC9B,MAAM,CAGR;AAED,wBAAgB,qBAAqB,IAAI,MAAM,CAE9C;AAED,wBAAgB,sBAAsB,IAAI,MAAM,CAE/C;AAED,wBAAgB,uBAAuB,CACrC,SAAS,EAAE,MAAM,EACjB,UAAU,CAAC,EAAE,gBAAgB,EAAE,GAC9B,MAAM,CAER"}

package/dist/ai/prompts.js

@@ -0,0 +1,153 @@
+// ============================================================
+// AI Visual Analysis — System Prompts & User Prompt Builders
+// ============================================================
+// ----------------------------------------------------------
+// System Prompts
+// ----------------------------------------------------------
+const ANALYZE_SCREEN = `You are a mobile app screenshot analyzer. Return ONLY valid JSON, no markdown or explanation.
+
+Coordinate system: origin (0,0) at top-left, x increases right, y increases down.
+
+JSON schema:
+{"description":"string","appName":"string","screenType":"string","elements":[{"description":"string","type":"string","text":"string","bounds":{"left":0,"top":0,"right":0,"bottom":0,"centerX":0,"centerY":0},"suggestedAction":"string","confidence":0.0}],"visibleText":["string"],"suggestions":["string"]}
+
+screenType: one of "login","settings","list","detail","dialog","home","search","menu","form","media","navigation","error","loading","onboarding","other".
+type: one of "button","text_field","checkbox","switch","icon","image","link","tab","text","toolbar","menu_item","list_item","card","other".
+suggestedAction: one of "tap","type","scroll","long_press","swipe","none".
+confidence: 0.0 to 1.0.
+
+Identify all interactive elements with precise bounding boxes. Include all visible text strings.`;
+const FIND_ELEMENT = `You are a UI element locator for mobile app screenshots. Return ONLY valid JSON, no markdown or explanation.
+
+Coordinate system: origin (0,0) at top-left, x increases right, y increases down.
+
+JSON schema:
+{"found":true,"element":{"description":"string","type":"string","text":"string","bounds":{"left":0,"top":0,"right":0,"bottom":0,"centerX":0,"centerY":0},"suggestedAction":"string","confidence":0.0},"confidence":0.0,"alternatives":[]}
+
+If no match, set found=false, element=null, confidence=0, and list any partial matches in alternatives.
+confidence: 0.0 to 1.0. Return the single best match as element, up to 3 runners-up in alternatives.`;
+const SUGGEST_ACTIONS = `You are a mobile app automation planner. Given the current screen and a goal, plan a sequence of actions. Return ONLY valid JSON, no markdown or explanation.
+
+Coordinate system: origin (0,0) at top-left, x increases right, y increases down.
+
+JSON schema:
+{"goal":"string","steps":[{"step":1,"action":"tap","target":"string","coordinates":{"x":0,"y":0},"text":null,"key":null,"swipe":null,"description":"string"}]}
+
+action: one of "tap","type","swipe","press_key","wait","long_press".
+For "type": set text to the string to enter.
+For "press_key": set key (e.g. "KEYCODE_BACK","KEYCODE_HOME","KEYCODE_ENTER").
+For "swipe": set swipe to {"startX":0,"startY":0,"endX":0,"endY":0}.
+For "wait": coordinates may be null.
+Provide precise coordinates for tap/long_press targets. Keep steps minimal.`;
+const VISUAL_DIFF = `You are a visual diff analyzer for mobile app screenshots. You will receive two images: the first is "before", the second is "after". Return ONLY valid JSON, no markdown or explanation.
+
+JSON schema:
+{"hasChanges":true,"summary":"string","changes":[{"description":"string","region":"string","type":"added|removed|changed"}]}
+
+region: one of "top-left","top-center","top-right","center-left","center","center-right","bottom-left","bottom-center","bottom-right".
+type: one of "added","removed","changed".
+If screens are identical, set hasChanges=false, summary="No changes detected", changes=[].`;
+const EXTRACT_TEXT = `You are an OCR engine for mobile app screenshots. Extract every readable text element. Return ONLY valid JSON, no markdown or explanation.
+
+JSON schema:
+{"texts":["string"]}
+
+Read in order: top to bottom, left to right. Include all labels, buttons, headers, body text, hints, placeholders, status bar text, and navigation items. Preserve original casing and punctuation.`;
+const VERIFY_SCREEN = `You are a screen state verifier for mobile app testing. Check whether a given assertion holds true for the screenshot. Return ONLY valid JSON, no markdown or explanation.
+
+JSON schema:
+{"verified":true,"confidence":0.0,"details":"string","evidence":["string"]}
+
+verified: true if assertion holds, false otherwise.
+confidence: 0.0 to 1.0.
+details: brief explanation of your judgment.
+evidence: list of specific visual observations supporting your conclusion.`;
+export const PROMPTS = {
+    ANALYZE_SCREEN,
+    FIND_ELEMENT,
+    SUGGEST_ACTIONS,
+    VISUAL_DIFF,
+    EXTRACT_TEXT,
+    VERIFY_SCREEN,
+};
+// ----------------------------------------------------------
+// Helper: summarize UIElement[] into compact form
+// ----------------------------------------------------------
+export function summarizeUIElements(elements) {
+    const results = [];
+    flattenAndSummarize(elements, results);
+    return results;
+}
+function flattenAndSummarize(elements, out) {
+    for (const el of elements) {
+        const actions = [];
+        if (el.clickable)
+            actions.push('tap');
+        if (el.scrollable)
+            actions.push('scroll');
+        if (el.focusable)
+            actions.push('focus');
+        // Only include elements that have at least some useful info
+        const hasText = el.text.length > 0;
+        const hasDesc = el.contentDescription.length > 0;
+        const hasId = el.resourceId.length > 0;
+        const hasActions = actions.length > 0;
+        if (hasText || hasDesc || hasId || hasActions) {
+            out.push({
+                idx: el.index,
+                text: el.text,
+                desc: el.contentDescription,
+                type: simplifyClassName(el.className),
+                id: stripPackageFromId(el.resourceId),
+                cx: el.bounds.centerX,
+                cy: el.bounds.centerY,
+                actions,
+            });
+        }
+        if (el.children) {
+            flattenAndSummarize(el.children, out);
+        }
+    }
+}
+function simplifyClassName(className) {
+    // "android.widget.Button" → "Button"
+    // "com.example.custom.MyView" → "MyView"
+    const dot = className.lastIndexOf('.');
+    return dot >= 0 ? className.slice(dot + 1) : className;
+}
+function stripPackageFromId(resourceId) {
+    // "com.example.app:id/login_btn" → "login_btn"
+    const slash = resourceId.lastIndexOf('/');
+    return slash >= 0 ? resourceId.slice(slash + 1) : resourceId;
+}
+// ----------------------------------------------------------
+// Format UI tree summary as compact string for prompt injection
+// ----------------------------------------------------------
+function formatUITreeContext(uiElements) {
+    if (!uiElements || uiElements.length === 0)
+        return '';
+    return `\n\nUI element tree:\n${JSON.stringify(uiElements)}`;
+}
+// ----------------------------------------------------------
+// User Prompt Builders
+// ----------------------------------------------------------
+export function buildAnalyzeScreenPrompt(uiElements) {
+    return `Analyze this mobile app screenshot.${formatUITreeContext(uiElements)}`;
+}
+export function buildFindElementPrompt(query, uiElements) {
+    return `Find the element matching: '${query}'${formatUITreeContext(uiElements)}`;
+}
+export function buildSuggestActionsPrompt(goal, currentApp, uiElements) {
+    const appCtx = currentApp ? ` Current app: ${currentApp}.` : '';
+    return `I want to: ${goal}.${appCtx}${formatUITreeContext(uiElements)}`;
+}
+export function buildVisualDiffPrompt() {
+    return 'Compare these two mobile app screenshots and describe what changed.';
+}
+export function buildExtractTextPrompt() {
+    return 'Extract all visible text from this mobile app screenshot.';
+}
+export function buildVerifyScreenPrompt(assertion, uiElements) {
+    return `Verify: ${assertion}${formatUITreeContext(uiElements)}`;
+}
+//# sourceMappingURL=prompts.js.map

package/dist/ai/prompts.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"prompts.js","sourceRoot":"","sources":["../../src/ai/prompts.ts"],"names":[],"mappings":"AAAA,+DAA+D;AAC/D,6DAA6D;AAC7D,+DAA+D;AAmB/D,6DAA6D;AAC7D,iBAAiB;AACjB,6DAA6D;AAE7D,MAAM,cAAc,GAAG;;;;;;;;;;;;iGAY0E,CAAC;AAElG,MAAM,YAAY,GAAG;;;;;;;;qGAQgF,CAAC;AAEtG,MAAM,eAAe,GAAG;;;;;;;;;;;;4EAYoD,CAAC;AAE7E,MAAM,WAAW,GAAG;;;;;;;2FAOuE,CAAC;AAE5F,MAAM,YAAY,GAAG;;;;;oMAK+K,CAAC;AAErM,MAAM,aAAa,GAAG;;;;;;;;2EAQqD,CAAC;AAE5E,MAAM,CAAC,MAAM,OAAO,GAAG;IACrB,cAAc;IACd,YAAY;IACZ,eAAe;IACf,WAAW;IACX,YAAY;IACZ,aAAa;CACL,CAAC;AAEX,6DAA6D;AAC7D,kDAAkD;AAClD,6DAA6D;AAE7D,MAAM,UAAU,mBAAmB,CAAC,QAAqB;IACvD,MAAM,OAAO,GAAuB,EAAE,CAAC;IACvC,mBAAmB,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IACvC,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,mBAAmB,CAAC,QAAqB,EAAE,GAAuB;IACzE,KAAK,MAAM,EAAE,IAAI,QAAQ,EAAE,CAAC;QAC1B,MAAM,OAAO,GAAa,EAAE,CAAC;QAC7B,IAAI,EAAE,CAAC,SAAS;YAAE,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACtC,IAAI,EAAE,CAAC,UAAU;YAAE,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC1C,IAAI,EAAE,CAAC,SAAS;YAAE,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAExC,4DAA4D;QAC5D,MAAM,OAAO,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;QACnC,MAAM,OAAO,GAAG,EAAE,CAAC,kBAAkB,CAAC,MAAM,GAAG,CAAC,CAAC;QACjD,MAAM,KAAK,GAAG,EAAE,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC;QACvC,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;QAEtC,IAAI,OAAO,IAAI,OAAO,IAAI,KAAK,IAAI,UAAU,EAAE,CAAC;YAC9C,GAAG,CAAC,IAAI,CAAC;gBACP,GAAG,EAAE,EAAE,CAAC,KAAK;gBACb,IAAI,EAAE,EAAE,CAAC,IAAI;gBACb,IAAI,EAAE,EAAE,CAAC,kBAAkB;gBAC3B,IAAI,EAAE,iBAAiB,CAAC,EAAE,CAAC,SAAS,CAAC;gBACrC,EAAE,EAAE,kBAAkB,CAAC,EAAE,CAAC,UAAU,CAAC;gBACrC,EAAE,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO;gBACrB,EAAE,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO;gBACrB,OAAO;aACR,CAAC,CAAC;QACL,CAAC;QAED,IAAI,EAAE,CAAC,QAAQ,EAAE,CAAC;YAChB,mBAAmB,CAAC,EAAE,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;QACxC,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,iBAAiB,CAAC,SAAiB;IAC1C,qCAAqC;IACrC,yCAAyC;IACzC,MAAM,GAAG,GAAG,SAAS,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IACvC,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AACzD,CAAC;AAED,SAAS,kBAAkB,CAAC,UAAkB;IAC5C,+CAA+C;IAC/C,MAAM,KAAK,GAAG,UAAU,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IAC1C,OAAO,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC;AAC/D,CAAC;AAED,6DAA6D;AAC7D,gEAAgE;AAChE,6DAA6D;AAE7D,SAAS,mBAAmB,CAAC,UAA+B;IAC1D,IAAI,CAAC,UAAU,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IACtD,OAAO,yBAAyB,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,EAAE,CAAC;AAC/D,CAAC;AAED,6DAA6D;AAC7D,uBAAuB;AACvB,6DAA6D;AAE7D,MAAM,UAAU,wBAAwB,CAAC,UAA+B;IACtE,OAAO,sCAAsC,mBAAmB,CAAC,UAAU,CAAC,EAAE,CAAC;AACjF,CAAC;AAED,MAAM,UAAU,sBAAsB,CAAC,KAAa,EAAE,UAA+B;IACnF,OAAO,+BAA+B,KAAK,IAAI,mBAAmB,CAAC,UAAU,CAAC,EAAE,CAAC;AACnF,CAAC;AAED,MAAM,UAAU,yBAAyB,CACvC,IAAY,EACZ,UAAmB,EACnB,UAA+B;IAE/B,MAAM,MAAM,GAAG,UAAU,CAAC,CAAC,CAAC,iBAAiB,UAAU,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;IAChE,OAAO,cAAc,IAAI,IAAI,MAAM,GAAG,mBAAmB,CAAC,UAAU,CAAC,EAAE,CAAC;AAC1E,CAAC;AAED,MAAM,UAAU,qBAAqB;IACnC,OAAO,qEAAqE,CAAC;AAC/E,CAAC;AAED,MAAM,UAAU,sBAAsB;IACpC,OAAO,2DAA2D,CAAC;AACrE,CAAC;AAED,MAAM,UAAU,uBAAuB,CACrC,SAAiB,EACjB,UAA+B;IAE/B,OAAO,WAAW,SAAS,GAAG,mBAAmB,CAAC,UAAU,CAAC,EAAE,CAAC;AAClE,CAAC"}

package/dist/drivers/android/adb.d.ts

@@ -0,0 +1,21 @@
+import type { ADBResult } from "../../types.js";
+export declare class ADB {
+    private readonly adbPath;
+    private readonly timeoutMs;
+    constructor(adbPath?: string, timeoutMs?: number);
+    /**
+     * Execute an ADB command and return structured result.
+     * If `deviceId` is provided, `-s deviceId` is prepended to args.
+     */
+    execute(args: string[], deviceId?: string): Promise<ADBResult>;
+    /**
+     * Execute an ADB command and return raw stdout as a Buffer.
+     * Useful for binary data such as screenshots (screencap -p).
+     * Throws on non-zero exit code.
+     */
+    executeBuffer(args: string[], deviceId?: string): Promise<Buffer>;
+    private buildArgs;
+    private handleExecError;
+}
+export default ADB;
+//# sourceMappingURL=adb.d.ts.map

package/dist/drivers/android/adb.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"adb.d.ts","sourceRoot":"","sources":["../../../src/drivers/android/adb.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAUhD,qBAAa,GAAG;IACd,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;gBAEvB,OAAO,GAAE,MAAc,EAAE,SAAS,GAAE,MAA2B;IAK3E;;;OAGG;IACG,OAAO,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC;IAoBpE;;;;OAIG;IACG,aAAa,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAqDvE,OAAO,CAAC,SAAS;IAOjB,OAAO,CAAC,eAAe;CAsCxB;AAED,eAAe,GAAG,CAAC"}