@predicatelabs/sdk 0.99.9

package/dist/verification.js

@@ -0,0 +1,530 @@
+"use strict";
+/**
+ * Verification primitives for agent assertion loops.
+ *
+ * This module provides assertion predicates and outcome types for runtime verification
+ * in agent loops. Assertions evaluate against the current browser state (snapshot/url)
+ * and record results into the trace.
+ *
+ * Key concepts:
+ * - AssertOutcome: Result of evaluating an assertion
+ * - AssertContext: Context provided to assertion predicates (snapshot, url, stepId)
+ * - Predicate: Callable that takes context and returns outcome
+ *
+ * @example
+ * ```typescript
+ * import { urlMatches, exists, AssertContext } from './verification';
+ *
+ * // Create predicates
+ * const onSearchPage = urlMatches(/\/s\?k=/);
+ * const resultsLoaded = exists("text~'Results'");
+ *
+ * // Evaluate against context
+ * const ctx: AssertContext = { snapshot, url: "https://example.com/s?k=shoes" };
+ * const outcome = onSearchPage(ctx);
+ * console.log(outcome.passed); // true
+ * ```
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.downloadCompleted = downloadCompleted;
+exports.urlMatches = urlMatches;
+exports.urlContains = urlContains;
+exports.exists = exists;
+exports.notExists = notExists;
+exports.elementCount = elementCount;
+exports.allOf = allOf;
+exports.anyOf = anyOf;
+exports.custom = custom;
+exports.isEnabled = isEnabled;
+exports.isDisabled = isDisabled;
+exports.isChecked = isChecked;
+exports.isUnchecked = isUnchecked;
+exports.valueEquals = valueEquals;
+exports.valueContains = valueContains;
+exports.isExpanded = isExpanded;
+exports.isCollapsed = isCollapsed;
+const query_1 = require("./query");
+/**
+ * Convert QuerySelector to string for display/logging.
+ */
+function selectorToString(selector) {
+    return typeof selector === 'string' ? selector : JSON.stringify(selector);
+}
+/**
+ * Predicate that passes if a browser download has completed.
+ *
+ * Notes:
+ * - This relies on `AssertContext.downloads` being populated by the runtime/browser.
+ */
+function downloadCompleted(filenameSubstring) {
+    return (ctx) => {
+        const downloads = ctx.downloads ?? [];
+        for (const d of downloads) {
+            if (String(d?.status ?? '') !== 'completed')
+                continue;
+            const fname = String(d?.filename ?? d?.suggested_filename ?? '');
+            if (!filenameSubstring || fname.includes(filenameSubstring)) {
+                return { passed: true, reason: '', details: { download: d } };
+            }
+        }
+        return {
+            passed: false,
+            reason: filenameSubstring
+                ? `no completed download matched: ${filenameSubstring}`
+                : 'no completed downloads',
+            details: { filenameSubstring, downloads },
+        };
+    };
+}
+/**
+ * Create a predicate that checks if current URL matches a regex pattern.
+ *
+ * @param pattern - Regular expression pattern or string to match against URL
+ * @returns Predicate function that evaluates URL matching
+ *
+ * @example
+ * ```typescript
+ * const pred = urlMatches(/\/search\?q=/);
+ * const ctx = { snapshot: null, url: "https://example.com/search?q=shoes", stepId: null };
+ * const outcome = pred(ctx);
+ * console.log(outcome.passed); // true
+ * ```
+ */
+function urlMatches(pattern) {
+    const rx = typeof pattern === 'string' ? new RegExp(pattern) : pattern;
+    return (ctx) => {
+        const url = ctx.url || '';
+        const ok = rx.test(url);
+        return {
+            passed: ok,
+            reason: ok ? '' : `url did not match pattern: ${pattern}`,
+            details: { pattern: String(pattern), url: url.substring(0, 200) },
+        };
+    };
+}
+/**
+ * Create a predicate that checks if current URL contains a substring.
+ *
+ * @param substring - String to search for in URL
+ * @returns Predicate function that evaluates URL containment
+ *
+ * @example
+ * ```typescript
+ * const pred = urlContains("/cart");
+ * const ctx = { snapshot: null, url: "https://example.com/cart/checkout", stepId: null };
+ * const outcome = pred(ctx);
+ * console.log(outcome.passed); // true
+ * ```
+ */
+function urlContains(substring) {
+    return (ctx) => {
+        const url = ctx.url || '';
+        const ok = url.includes(substring);
+        return {
+            passed: ok,
+            reason: ok ? '' : `url does not contain: ${substring}`,
+            details: { substring, url: url.substring(0, 200) },
+        };
+    };
+}
+/**
+ * Create a predicate that checks if elements matching selector exist.
+ *
+ * Uses the SDK's query engine to find matching elements.
+ *
+ * @param selector - Semantic selector string (e.g., "role=button text~'Sign in'")
+ * @returns Predicate function that evaluates element existence
+ *
+ * @example
+ * ```typescript
+ * const pred = exists("text~'Results'");
+ * // Will check if snapshot contains elements with "Results" in text
+ * ```
+ */
+function exists(selector) {
+    const selectorStr = selectorToString(selector);
+    return (ctx) => {
+        const snap = ctx.snapshot;
+        if (!snap) {
+            return {
+                passed: false,
+                reason: 'no snapshot available',
+                details: { selector: selectorStr, reason_code: 'no_snapshot' },
+            };
+        }
+        const matches = (0, query_1.query)(snap, selector);
+        const ok = matches.length > 0;
+        return {
+            passed: ok,
+            reason: ok ? '' : `no elements matched selector: ${selectorStr}`,
+            details: {
+                selector: selectorStr,
+                matched: matches.length,
+                reason_code: ok ? 'ok' : 'no_match',
+            },
+        };
+    };
+}
+/**
+ * Create a predicate that checks that NO elements match the selector.
+ *
+ * Useful for asserting that error messages, loading spinners, etc. are gone.
+ *
+ * @param selector - Semantic selector string
+ * @returns Predicate function that evaluates element non-existence
+ *
+ * @example
+ * ```typescript
+ * const pred = notExists("text~'Loading'");
+ * // Will pass if no elements contain "Loading" text
+ * ```
+ */
+function notExists(selector) {
+    const selectorStr = selectorToString(selector);
+    return (ctx) => {
+        const snap = ctx.snapshot;
+        if (!snap) {
+            return {
+                passed: false,
+                reason: 'no snapshot available',
+                details: { selector: selectorStr, reason_code: 'no_snapshot' },
+            };
+        }
+        const matches = (0, query_1.query)(snap, selector);
+        const ok = matches.length === 0;
+        return {
+            passed: ok,
+            reason: ok ? '' : `found ${matches.length} elements matching: ${selectorStr}`,
+            details: {
+                selector: selectorStr,
+                matched: matches.length,
+                reason_code: ok ? 'ok' : 'unexpected_match',
+            },
+        };
+    };
+}
+/**
+ * Create a predicate that checks the number of matching elements.
+ *
+ * @param selector - Semantic selector string
+ * @param options - Count constraints
+ * @returns Predicate function that evaluates element count
+ *
+ * @example
+ * ```typescript
+ * const pred = elementCount("role=button", { minCount: 1, maxCount: 5 });
+ * // Will pass if 1-5 buttons found
+ * ```
+ */
+function elementCount(selector, options = {}) {
+    const { minCount = 0, maxCount } = options;
+    const selectorStr = selectorToString(selector);
+    return (ctx) => {
+        const snap = ctx.snapshot;
+        if (!snap) {
+            return {
+                passed: false,
+                reason: 'no snapshot available',
+                details: { selector: selectorStr, minCount, maxCount },
+            };
+        }
+        const matches = (0, query_1.query)(snap, selector);
+        const count = matches.length;
+        let ok = count >= minCount;
+        if (maxCount !== undefined) {
+            ok = ok && count <= maxCount;
+        }
+        let reason = '';
+        if (!ok) {
+            if (maxCount !== undefined) {
+                reason = `expected ${minCount}-${maxCount} elements, found ${count}`;
+            }
+            else {
+                reason = `expected at least ${minCount} elements, found ${count}`;
+            }
+        }
+        return {
+            passed: ok,
+            reason,
+            details: {
+                selector: selectorStr,
+                matched: count,
+                minCount,
+                maxCount,
+            },
+        };
+    };
+}
+/**
+ * Create a predicate that passes only if ALL sub-predicates pass.
+ *
+ * @param predicates - Predicate functions to combine with AND logic
+ * @returns Combined predicate
+ *
+ * @example
+ * ```typescript
+ * const pred = allOf(urlContains("/cart"), exists("text~'Checkout'"));
+ * // Will pass only if both conditions are true
+ * ```
+ */
+function allOf(...predicates) {
+    return (ctx) => {
+        const failedReasons = [];
+        const allDetails = [];
+        for (const p of predicates) {
+            const outcome = p(ctx);
+            allDetails.push(outcome.details);
+            if (!outcome.passed) {
+                failedReasons.push(outcome.reason);
+            }
+        }
+        const ok = failedReasons.length === 0;
+        return {
+            passed: ok,
+            reason: failedReasons.join('; '),
+            details: { subPredicates: allDetails, failedCount: failedReasons.length },
+        };
+    };
+}
+/**
+ * Create a predicate that passes if ANY sub-predicate passes.
+ *
+ * @param predicates - Predicate functions to combine with OR logic
+ * @returns Combined predicate
+ *
+ * @example
+ * ```typescript
+ * const pred = anyOf(exists("text~'Success'"), exists("text~'Complete'"));
+ * // Will pass if either condition is true
+ * ```
+ */
+function anyOf(...predicates) {
+    return (ctx) => {
+        const allReasons = [];
+        const allDetails = [];
+        for (let i = 0; i < predicates.length; i++) {
+            const outcome = predicates[i](ctx);
+            allDetails.push(outcome.details);
+            if (outcome.passed) {
+                return {
+                    passed: true,
+                    reason: '',
+                    details: { subPredicates: allDetails, matchedAtIndex: i },
+                };
+            }
+            allReasons.push(outcome.reason);
+        }
+        return {
+            passed: false,
+            reason: `none of ${predicates.length} predicates passed: ${allReasons.join('; ')}`,
+            details: { subPredicates: allDetails },
+        };
+    };
+}
+/**
+ * Create a predicate from a custom function.
+ *
+ * @param checkFn - Function that takes AssertContext and returns boolean
+ * @param label - Label for debugging/display
+ * @returns Predicate wrapping the custom function
+ *
+ * @example
+ * ```typescript
+ * const pred = custom(
+ *   (ctx) => ctx.snapshot !== null && ctx.snapshot.elements.length > 10,
+ *   "has_many_elements"
+ * );
+ * ```
+ */
+function custom(checkFn, label = 'custom') {
+    return (ctx) => {
+        try {
+            const ok = checkFn(ctx);
+            return {
+                passed: ok,
+                reason: ok ? '' : `custom check '${label}' returned false`,
+                details: { label },
+            };
+        }
+        catch (e) {
+            return {
+                passed: false,
+                reason: `custom check '${label}' raised exception: ${e}`,
+                details: { label, error: String(e) },
+            };
+        }
+    };
+}
+// ============================================================================
+// v1 state-aware predicates (deterministic, schema-driven)
+// ============================================================================
+function isEnabled(selector) {
+    const selectorStr = selectorToString(selector);
+    return (ctx) => {
+        const snap = ctx.snapshot;
+        if (!snap) {
+            return { passed: false, reason: 'no snapshot available', details: { selector: selectorStr } };
+        }
+        const matches = (0, query_1.query)(snap, selector);
+        if (matches.length === 0) {
+            return {
+                passed: false,
+                reason: `no elements matched selector: ${selectorStr}`,
+                details: { selector: selectorStr, matched: 0, reason_code: 'no_match' },
+            };
+        }
+        const ok = matches.some(m => m.disabled !== true);
+        return {
+            passed: ok,
+            reason: ok ? '' : `all matched elements are disabled: ${selectorStr}`,
+            details: {
+                selector: selectorStr,
+                matched: matches.length,
+                reason_code: ok ? 'ok' : 'state_mismatch',
+            },
+        };
+    };
+}
+function isDisabled(selector) {
+    const selectorStr = selectorToString(selector);
+    return (ctx) => {
+        const snap = ctx.snapshot;
+        if (!snap) {
+            return { passed: false, reason: 'no snapshot available', details: { selector: selectorStr } };
+        }
+        const matches = (0, query_1.query)(snap, selector);
+        const ok = matches.some(m => m.disabled === true);
+        return {
+            passed: ok,
+            reason: ok ? '' : `no matched elements are disabled: ${selectorStr}`,
+            details: {
+                selector: selectorStr,
+                matched: matches.length,
+                reason_code: ok ? 'ok' : 'state_mismatch',
+            },
+        };
+    };
+}
+function isChecked(selector) {
+    const selectorStr = selectorToString(selector);
+    return (ctx) => {
+        const snap = ctx.snapshot;
+        if (!snap) {
+            return { passed: false, reason: 'no snapshot available', details: { selector: selectorStr } };
+        }
+        const matches = (0, query_1.query)(snap, selector);
+        const ok = matches.some(m => m.checked === true);
+        return {
+            passed: ok,
+            reason: ok ? '' : `no matched elements are checked: ${selectorStr}`,
+            details: {
+                selector: selectorStr,
+                matched: matches.length,
+                reason_code: ok ? 'ok' : 'state_mismatch',
+            },
+        };
+    };
+}
+function isUnchecked(selector) {
+    const selectorStr = selectorToString(selector);
+    return (ctx) => {
+        const snap = ctx.snapshot;
+        if (!snap) {
+            return { passed: false, reason: 'no snapshot available', details: { selector: selectorStr } };
+        }
+        const matches = (0, query_1.query)(snap, selector);
+        const ok = matches.some(m => m.checked !== true);
+        return {
+            passed: ok,
+            reason: ok ? '' : `all matched elements are checked: ${selectorStr}`,
+            details: {
+                selector: selectorStr,
+                matched: matches.length,
+                reason_code: ok ? 'ok' : 'state_mismatch',
+            },
+        };
+    };
+}
+function valueEquals(selector, expected) {
+    const selectorStr = selectorToString(selector);
+    return (ctx) => {
+        const snap = ctx.snapshot;
+        if (!snap) {
+            return { passed: false, reason: 'no snapshot available', details: { selector: selectorStr } };
+        }
+        const matches = (0, query_1.query)(snap, selector);
+        const ok = matches.some(m => (m.value ?? '') === expected);
+        return {
+            passed: ok,
+            reason: ok ? '' : `no matched elements had value == '${expected}'`,
+            details: {
+                selector: selectorStr,
+                expected,
+                matched: matches.length,
+                reason_code: ok ? 'ok' : 'state_mismatch',
+            },
+        };
+    };
+}
+function valueContains(selector, substring) {
+    const selectorStr = selectorToString(selector);
+    return (ctx) => {
+        const snap = ctx.snapshot;
+        if (!snap) {
+            return { passed: false, reason: 'no snapshot available', details: { selector: selectorStr } };
+        }
+        const matches = (0, query_1.query)(snap, selector);
+        const ok = matches.some(m => (m.value ?? '').toLowerCase().includes(substring.toLowerCase()));
+        return {
+            passed: ok,
+            reason: ok ? '' : `no matched elements had value containing '${substring}'`,
+            details: {
+                selector: selectorStr,
+                substring,
+                matched: matches.length,
+                reason_code: ok ? 'ok' : 'state_mismatch',
+            },
+        };
+    };
+}
+function isExpanded(selector) {
+    const selectorStr = selectorToString(selector);
+    return (ctx) => {
+        const snap = ctx.snapshot;
+        if (!snap) {
+            return { passed: false, reason: 'no snapshot available', details: { selector: selectorStr } };
+        }
+        const matches = (0, query_1.query)(snap, selector);
+        const ok = matches.some(m => m.expanded === true);
+        return {
+            passed: ok,
+            reason: ok ? '' : `no matched elements are expanded: ${selectorStr}`,
+            details: {
+                selector: selectorStr,
+                matched: matches.length,
+                reason_code: ok ? 'ok' : 'state_mismatch',
+            },
+        };
+    };
+}
+function isCollapsed(selector) {
+    const selectorStr = selectorToString(selector);
+    return (ctx) => {
+        const snap = ctx.snapshot;
+        if (!snap) {
+            return { passed: false, reason: 'no snapshot available', details: { selector: selectorStr } };
+        }
+        const matches = (0, query_1.query)(snap, selector);
+        const ok = matches.some(m => m.expanded !== true);
+        return {
+            passed: ok,
+            reason: ok ? '' : `all matched elements are expanded: ${selectorStr}`,
+            details: {
+                selector: selectorStr,
+                matched: matches.length,
+                reason_code: ok ? 'ok' : 'state_mismatch',
+            },
+        };
+    };
+}
+//# sourceMappingURL=verification.js.map

package/dist/verification.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"verification.js","sourceRoot":"","sources":["../src/verification.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;;AAqDH,8CAkBC;AAgBD,gCAYC;AAgBD,kCAUC;AAgBD,wBAwBC;AAgBD,8BAwBC;AAeD,oCA6CC;AAcD,sBAoBC;AAcD,sBAwBC;AAiBD,wBAoBC;AAMD,8BA4BC;AAED,gCAmBC;AAED,8BAmBC;AAED,kCAmBC;AAED,kCAoBC;AAED,sCAoBC;AAED,gCAmBC;AAED,kCAmBC;AA1iBD,mCAAgC;AAEhC;;GAEG;AACH,SAAS,gBAAgB,CAAC,QAAuB;IAC/C,OAAO,OAAO,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;AAC5E,CAAC;AAqCD;;;;;GAKG;AACH,SAAgB,iBAAiB,CAAC,iBAA0B;IAC1D,OAAO,CAAC,GAAkB,EAAiB,EAAE;QAC3C,MAAM,SAAS,GAAG,GAAG,CAAC,SAAS,IAAI,EAAE,CAAC;QACtC,KAAK,MAAM,CAAC,IAAI,SAAS,EAAE,CAAC;YAC1B,IAAI,MAAM,CAAC,CAAC,EAAE,MAAM,IAAI,EAAE,CAAC,KAAK,WAAW;gBAAE,SAAS;YACtD,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,EAAE,QAAQ,IAAI,CAAC,EAAE,kBAAkB,IAAI,EAAE,CAAC,CAAC;YACjE,IAAI,CAAC,iBAAiB,IAAI,KAAK,CAAC,QAAQ,CAAC,iBAAiB,CAAC,EAAE,CAAC;gBAC5D,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,QAAQ,EAAE,CAAC,EAAE,EAAE,CAAC;YAChE,CAAC;QACH,CAAC;QACD,OAAO;YACL,MAAM,EAAE,KAAK;YACb,MAAM,EAAE,iBAAiB;gBACvB,CAAC,CAAC,kCAAkC,iBAAiB,EAAE;gBACvD,CAAC,CAAC,wBAAwB;YAC5B,OAAO,EAAE,EAAE,iBAAiB,EAAE,SAAS,EAAE;SAC1C,CAAC;IACJ,CAAC,CAAC;AACJ,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,SAAgB,UAAU,CAAC,OAAwB;IACjD,MAAM,EAAE,GAAG,OAAO,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;IAEvE,OAAO,CAAC,GAAkB,EAAiB,EAAE;QAC3C,MAAM,GAAG,GAAG,GAAG,CAAC,GAAG,IAAI,EAAE,CAAC;QAC1B,MAAM,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACxB,OAAO;YACL,MAAM,EAAE,EAAE;YACV,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,8BAA8B,OAAO,EAAE;YACzD,OAAO,EAAE,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE;SAClE,CAAC;IACJ,CAAC,CAAC;AACJ,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,SAAgB,WAAW,CAAC,SAAiB;IAC3C,OAAO,CAAC,GAAkB,EAAiB,EAAE;QAC3C,MAAM,GAAG,GAAG,GAAG,CAAC,GAAG,IAAI,EAAE,CAAC;QAC1B,MAAM,EAAE,GAAG,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;QACnC,OAAO;YACL,MAAM,EAAE,EAAE;YACV,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,yBAAyB,SAAS,EAAE;YACtD,OAAO,EAAE,EAAE,SAAS,EAAE,GAAG,EAAE,GAAG,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE;SACnD,CAAC;IACJ,CAAC,CAAC;AACJ,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,SAAgB,MAAM,CAAC,QAAuB;IAC5C,MAAM,WAAW,GAAG,gBAAgB,CAAC,QAAQ,CAAC,CAAC;IAC/C,OAAO,CAAC,GAAkB,EAAiB,EAAE;QAC3C,MAAM,IAAI,GAAG,GAAG,CAAC,QAAQ,CAAC;QAC1B,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,OAAO;gBACL,MAAM,EAAE,KAAK;gBACb,MAAM,EAAE,uBAAuB;gBAC/B,OAAO,EAAE,EAAE,QAAQ,EAAE,WAAW,EAAE,WAAW,EAAE,aAAa,EAAE;aAC/D,CAAC;QACJ,CAAC;QAED,MAAM,OAAO,GAAG,IAAA,aAAK,EAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QACtC,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;QAC9B,OAAO;YACL,MAAM,EAAE,EAAE;YACV,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,iCAAiC,WAAW,EAAE;YAChE,OAAO,EAAE;gBACP,QAAQ,EAAE,WAAW;gBACrB,OAAO,EAAE,OAAO,CAAC,MAAM;gBACvB,WAAW,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU;aACpC;SACF,CAAC;IACJ,CAAC,CAAC;AACJ,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,SAAgB,SAAS,CAAC,QAAuB;IAC/C,MAAM,WAAW,GAAG,gBAAgB,CAAC,QAAQ,CAAC,CAAC;IAC/C,OAAO,CAAC,GAAkB,EAAiB,EAAE;QAC3C,MAAM,IAAI,GAAG,GAAG,CAAC,QAAQ,CAAC;QAC1B,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,OAAO;gBACL,MAAM,EAAE,KAAK;gBACb,MAAM,EAAE,uBAAuB;gBAC/B,OAAO,EAAE,EAAE,QAAQ,EAAE,WAAW,EAAE,WAAW,EAAE,aAAa,EAAE;aAC/D,CAAC;QACJ,CAAC;QAED,MAAM,OAAO,GAAG,IAAA,aAAK,EAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QACtC,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC;QAChC,OAAO;YACL,MAAM,EAAE,EAAE;YACV,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,OAAO,CAAC,MAAM,uBAAuB,WAAW,EAAE;YAC7E,OAAO,EAAE;gBACP,QAAQ,EAAE,WAAW;gBACrB,OAAO,EAAE,OAAO,CAAC,MAAM;gBACvB,WAAW,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,kBAAkB;aAC5C;SACF,CAAC;IACJ,CAAC,CAAC;AACJ,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,SAAgB,YAAY,CAC1B,QAAuB,EACvB,UAAoD,EAAE;IAEtD,MAAM,EAAE,QAAQ,GAAG,CAAC,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC;IAC3C,MAAM,WAAW,GAAG,gBAAgB,CAAC,QAAQ,CAAC,CAAC;IAE/C,OAAO,CAAC,GAAkB,EAAiB,EAAE;QAC3C,MAAM,IAAI,GAAG,GAAG,CAAC,QAAQ,CAAC;QAC1B,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,OAAO;gBACL,MAAM,EAAE,KAAK;gBACb,MAAM,EAAE,uBAAuB;gBAC/B,OAAO,EAAE,EAAE,QAAQ,EAAE,WAAW,EAAE,QAAQ,EAAE,QAAQ,EAAE;aACvD,CAAC;QACJ,CAAC;QAED,MAAM,OAAO,GAAG,IAAA,aAAK,EAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QACtC,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC;QAE7B,IAAI,EAAE,GAAG,KAAK,IAAI,QAAQ,CAAC;QAC3B,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,EAAE,GAAG,EAAE,IAAI,KAAK,IAAI,QAAQ,CAAC;QAC/B,CAAC;QAED,IAAI,MAAM,GAAG,EAAE,CAAC;QAChB,IAAI,CAAC,EAAE,EAAE,CAAC;YACR,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;gBAC3B,MAAM,GAAG,YAAY,QAAQ,IAAI,QAAQ,oBAAoB,KAAK,EAAE,CAAC;YACvE,CAAC;iBAAM,CAAC;gBACN,MAAM,GAAG,qBAAqB,QAAQ,oBAAoB,KAAK,EAAE,CAAC;YACpE,CAAC;QACH,CAAC;QAED,OAAO;YACL,MAAM,EAAE,EAAE;YACV,MAAM;YACN,OAAO,EAAE;gBACP,QAAQ,EAAE,WAAW;gBACrB,OAAO,EAAE,KAAK;gBACd,QAAQ;gBACR,QAAQ;aACT;SACF,CAAC;IACJ,CAAC,CAAC;AACJ,CAAC;AAED;;;;;;;;;;;GAWG;AACH,SAAgB,KAAK,CAAC,GAAG,UAAuB;IAC9C,OAAO,CAAC,GAAkB,EAAiB,EAAE;QAC3C,MAAM,aAAa,GAAa,EAAE,CAAC;QACnC,MAAM,UAAU,GAA0B,EAAE,CAAC;QAE7C,KAAK,MAAM,CAAC,IAAI,UAAU,EAAE,CAAC;YAC3B,MAAM,OAAO,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;YACvB,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YACjC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;gBACpB,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;YACrC,CAAC;QACH,CAAC;QAED,MAAM,EAAE,GAAG,aAAa,CAAC,MAAM,KAAK,CAAC,CAAC;QACtC,OAAO;YACL,MAAM,EAAE,EAAE;YACV,MAAM,EAAE,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC;YAChC,OAAO,EAAE,EAAE,aAAa,EAAE,UAAU,EAAE,WAAW,EAAE,aAAa,CAAC,MAAM,EAAE;SAC1E,CAAC;IACJ,CAAC,CAAC;AACJ,CAAC;AAED;;;;;;;;;;;GAWG;AACH,SAAgB,KAAK,CAAC,GAAG,UAAuB;IAC9C,OAAO,CAAC,GAAkB,EAAiB,EAAE;QAC3C,MAAM,UAAU,GAAa,EAAE,CAAC;QAChC,MAAM,UAAU,GAA0B,EAAE,CAAC;QAE7C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3C,MAAM,OAAO,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;YACnC,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YACjC,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;gBACnB,OAAO;oBACL,MAAM,EAAE,IAAI;oBACZ,MAAM,EAAE,EAAE;oBACV,OAAO,EAAE,EAAE,aAAa,EAAE,UAAU,EAAE,cAAc,EAAE,CAAC,EAAE;iBAC1D,CAAC;YACJ,CAAC;YACD,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAClC,CAAC;QAED,OAAO;YACL,MAAM,EAAE,KAAK;YACb,MAAM,EAAE,WAAW,UAAU,CAAC,MAAM,uBAAuB,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;YAClF,OAAO,EAAE,EAAE,aAAa,EAAE,UAAU,EAAE;SACvC,CAAC;IACJ,CAAC,CAAC;AACJ,CAAC;AAED;;;;;;;;;;;;;;GAcG;AACH,SAAgB,MAAM,CACpB,OAAwC,EACxC,QAAgB,QAAQ;IAExB,OAAO,CAAC,GAAkB,EAAiB,EAAE;QAC3C,IAAI,CAAC;YACH,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;YACxB,OAAO;gBACL,MAAM,EAAE,EAAE;gBACV,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,iBAAiB,KAAK,kBAAkB;gBAC1D,OAAO,EAAE,EAAE,KAAK,EAAE;aACnB,CAAC;QACJ,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO;gBACL,MAAM,EAAE,KAAK;gBACb,MAAM,EAAE,iBAAiB,KAAK,uBAAuB,CAAC,EAAE;gBACxD,OAAO,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE;aACrC,CAAC;QACJ,CAAC;IACH,CAAC,CAAC;AACJ,CAAC;AAED,+EAA+E;AAC/E,2DAA2D;AAC3D,+EAA+E;AAE/E,SAAgB,SAAS,CAAC,QAAuB;IAC/C,MAAM,WAAW,GAAG,gBAAgB,CAAC,QAAQ,CAAC,CAAC;IAC/C,OAAO,CAAC,GAAkB,EAAiB,EAAE;QAC3C,MAAM,IAAI,GAAG,GAAG,CAAC,QAAQ,CAAC;QAC1B,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,uBAAuB,EAAE,OAAO,EAAE,EAAE,QAAQ,EAAE,WAAW,EAAE,EAAE,CAAC;QAChG,CAAC;QAED,MAAM,OAAO,GAAG,IAAA,aAAK,EAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QACtC,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,OAAO;gBACL,MAAM,EAAE,KAAK;gBACb,MAAM,EAAE,iCAAiC,WAAW,EAAE;gBACtD,OAAO,EAAE,EAAE,QAAQ,EAAE,WAAW,EAAE,OAAO,EAAE,CAAC,EAAE,WAAW,EAAE,UAAU,EAAE;aACxE,CAAC;QACJ,CAAC;QAED,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,IAAI,CAAC,CAAC;QAClD,OAAO;YACL,MAAM,EAAE,EAAE;YACV,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,sCAAsC,WAAW,EAAE;YACrE,OAAO,EAAE;gBACP,QAAQ,EAAE,WAAW;gBACrB,OAAO,EAAE,OAAO,CAAC,MAAM;gBACvB,WAAW,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,gBAAgB;aAC1C;SACF,CAAC;IACJ,CAAC,CAAC;AACJ,CAAC;AAED,SAAgB,UAAU,CAAC,QAAuB;IAChD,MAAM,WAAW,GAAG,gBAAgB,CAAC,QAAQ,CAAC,CAAC;IAC/C,OAAO,CAAC,GAAkB,EAAiB,EAAE;QAC3C,MAAM,IAAI,GAAG,GAAG,CAAC,QAAQ,CAAC;QAC1B,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,uBAAuB,EAAE,OAAO,EAAE,EAAE,QAAQ,EAAE,WAAW,EAAE,EAAE,CAAC;QAChG,CAAC;QACD,MAAM,OAAO,GAAG,IAAA,aAAK,EAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QACtC,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,IAAI,CAAC,CAAC;QAClD,OAAO;YACL,MAAM,EAAE,EAAE;YACV,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,qCAAqC,WAAW,EAAE;YACpE,OAAO,EAAE;gBACP,QAAQ,EAAE,WAAW;gBACrB,OAAO,EAAE,OAAO,CAAC,MAAM;gBACvB,WAAW,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,gBAAgB;aAC1C;SACF,CAAC;IACJ,CAAC,CAAC;AACJ,CAAC;AAED,SAAgB,SAAS,CAAC,QAAuB;IAC/C,MAAM,WAAW,GAAG,gBAAgB,CAAC,QAAQ,CAAC,CAAC;IAC/C,OAAO,CAAC,GAAkB,EAAiB,EAAE;QAC3C,MAAM,IAAI,GAAG,GAAG,CAAC,QAAQ,CAAC;QAC1B,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,uBAAuB,EAAE,OAAO,EAAE,EAAE,QAAQ,EAAE,WAAW,EAAE,EAAE,CAAC;QAChG,CAAC;QACD,MAAM,OAAO,GAAG,IAAA,aAAK,EAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QACtC,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,IAAI,CAAC,CAAC;QACjD,OAAO;YACL,MAAM,EAAE,EAAE;YACV,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,oCAAoC,WAAW,EAAE;YACnE,OAAO,EAAE;gBACP,QAAQ,EAAE,WAAW;gBACrB,OAAO,EAAE,OAAO,CAAC,MAAM;gBACvB,WAAW,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,gBAAgB;aAC1C;SACF,CAAC;IACJ,CAAC,CAAC;AACJ,CAAC;AAED,SAAgB,WAAW,CAAC,QAAuB;IACjD,MAAM,WAAW,GAAG,gBAAgB,CAAC,QAAQ,CAAC,CAAC;IAC/C,OAAO,CAAC,GAAkB,EAAiB,EAAE;QAC3C,MAAM,IAAI,GAAG,GAAG,CAAC,QAAQ,CAAC;QAC1B,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,uBAAuB,EAAE,OAAO,EAAE,EAAE,QAAQ,EAAE,WAAW,EAAE,EAAE,CAAC;QAChG,CAAC;QACD,MAAM,OAAO,GAAG,IAAA,aAAK,EAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QACtC,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,IAAI,CAAC,CAAC;QACjD,OAAO;YACL,MAAM,EAAE,EAAE;YACV,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,qCAAqC,WAAW,EAAE;YACpE,OAAO,EAAE;gBACP,QAAQ,EAAE,WAAW;gBACrB,OAAO,EAAE,OAAO,CAAC,MAAM;gBACvB,WAAW,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,gBAAgB;aAC1C;SACF,CAAC;IACJ,CAAC,CAAC;AACJ,CAAC;AAED,SAAgB,WAAW,CAAC,QAAuB,EAAE,QAAgB;IACnE,MAAM,WAAW,GAAG,gBAAgB,CAAC,QAAQ,CAAC,CAAC;IAC/C,OAAO,CAAC,GAAkB,EAAiB,EAAE;QAC3C,MAAM,IAAI,GAAG,GAAG,CAAC,QAAQ,CAAC;QAC1B,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,uBAAuB,EAAE,OAAO,EAAE,EAAE,QAAQ,EAAE,WAAW,EAAE,EAAE,CAAC;QAChG,CAAC;QACD,MAAM,OAAO,GAAG,IAAA,aAAK,EAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QACtC,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC,KAAK,QAAQ,CAAC,CAAC;QAC3D,OAAO;YACL,MAAM,EAAE,EAAE;YACV,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,qCAAqC,QAAQ,GAAG;YAClE,OAAO,EAAE;gBACP,QAAQ,EAAE,WAAW;gBACrB,QAAQ;gBACR,OAAO,EAAE,OAAO,CAAC,MAAM;gBACvB,WAAW,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,gBAAgB;aAC1C;SACF,CAAC;IACJ,CAAC,CAAC;AACJ,CAAC;AAED,SAAgB,aAAa,CAAC,QAAuB,EAAE,SAAiB;IACtE,MAAM,WAAW,GAAG,gBAAgB,CAAC,QAAQ,CAAC,CAAC;IAC/C,OAAO,CAAC,GAAkB,EAAiB,EAAE;QAC3C,MAAM,IAAI,GAAG,GAAG,CAAC,QAAQ,CAAC;QAC1B,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,uBAAuB,EAAE,OAAO,EAAE,EAAE,QAAQ,EAAE,WAAW,EAAE,EAAE,CAAC;QAChG,CAAC;QACD,MAAM,OAAO,GAAG,IAAA,aAAK,EAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QACtC,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;QAC9F,OAAO;YACL,MAAM,EAAE,EAAE;YACV,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,6CAA6C,SAAS,GAAG;YAC3E,OAAO,EAAE;gBACP,QAAQ,EAAE,WAAW;gBACrB,SAAS;gBACT,OAAO,EAAE,OAAO,CAAC,MAAM;gBACvB,WAAW,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,gBAAgB;aAC1C;SACF,CAAC;IACJ,CAAC,CAAC;AACJ,CAAC;AAED,SAAgB,UAAU,CAAC,QAAuB;IAChD,MAAM,WAAW,GAAG,gBAAgB,CAAC,QAAQ,CAAC,CAAC;IAC/C,OAAO,CAAC,GAAkB,EAAiB,EAAE;QAC3C,MAAM,IAAI,GAAG,GAAG,CAAC,QAAQ,CAAC;QAC1B,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,uBAAuB,EAAE,OAAO,EAAE,EAAE,QAAQ,EAAE,WAAW,EAAE,EAAE,CAAC;QAChG,CAAC;QACD,MAAM,OAAO,GAAG,IAAA,aAAK,EAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QACtC,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,IAAI,CAAC,CAAC;QAClD,OAAO;YACL,MAAM,EAAE,EAAE;YACV,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,qCAAqC,WAAW,EAAE;YACpE,OAAO,EAAE;gBACP,QAAQ,EAAE,WAAW;gBACrB,OAAO,EAAE,OAAO,CAAC,MAAM;gBACvB,WAAW,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,gBAAgB;aAC1C;SACF,CAAC;IACJ,CAAC,CAAC;AACJ,CAAC;AAED,SAAgB,WAAW,CAAC,QAAuB;IACjD,MAAM,WAAW,GAAG,gBAAgB,CAAC,QAAQ,CAAC,CAAC;IAC/C,OAAO,CAAC,GAAkB,EAAiB,EAAE;QAC3C,MAAM,IAAI,GAAG,GAAG,CAAC,QAAQ,CAAC;QAC1B,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,uBAAuB,EAAE,OAAO,EAAE,EAAE,QAAQ,EAAE,WAAW,EAAE,EAAE,CAAC;QAChG,CAAC;QACD,MAAM,OAAO,GAAG,IAAA,aAAK,EAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QACtC,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,IAAI,CAAC,CAAC;QAClD,OAAO;YACL,MAAM,EAAE,EAAE;YACV,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,sCAAsC,WAAW,EAAE;YACrE,OAAO,EAAE;gBACP,QAAQ,EAAE,WAAW;gBACrB,OAAO,EAAE,OAAO,CAAC,MAAM;gBACvB,WAAW,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,gBAAgB;aAC1C;SACF,CAAC;IACJ,CAAC,CAAC;AACJ,CAAC"}

package/dist/vision-executor.d.ts

@@ -0,0 +1,18 @@
+/**
+ * Vision executor primitives (shared parsing/execution helpers).
+ *
+ * This is used by higher-level agents when falling back to a vision model to propose
+ * coordinate-based actions.
+ */
+export type VisionExecutorActionKind = 'click_xy' | 'click_rect' | 'press' | 'type' | 'finish';
+export interface VisionExecutorAction {
+    kind: VisionExecutorActionKind;
+    args: Record<string, any>;
+}
+export declare function parseVisionExecutorAction(text: string): VisionExecutorAction;
+export declare function executeVisionExecutorAction(params: {
+    backend: any;
+    page?: any;
+    action: VisionExecutorAction;
+}): Promise<void>;
+//# sourceMappingURL=vision-executor.d.ts.map

package/dist/vision-executor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"vision-executor.d.ts","sourceRoot":"","sources":["../src/vision-executor.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,MAAM,MAAM,wBAAwB,GAAG,UAAU,GAAG,YAAY,GAAG,OAAO,GAAG,MAAM,GAAG,QAAQ,CAAC;AAE/F,MAAM,WAAW,oBAAoB;IACnC,IAAI,EAAE,wBAAwB,CAAC;IAC/B,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;CAC3B;AAED,wBAAgB,yBAAyB,CAAC,IAAI,EAAE,MAAM,GAAG,oBAAoB,CA0B5E;AAED,wBAAsB,2BAA2B,CAAC,MAAM,EAAE;IACxD,OAAO,EAAE,GAAG,CAAC;IACb,IAAI,CAAC,EAAE,GAAG,CAAC;IACX,MAAM,EAAE,oBAAoB,CAAC;CAC9B,GAAG,OAAO,CAAC,IAAI,CAAC,CA6BhB"}

package/dist/vision-executor.js

@@ -0,0 +1,60 @@
+"use strict";
+/**
+ * Vision executor primitives (shared parsing/execution helpers).
+ *
+ * This is used by higher-level agents when falling back to a vision model to propose
+ * coordinate-based actions.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parseVisionExecutorAction = parseVisionExecutorAction;
+exports.executeVisionExecutorAction = executeVisionExecutorAction;
+function parseVisionExecutorAction(text) {
+    const t = String(text || '')
+        .replace(/```[\w]*\n?/g, '')
+        .trim();
+    if (/^FINISH\s*\(\s*\)\s*$/i.test(t))
+        return { kind: 'finish', args: {} };
+    let m = t.match(/^PRESS\s*\(\s*["']([^"']+)["']\s*\)\s*$/i);
+    if (m)
+        return { kind: 'press', args: { key: m[1] } };
+    m = t.match(/^TYPE\s*\(\s*["']([\s\S]*?)["']\s*\)\s*$/i);
+    if (m)
+        return { kind: 'type', args: { text: m[1] } };
+    m = t.match(/^CLICK_XY\s*\(\s*(-?\d+(?:\.\d+)?)\s*,\s*(-?\d+(?:\.\d+)?)\s*\)\s*$/i);
+    if (m)
+        return { kind: 'click_xy', args: { x: Number(m[1]), y: Number(m[2]) } };
+    m = t.match(/^CLICK_RECT\s*\(\s*(-?\d+(?:\.\d+)?)\s*,\s*(-?\d+(?:\.\d+)?)\s*,\s*(-?\d+(?:\.\d+)?)\s*,\s*(-?\d+(?:\.\d+)?)\s*\)\s*$/i);
+    if (m)
+        return {
+            kind: 'click_rect',
+            args: { x: Number(m[1]), y: Number(m[2]), w: Number(m[3]), h: Number(m[4]) },
+        };
+    throw new Error(`unrecognized vision action: ${t.slice(0, 200)}`);
+}
+async function executeVisionExecutorAction(params) {
+    const { backend, page, action } = params;
+    if (action.kind === 'click_xy') {
+        await backend.mouse_click(Number(action.args.x), Number(action.args.y));
+        return;
+    }
+    if (action.kind === 'click_rect') {
+        const cx = Number(action.args.x) + Number(action.args.w) / 2;
+        const cy = Number(action.args.y) + Number(action.args.h) / 2;
+        await backend.mouse_click(cx, cy);
+        return;
+    }
+    if (action.kind === 'press') {
+        if (!page)
+            throw new Error('PRESS requires a Playwright page');
+        await page.keyboard.press(String(action.args.key));
+        return;
+    }
+    if (action.kind === 'type') {
+        await backend.type_text(String(action.args.text));
+        return;
+    }
+    if (action.kind === 'finish')
+        return;
+    throw new Error(`unknown vision action kind: ${action.kind}`);
+}
+//# sourceMappingURL=vision-executor.js.map

package/dist/vision-executor.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"vision-executor.js","sourceRoot":"","sources":["../src/vision-executor.ts"],"names":[],"mappings":";AAAA;;;;;GAKG;;AASH,8DA0BC;AAED,kEAiCC;AA7DD,SAAgB,yBAAyB,CAAC,IAAY;IACpD,MAAM,CAAC,GAAG,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;SACzB,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC;SAC3B,IAAI,EAAE,CAAC;IAEV,IAAI,wBAAwB,CAAC,IAAI,CAAC,CAAC,CAAC;QAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC;IAE1E,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,0CAA0C,CAAC,CAAC;IAC5D,IAAI,CAAC;QAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;IAErD,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,2CAA2C,CAAC,CAAC;IACzD,IAAI,CAAC;QAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;IAErD,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,sEAAsE,CAAC,CAAC;IACpF,IAAI,CAAC;QAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;IAE/E,CAAC,GAAG,CAAC,CAAC,KAAK,CACT,wHAAwH,CACzH,CAAC;IACF,IAAI,CAAC;QACH,OAAO;YACL,IAAI,EAAE,YAAY;YAClB,IAAI,EAAE,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE;SAC7E,CAAC;IAEJ,MAAM,IAAI,KAAK,CAAC,+BAA+B,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;AACpE,CAAC;AAEM,KAAK,UAAU,2BAA2B,CAAC,MAIjD;IACC,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM,CAAC;IAEzC,IAAI,MAAM,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;QAC/B,MAAM,OAAO,CAAC,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QACxE,OAAO;IACT,CAAC;IAED,IAAI,MAAM,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;QACjC,MAAM,EAAE,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAC7D,MAAM,EAAE,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAC7D,MAAM,OAAO,CAAC,WAAW,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;QAClC,OAAO;IACT,CAAC;IAED,IAAI,MAAM,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;QAC5B,IAAI,CAAC,IAAI;YAAE,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;QAC/D,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;QACnD,OAAO;IACT,CAAC;IAED,IAAI,MAAM,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;QAC3B,MAAM,OAAO,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;QAClD,OAAO;IACT,CAAC;IAED,IAAI,MAAM,CAAC,IAAI,KAAK,QAAQ;QAAE,OAAO;IAErC,MAAM,IAAI,KAAK,CAAC,+BAAgC,MAAc,CAAC,IAAI,EAAE,CAAC,CAAC;AACzE,CAAC"}