@hoilab/ada-cli 0.84.21 → 0.85.0

package/dist/core/computer/motor/actions.js

@@ -0,0 +1,157 @@
+import { toFiniteNumber } from "./platform/coerce.js";
+function mouseButton(value) {
+    return value === "right" || value === "middle" ? value : "left";
+}
+function clickCount(value, fallback = 1) {
+    return Math.max(1, Math.min(3, Math.round(toFiniteNumber(value, fallback))));
+}
+function scrollDelta(value) {
+    return Math.max(-10_000, Math.min(10_000, Math.round(toFiniteNumber(value, 0))));
+}
+function keys(value) {
+    if (!Array.isArray(value) || value.length === 0)
+        throw new Error("keypress.keys must contain at least one key.");
+    return value.map((key) => String(key));
+}
+function path(value, env) {
+    if (!Array.isArray(value) || value.length < 2)
+        throw new Error("drag.path must contain at least two points.");
+    return value.map((point, index) => {
+        const x = Array.isArray(point) ? toFiniteNumber(point[0], NaN) : toFiniteNumber(point?.x, NaN);
+        const y = Array.isArray(point) ? toFiniteNumber(point[1], NaN) : toFiniteNumber(point?.y, NaN);
+        env.validatePoint(x, y, `Drag point ${index + 1}`);
+        return { x, y };
+    });
+}
+function nativeTarget(action, operation, env) {
+    if (action.ref?.trim()) {
+        const node = env.node(action.ref.trim());
+        const semanticClick = operation === "click" || operation === "press";
+        if (semanticClick && node.isTextInput) {
+            const point = env.center(node);
+            env.validatePoint(point.x, point.y);
+            return point;
+        }
+        const onlyIncidentalActions = node.actions.every((candidate) => candidate === "AXShowMenu" || candidate === "AXScrollToVisible");
+        if (node.wireRef &&
+            !node.pictureOnly &&
+            (!semanticClick || node.canPress || node.canFocus || node.canSetValue || !onlyIncidentalActions)) {
+            return { ref: node.wireRef };
+        }
+        const point = env.center(node);
+        env.validatePoint(point.x, point.y);
+        return point;
+    }
+    const x = toFiniteNumber(action.x, NaN);
+    const y = toFiniteNumber(action.y, NaN);
+    if (Number.isFinite(x) && Number.isFinite(y)) {
+        env.validatePoint(x, y);
+        return { x, y };
+    }
+    if (operation === "drag" && action.path?.length)
+        return path(action.path, env)[0];
+    throw new Error(`${operation} requires either ref or both x and y.`);
+}
+function focusedTarget(env) {
+    if (!env.image)
+        throw new Error("Focused keyboard input requires an image-bearing state.");
+    return { focus: { x: Math.floor(env.image.width / 2), y: Math.floor(env.image.height / 2) } };
+}
+function containsEditable(node) {
+    if (node.canSetValue || node.role.toLowerCase().includes("text"))
+        return true;
+    return node.children.some(containsEditable);
+}
+export function prepareAction(action, state, env) {
+    const operation = action.action;
+    const usesCurrentFocus = !env.headless && state.currentFocus && !action.ref && (operation === "typeText" || operation === "keypress");
+    const target = usesCurrentFocus ? focusedTarget(env) : nativeTarget(action, operation, env);
+    const establishesFocus = !env.headless &&
+        Boolean(action.ref) &&
+        (operation === "click" || operation === "press") &&
+        containsEditable(env.node(action.ref));
+    const needsForeground = !env.headless && (operation === "click" || operation === "press") && "x" in target;
+    switch (operation) {
+        case "press":
+        case "click":
+            return {
+                action: operation,
+                target,
+                params: { button: mouseButton(action.button), clickCount: clickCount(action.clickCount) },
+                establishesFocus,
+                usesCurrentFocus: false,
+                needsForeground,
+            };
+        case "setText":
+            return {
+                action: operation,
+                target,
+                params: { text: action.text ?? "" },
+                establishesFocus: false,
+                usesCurrentFocus: false,
+                needsForeground: false,
+            };
+        case "typeText":
+            return {
+                action: operation,
+                target,
+                params: { text: action.text ?? "" },
+                establishesFocus: false,
+                usesCurrentFocus,
+                needsForeground: false,
+            };
+        case "keypress":
+            return {
+                action: operation,
+                target,
+                params: { keys: keys(action.keys) },
+                establishesFocus: false,
+                usesCurrentFocus,
+                needsForeground: false,
+            };
+        case "scroll":
+            return {
+                action: operation,
+                target,
+                params: { scrollX: scrollDelta(action.scrollX), scrollY: scrollDelta(action.scrollY) },
+                establishesFocus: false,
+                usesCurrentFocus: false,
+                needsForeground: false,
+            };
+        case "drag":
+            return {
+                action: operation,
+                target,
+                params: { path: path(action.path, env) },
+                establishesFocus: false,
+                usesCurrentFocus: false,
+                needsForeground: false,
+            };
+        case "moveMouse":
+            return {
+                action: operation,
+                target,
+                params: {},
+                establishesFocus: false,
+                usesCurrentFocus: false,
+                needsForeground: false,
+            };
+    }
+}
+export function canRetryInForeground(action, outcome, headless) {
+    return !headless && outcome === "didnt" && (action.action === "typeText" || action.action === "keypress");
+}
+export function outcomeAfterCheck(current, check) {
+    if (check === "verified")
+        return "worked";
+    if (check === "failed")
+        return "didnt";
+    return current;
+}
+export function outcomeAfterObservedValues(current, actions, valueForRef) {
+    if (actions.length === 0 || actions.some((action) => action.action !== "setText" || !action.ref))
+        return current;
+    const matches = actions.every((action) => valueForRef(action.ref) === (action.text ?? ""));
+    return matches ? "worked" : current;
+}
+//# sourceMappingURL=actions.js.map

package/dist/core/computer/motor/actions.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"actions.js","sourceRoot":"","sources":["../../../../src/core/computer/motor/actions.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAiFtD,SAAS,WAAW,CAAC,KAAc,EAAmB;IACrD,OAAO,KAAK,KAAK,OAAO,IAAI,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC;AAAA,CAChE;AAED,SAAS,UAAU,CAAC,KAAc,EAAE,QAAQ,GAAG,CAAC,EAAU;IACzD,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;AAAA,CAC7E;AAED,SAAS,WAAW,CAAC,KAAc,EAAU;IAC5C,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAAA,CACjF;AAED,SAAS,IAAI,CAAC,KAAc,EAAY;IACvC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;IACjH,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;AAAA,CACvC;AAED,SAAS,IAAI,CAAC,KAAuB,EAAE,GAAsB,EAAmC;IAC/F,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;IAC9G,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC;QAClC,MAAM,CAAC,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,KAAK,EAAE,CAAC,EAAE,GAAG,CAAC,CAAC;QAC/F,MAAM,CAAC,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,KAAK,EAAE,CAAC,EAAE,GAAG,CAAC,CAAC;QAC/F,GAAG,CAAC,aAAa,CAAC,CAAC,EAAE,CAAC,EAAE,cAAc,KAAK,GAAG,CAAC,EAAE,CAAC,CAAC;QACnD,OAAO,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC;IAAA,CAChB,CAAC,CAAC;AAAA,CACH;AAED,SAAS,YAAY,CAAC,MAAgB,EAAE,SAAmC,EAAE,GAAsB,EAAgB;IAClH,IAAI,MAAM,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC;QACxB,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;QACzC,MAAM,aAAa,GAAG,SAAS,KAAK,OAAO,IAAI,SAAS,KAAK,OAAO,CAAC;QACrE,IAAI,aAAa,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACvC,MAAM,KAAK,GAAG,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YAC/B,GAAG,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;YACpC,OAAO,KAAK,CAAC;QACd,CAAC;QACD,MAAM,qBAAqB,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAC/C,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,KAAK,YAAY,IAAI,SAAS,KAAK,mBAAmB,CAC9E,CAAC;QACF,IACC,IAAI,CAAC,OAAO;YACZ,CAAC,IAAI,CAAC,WAAW;YACjB,CAAC,CAAC,aAAa,IAAI,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,WAAW,IAAI,CAAC,qBAAqB,CAAC,EAC/F,CAAC;YACF,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC;QAC9B,CAAC;QACD,MAAM,KAAK,GAAG,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAC/B,GAAG,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;QACpC,OAAO,KAAK,CAAC;IACd,CAAC;IACD,MAAM,CAAC,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IACxC,MAAM,CAAC,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IACxC,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;QAC9C,GAAG,CAAC,aAAa,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACxB,OAAO,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC;IACjB,CAAC;IACD,IAAI,SAAS,KAAK,MAAM,IAAI,MAAM,CAAC,IAAI,EAAE,MAAM;QAAE,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IAClF,MAAM,IAAI,KAAK,CAAC,GAAG,SAAS,uCAAuC,CAAC,CAAC;AAAA,CACrE;AAED,SAAS,aAAa,CAAC,GAAsB,EAAgB;IAC5D,IAAI,CAAC,GAAG,CAAC,KAAK;QAAE,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;IAC3F,OAAO,EAAE,KAAK,EAAE,EAAE,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,EAAE,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,EAAE,CAAC;AAAA,CAC9F;AAED,SAAS,gBAAgB,CAAC,IAAiB,EAAW;IACrD,IAAI,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC;QAAE,OAAO,IAAI,CAAC;IAC9E,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;AAAA,CAC5C;AAED,MAAM,UAAU,aAAa,CAAC,MAAgB,EAAE,KAAkB,EAAE,GAAsB,EAAkB;IAC3G,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC;IAChC,MAAM,gBAAgB,GACrB,CAAC,GAAG,CAAC,QAAQ,IAAI,KAAK,CAAC,YAAY,IAAI,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC,SAAS,KAAK,UAAU,IAAI,SAAS,KAAK,UAAU,CAAC,CAAC;IAC9G,MAAM,MAAM,GAAG,gBAAgB,CAAC,CAAC,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,MAAM,EAAE,SAAS,EAAE,GAAG,CAAC,CAAC;IAC5F,MAAM,gBAAgB,GACrB,CAAC,GAAG,CAAC,QAAQ;QACb,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC;QACnB,CAAC,SAAS,KAAK,OAAO,IAAI,SAAS,KAAK,OAAO,CAAC;QAChD,gBAAgB,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,GAAI,CAAC,CAAC,CAAC;IACzC,MAAM,eAAe,GAAG,CAAC,GAAG,CAAC,QAAQ,IAAI,CAAC,SAAS,KAAK,OAAO,IAAI,SAAS,KAAK,OAAO,CAAC,IAAI,GAAG,IAAI,MAAM,CAAC;IAE3G,QAAQ,SAAS,EAAE,CAAC;QACnB,KAAK,OAAO,CAAC;QACb,KAAK,OAAO;YACX,OAAO;gBACN,MAAM,EAAE,SAAS;gBACjB,MAAM;gBACN,MAAM,EAAE,EAAE,MAAM,EAAE,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,UAAU,EAAE,UAAU,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE;gBACzF,gBAAgB;gBAChB,gBAAgB,EAAE,KAAK;gBACvB,eAAe;aACf,CAAC;QACH,KAAK,SAAS;YACb,OAAO;gBACN,MAAM,EAAE,SAAS;gBACjB,MAAM;gBACN,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,IAAI,EAAE,EAAE;gBACnC,gBAAgB,EAAE,KAAK;gBACvB,gBAAgB,EAAE,KAAK;gBACvB,eAAe,EAAE,KAAK;aACtB,CAAC;QACH,KAAK,UAAU;YACd,OAAO;gBACN,MAAM,EAAE,SAAS;gBACjB,MAAM;gBACN,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,IAAI,EAAE,EAAE;gBACnC,gBAAgB,EAAE,KAAK;gBACvB,gBAAgB;gBAChB,eAAe,EAAE,KAAK;aACtB,CAAC;QACH,KAAK,UAAU;YACd,OAAO;gBACN,MAAM,EAAE,SAAS;gBACjB,MAAM;gBACN,MAAM,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE;gBACnC,gBAAgB,EAAE,KAAK;gBACvB,gBAAgB;gBAChB,eAAe,EAAE,KAAK;aACtB,CAAC;QACH,KAAK,QAAQ;YACZ,OAAO;gBACN,MAAM,EAAE,SAAS;gBACjB,MAAM;gBACN,MAAM,EAAE,EAAE,OAAO,EAAE,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE;gBACtF,gBAAgB,EAAE,KAAK;gBACvB,gBAAgB,EAAE,KAAK;gBACvB,eAAe,EAAE,KAAK;aACtB,CAAC;QACH,KAAK,MAAM;YACV,OAAO;gBACN,MAAM,EAAE,SAAS;gBACjB,MAAM;gBACN,MAAM,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,GAAG,CAAC,EAAE;gBACxC,gBAAgB,EAAE,KAAK;gBACvB,gBAAgB,EAAE,KAAK;gBACvB,eAAe,EAAE,KAAK;aACtB,CAAC;QACH,KAAK,WAAW;YACf,OAAO;gBACN,MAAM,EAAE,SAAS;gBACjB,MAAM;gBACN,MAAM,EAAE,EAAE;gBACV,gBAAgB,EAAE,KAAK;gBACvB,gBAAgB,EAAE,KAAK;gBACvB,eAAe,EAAE,KAAK;aACtB,CAAC;IACJ,CAAC;AAAA,CACD;AAED,MAAM,UAAU,oBAAoB,CACnC,MAAsB,EACtB,OAAuC,EACvC,QAAiB,EACP;IACV,OAAO,CAAC,QAAQ,IAAI,OAAO,KAAK,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,KAAK,UAAU,IAAI,MAAM,CAAC,MAAM,KAAK,UAAU,CAAC,CAAC;AAAA,CAC1G;AAED,MAAM,UAAU,iBAAiB,CAChC,OAAuC,EACvC,KAA4C,EACX;IACjC,IAAI,KAAK,KAAK,UAAU;QAAE,OAAO,QAAQ,CAAC;IAC1C,IAAI,KAAK,KAAK,QAAQ;QAAE,OAAO,OAAO,CAAC;IACvC,OAAO,OAAO,CAAC;AAAA,CACf;AAED,MAAM,UAAU,0BAA0B,CACzC,OAAuC,EACvC,OAAmB,EACnB,WAAgD,EACf;IACjC,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,KAAK,SAAS,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC;QAAE,OAAO,OAAO,CAAC;IACjH,MAAM,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,WAAW,CAAC,MAAM,CAAC,GAAI,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,CAAC;IAC5F,OAAO,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC;AAAA,CACpC","sourcesContent":["import type { MouseButtonName, UiAction } from \"./contract.ts\";\nimport type { OutlineNode } from \"./outline.ts\";\nimport { toFiniteNumber } from \"./platform/coerce.ts\";\n\nexport type ActionTarget = { ref: string } | { x: number; y: number } | { focus: { x: number; y: number } };\n\nexport type PreparedAction =\n\t| {\n\t\t\taction: \"press\" | \"click\";\n\t\t\ttarget: ActionTarget;\n\t\t\tparams: { button?: MouseButtonName; clickCount?: number };\n\t\t\testablishesFocus: boolean;\n\t\t\tusesCurrentFocus: false;\n\t\t\tneedsForeground: boolean;\n\t  }\n\t| {\n\t\t\taction: \"setText\";\n\t\t\ttarget: ActionTarget;\n\t\t\tparams: { text: string };\n\t\t\testablishesFocus: false;\n\t\t\tusesCurrentFocus: false;\n\t\t\tneedsForeground: false;\n\t  }\n\t| {\n\t\t\taction: \"typeText\";\n\t\t\ttarget: ActionTarget;\n\t\t\tparams: { text: string };\n\t\t\testablishesFocus: false;\n\t\t\tusesCurrentFocus: boolean;\n\t\t\tneedsForeground: false;\n\t  }\n\t| {\n\t\t\taction: \"keypress\";\n\t\t\ttarget: ActionTarget;\n\t\t\tparams: { keys: string[] };\n\t\t\testablishesFocus: false;\n\t\t\tusesCurrentFocus: boolean;\n\t\t\tneedsForeground: false;\n\t  }\n\t| {\n\t\t\taction: \"scroll\";\n\t\t\ttarget: ActionTarget;\n\t\t\tparams: { scrollX: number; scrollY: number };\n\t\t\testablishesFocus: false;\n\t\t\tusesCurrentFocus: false;\n\t\t\tneedsForeground: false;\n\t  }\n\t| {\n\t\t\taction: \"drag\";\n\t\t\ttarget: ActionTarget;\n\t\t\tparams: { path: Array<{ x: number; y: number }> };\n\t\t\testablishesFocus: false;\n\t\t\tusesCurrentFocus: false;\n\t\t\tneedsForeground: false;\n\t  }\n\t| {\n\t\t\taction: \"moveMouse\";\n\t\t\ttarget: ActionTarget;\n\t\t\tparams: Record<string, never>;\n\t\t\testablishesFocus: false;\n\t\t\tusesCurrentFocus: false;\n\t\t\tneedsForeground: false;\n\t  }\n\t| {\n\t\t\taction: \"wait\";\n\t\t\tparams: { ms: number };\n\t\t\testablishesFocus: false;\n\t\t\tusesCurrentFocus: false;\n\t\t\tneedsForeground: false;\n\t  };\n\nexport interface ActionState {\n\tcurrentFocus: boolean;\n}\n\nexport interface ActionEnvironment {\n\theadless: boolean;\n\timage?: { width: number; height: number };\n\tnode(ref: string): OutlineNode;\n\tcenter(node: OutlineNode): { x: number; y: number };\n\tvalidatePoint(x: number, y: number, label?: string): void;\n}\n\nfunction mouseButton(value: unknown): MouseButtonName {\n\treturn value === \"right\" || value === \"middle\" ? value : \"left\";\n}\n\nfunction clickCount(value: unknown, fallback = 1): number {\n\treturn Math.max(1, Math.min(3, Math.round(toFiniteNumber(value, fallback))));\n}\n\nfunction scrollDelta(value: unknown): number {\n\treturn Math.max(-10_000, Math.min(10_000, Math.round(toFiniteNumber(value, 0))));\n}\n\nfunction keys(value: unknown): string[] {\n\tif (!Array.isArray(value) || value.length === 0) throw new Error(\"keypress.keys must contain at least one key.\");\n\treturn value.map((key) => String(key));\n}\n\nfunction path(value: UiAction[\"path\"], env: ActionEnvironment): Array<{ x: number; y: number }> {\n\tif (!Array.isArray(value) || value.length < 2) throw new Error(\"drag.path must contain at least two points.\");\n\treturn value.map((point, index) => {\n\t\tconst x = Array.isArray(point) ? toFiniteNumber(point[0], NaN) : toFiniteNumber(point?.x, NaN);\n\t\tconst y = Array.isArray(point) ? toFiniteNumber(point[1], NaN) : toFiniteNumber(point?.y, NaN);\n\t\tenv.validatePoint(x, y, `Drag point ${index + 1}`);\n\t\treturn { x, y };\n\t});\n}\n\nfunction nativeTarget(action: UiAction, operation: PreparedAction[\"action\"], env: ActionEnvironment): ActionTarget {\n\tif (action.ref?.trim()) {\n\t\tconst node = env.node(action.ref.trim());\n\t\tconst semanticClick = operation === \"click\" || operation === \"press\";\n\t\tif (semanticClick && node.isTextInput) {\n\t\t\tconst point = env.center(node);\n\t\t\tenv.validatePoint(point.x, point.y);\n\t\t\treturn point;\n\t\t}\n\t\tconst onlyIncidentalActions = node.actions.every(\n\t\t\t(candidate) => candidate === \"AXShowMenu\" || candidate === \"AXScrollToVisible\",\n\t\t);\n\t\tif (\n\t\t\tnode.wireRef &&\n\t\t\t!node.pictureOnly &&\n\t\t\t(!semanticClick || node.canPress || node.canFocus || node.canSetValue || !onlyIncidentalActions)\n\t\t) {\n\t\t\treturn { ref: node.wireRef };\n\t\t}\n\t\tconst point = env.center(node);\n\t\tenv.validatePoint(point.x, point.y);\n\t\treturn point;\n\t}\n\tconst x = toFiniteNumber(action.x, NaN);\n\tconst y = toFiniteNumber(action.y, NaN);\n\tif (Number.isFinite(x) && Number.isFinite(y)) {\n\t\tenv.validatePoint(x, y);\n\t\treturn { x, y };\n\t}\n\tif (operation === \"drag\" && action.path?.length) return path(action.path, env)[0];\n\tthrow new Error(`${operation} requires either ref or both x and y.`);\n}\n\nfunction focusedTarget(env: ActionEnvironment): ActionTarget {\n\tif (!env.image) throw new Error(\"Focused keyboard input requires an image-bearing state.\");\n\treturn { focus: { x: Math.floor(env.image.width / 2), y: Math.floor(env.image.height / 2) } };\n}\n\nfunction containsEditable(node: OutlineNode): boolean {\n\tif (node.canSetValue || node.role.toLowerCase().includes(\"text\")) return true;\n\treturn node.children.some(containsEditable);\n}\n\nexport function prepareAction(action: UiAction, state: ActionState, env: ActionEnvironment): PreparedAction {\n\tconst operation = action.action;\n\tconst usesCurrentFocus =\n\t\t!env.headless && state.currentFocus && !action.ref && (operation === \"typeText\" || operation === \"keypress\");\n\tconst target = usesCurrentFocus ? focusedTarget(env) : nativeTarget(action, operation, env);\n\tconst establishesFocus =\n\t\t!env.headless &&\n\t\tBoolean(action.ref) &&\n\t\t(operation === \"click\" || operation === \"press\") &&\n\t\tcontainsEditable(env.node(action.ref!));\n\tconst needsForeground = !env.headless && (operation === \"click\" || operation === \"press\") && \"x\" in target;\n\n\tswitch (operation) {\n\t\tcase \"press\":\n\t\tcase \"click\":\n\t\t\treturn {\n\t\t\t\taction: operation,\n\t\t\t\ttarget,\n\t\t\t\tparams: { button: mouseButton(action.button), clickCount: clickCount(action.clickCount) },\n\t\t\t\testablishesFocus,\n\t\t\t\tusesCurrentFocus: false,\n\t\t\t\tneedsForeground,\n\t\t\t};\n\t\tcase \"setText\":\n\t\t\treturn {\n\t\t\t\taction: operation,\n\t\t\t\ttarget,\n\t\t\t\tparams: { text: action.text ?? \"\" },\n\t\t\t\testablishesFocus: false,\n\t\t\t\tusesCurrentFocus: false,\n\t\t\t\tneedsForeground: false,\n\t\t\t};\n\t\tcase \"typeText\":\n\t\t\treturn {\n\t\t\t\taction: operation,\n\t\t\t\ttarget,\n\t\t\t\tparams: { text: action.text ?? \"\" },\n\t\t\t\testablishesFocus: false,\n\t\t\t\tusesCurrentFocus,\n\t\t\t\tneedsForeground: false,\n\t\t\t};\n\t\tcase \"keypress\":\n\t\t\treturn {\n\t\t\t\taction: operation,\n\t\t\t\ttarget,\n\t\t\t\tparams: { keys: keys(action.keys) },\n\t\t\t\testablishesFocus: false,\n\t\t\t\tusesCurrentFocus,\n\t\t\t\tneedsForeground: false,\n\t\t\t};\n\t\tcase \"scroll\":\n\t\t\treturn {\n\t\t\t\taction: operation,\n\t\t\t\ttarget,\n\t\t\t\tparams: { scrollX: scrollDelta(action.scrollX), scrollY: scrollDelta(action.scrollY) },\n\t\t\t\testablishesFocus: false,\n\t\t\t\tusesCurrentFocus: false,\n\t\t\t\tneedsForeground: false,\n\t\t\t};\n\t\tcase \"drag\":\n\t\t\treturn {\n\t\t\t\taction: operation,\n\t\t\t\ttarget,\n\t\t\t\tparams: { path: path(action.path, env) },\n\t\t\t\testablishesFocus: false,\n\t\t\t\tusesCurrentFocus: false,\n\t\t\t\tneedsForeground: false,\n\t\t\t};\n\t\tcase \"moveMouse\":\n\t\t\treturn {\n\t\t\t\taction: operation,\n\t\t\t\ttarget,\n\t\t\t\tparams: {},\n\t\t\t\testablishesFocus: false,\n\t\t\t\tusesCurrentFocus: false,\n\t\t\t\tneedsForeground: false,\n\t\t\t};\n\t}\n}\n\nexport function canRetryInForeground(\n\taction: PreparedAction,\n\toutcome: \"worked\" | \"didnt\" | \"unknown\",\n\theadless: boolean,\n): boolean {\n\treturn !headless && outcome === \"didnt\" && (action.action === \"typeText\" || action.action === \"keypress\");\n}\n\nexport function outcomeAfterCheck(\n\tcurrent: \"worked\" | \"didnt\" | \"unknown\",\n\tcheck: \"verified\" | \"preexisting\" | \"failed\",\n): \"worked\" | \"didnt\" | \"unknown\" {\n\tif (check === \"verified\") return \"worked\";\n\tif (check === \"failed\") return \"didnt\";\n\treturn current;\n}\n\nexport function outcomeAfterObservedValues(\n\tcurrent: \"worked\" | \"didnt\" | \"unknown\",\n\tactions: UiAction[],\n\tvalueForRef: (ref: string) => string | undefined,\n): \"worked\" | \"didnt\" | \"unknown\" {\n\tif (actions.length === 0 || actions.some((action) => action.action !== \"setText\" || !action.ref)) return current;\n\tconst matches = actions.every((action) => valueForRef(action.ref!) === (action.text ?? \"\"));\n\treturn matches ? \"worked\" : current;\n}\n"]}

package/dist/core/computer/motor/bridge.d.ts

@@ -0,0 +1,232 @@
+import type { AgentToolResult, AgentToolUpdateCallback, ExtensionContext } from "../../extensions/types.ts";
+import { type CdpConsoleEntry } from "./cdp.ts";
+import { type ActParams, type EvaluateBrowserParams, type ExpandUiParams, type FindParams, type InspectUiParams, type LaunchBrowserParams, type NavigateBrowserParams, type ObserveParams, type ReadTextParams, type SearchUiParams, type WaitForParams } from "./contract.ts";
+import { type WindowNote } from "./note.ts";
+import { type OutlineChange, type SerializedOutline, type SerializedOutlineNode, type SerializedOutlineSearchMatch } from "./outline.ts";
+import type { FramePoints, HelperActPerformed, HelperActResult, NativeInputDelivery, PlatformDiagnostics } from "./platform/types.ts";
+export type { ActParams, EvaluateBrowserParams, ExpandUiParams, FindParams, ImageMode, InspectUiParams, LaunchBrowserParams, MouseButtonName, NavigateBrowserParams, ObserveParams, ObserveTargetParams, ReadTextParams, RootSelector, SearchUiParams, StateTargetParams, UiAction, WaitForParams, } from "./contract.ts";
+interface ActivationFlags {
+    activated: boolean;
+    unminimized: boolean;
+    raised: boolean;
+}
+type ExecutionVariant = "stealth" | "default";
+type ActionDelivery = "ax" | NativeInputDelivery;
+type DeliveryPolicy = "ax_only" | "background" | "default" | "foreground";
+type ActOutcome = "worked" | "didnt" | "unknown";
+interface ExecutionTrace {
+    strategy: "look" | "act" | "wait" | "browser_open_location" | "cdp_navigate";
+    runtimeMode?: ExecutionVariant;
+    variant?: ExecutionVariant;
+    stealthCompatible?: boolean;
+    delivery?: ActionDelivery;
+    deliveryPolicy?: DeliveryPolicy;
+    outcome?: ActOutcome;
+    performed?: HelperActPerformed;
+    evidence?: Record<string, unknown>;
+    error?: HelperActResult["error"];
+    rootDelta?: HelperActResult["rootDelta"];
+    steps?: ExecutionTrace[];
+    actionCount?: number;
+    stoppedAt?: number;
+    backgroundFirst?: boolean;
+    escalatedToForeground?: boolean;
+    escalationReason?: string;
+    backgroundAttempt?: {
+        outcome: "foreground_required" | "didnt";
+        reason: string;
+    };
+    verification?: {
+        status: "verified" | "preexisting" | "failed";
+        text?: string;
+        role?: string;
+        value?: string;
+        gone?: boolean;
+        timeoutMs: number;
+    };
+}
+interface ComputerUseDetails {
+    tool: string;
+    target: {
+        app: string;
+        bundleId?: string;
+        pid: number;
+        windowTitle: string;
+        windowId: number;
+        windowRef?: string;
+        nativeWindowRef?: string;
+    };
+    capture: {
+        stateId: string;
+        width: number;
+        height: number;
+        scaleFactor: number;
+        timestamp: number;
+        coordinateSpace: "window-relative-screenshot-pixels";
+    };
+    lookId?: string;
+    view: "full" | "diff";
+    baseStateId?: string;
+    changes?: OutlineChange[];
+    viewReason?: "root_replaced" | "change_budget_exceeded" | "identity_confidence_low";
+    renderedOutline?: string;
+    outline?: SerializedOutline;
+    note?: WindowNote;
+    activation: ActivationFlags;
+    execution: ExecutionTrace;
+    config?: {
+        browser_use: boolean;
+        headless: boolean;
+    };
+    helper?: PlatformDiagnostics;
+    status?: "ok";
+    axDiagnostics?: {
+        reason?: string;
+        message?: string;
+        debug?: unknown;
+    };
+    /** Recent browser console messages/exceptions; only present when CDP is active. */
+    console?: CdpConsoleEntry[];
+    imageReason?: "fallback_recovery" | "browser_ax_window_unavailable" | "no_ax_targets" | "sparse_ax_targets" | "weak_ax_targets" | "unlabeled_ax_targets" | "duplicated_ax_labels" | "browser_wait_verification";
+}
+interface TerminalDesktopActionDetails {
+    tool: "act_ui";
+    status: "target_closed" | "post_action_observation_failed";
+    baseStateId: string;
+    target: {
+        app: string;
+        bundleId?: string;
+        pid: number;
+        windowTitle: string;
+        windowId: number;
+        windowRef?: string;
+        nativeWindowRef?: string;
+    };
+    execution: ExecutionTrace;
+    error: {
+        code: string;
+        message: string;
+    };
+}
+interface ListWindowsDetails {
+    tool: "find_roots";
+    query: FindParams;
+    totalMatches?: number;
+    returned?: number;
+    hasMore?: boolean;
+    windows: Array<{
+        app: string;
+        bundleId?: string;
+        pid: number;
+        kind: string;
+        windowTitle: string;
+        windowId?: number;
+        windowRef: string;
+        nativeWindowRef?: string;
+        framePoints: FramePoints;
+        scaleFactor: number;
+        isMinimized: boolean;
+        isOnscreen: boolean;
+        isMain: boolean;
+        isFocused: boolean;
+        isModal: boolean;
+        sheetCount?: number;
+        role?: string;
+        subrole?: string;
+        pairing?: {
+            confidence: "exact" | "high" | "low";
+            score: number;
+        };
+        zOrder: number;
+        browserUseAllowed: boolean;
+        score: number;
+        url?: string;
+    }>;
+    config: {
+        browser_use: boolean;
+        headless: boolean;
+    };
+}
+interface BrowserObservationDetails {
+    tool: string;
+    kind: "browser_page";
+    stateId: string;
+    baseStateId?: string;
+    view: "full" | "diff";
+    changes?: OutlineChange[];
+    root: {
+        ref: string;
+        kind: "browser_page";
+        title: string;
+        url: string;
+    };
+    outline: SerializedOutline;
+    renderedOutline: string;
+}
+interface EvaluateBrowserDetails {
+    tool: "evaluate_browser";
+    baseStateId: string;
+    stateId: string;
+    view: "full" | "diff";
+    changes?: OutlineChange[];
+    outline: SerializedOutline;
+    renderedOutline: string;
+}
+interface ReadTextDetails {
+    tool: "read_text";
+    ref?: string;
+    offset: number;
+    limit: number;
+    totalChars?: number;
+    totalBytes?: number;
+    hasMore: boolean;
+    complete?: boolean;
+    text: string;
+}
+interface WaitForDetails {
+    tool: "wait_for";
+    stateId: string;
+    baseStateId?: string;
+    view: "full" | "diff";
+    changes?: OutlineChange[];
+    found: boolean;
+    gone?: boolean;
+    timedOut?: boolean;
+    target?: SerializedOutlineSearchMatch;
+    nodeCount?: number;
+    text?: string;
+    role?: string;
+    value?: string;
+    scopeRef?: string;
+    outline: SerializedOutline;
+    renderedOutline: string;
+}
+interface OutlineToolDetails {
+    tool: "search_ui" | "expand_ui" | "inspect_ui";
+    stateId?: string;
+    lookId?: string;
+    outline?: SerializedOutline;
+    renderedOutline?: string;
+    totalMatches?: number;
+    returned?: number;
+    hasMore?: boolean;
+    matches?: SerializedOutlineSearchMatch[];
+    target?: SerializedOutlineNode;
+    note?: WindowNote;
+}
+/** Release handles and state owned by the current Pi session. */
+export declare function shutdownComputerUseSession(): Promise<void>;
+export declare function ensureComputerUseSetup(ctx: ExtensionContext, signal?: AbortSignal): Promise<void>;
+export declare const executeFind: (_toolCallId: string, params: FindParams, signal: AbortSignal | undefined, _onUpdate: AgentToolUpdateCallback<ListWindowsDetails> | undefined, ctx: ExtensionContext) => Promise<AgentToolResult<ListWindowsDetails>>;
+export declare const executeReadText: (_toolCallId: string, params: ReadTextParams, signal: AbortSignal | undefined, _onUpdate: AgentToolUpdateCallback<ReadTextDetails> | undefined, ctx: ExtensionContext) => Promise<AgentToolResult<ReadTextDetails>>;
+export declare const executeWaitFor: (_toolCallId: string, params: WaitForParams, signal: AbortSignal | undefined, _onUpdate: AgentToolUpdateCallback<WaitForDetails> | undefined, ctx: ExtensionContext) => Promise<AgentToolResult<WaitForDetails>>;
+export declare const executeObserve: (_toolCallId: string, params: ObserveParams, signal: AbortSignal | undefined, _onUpdate: AgentToolUpdateCallback<BrowserObservationDetails | ComputerUseDetails> | undefined, ctx: ExtensionContext) => Promise<AgentToolResult<BrowserObservationDetails | ComputerUseDetails>>;
+export declare const executeSearchUi: (_toolCallId: string, params: SearchUiParams, signal: AbortSignal | undefined, _onUpdate: AgentToolUpdateCallback<OutlineToolDetails> | undefined, ctx: ExtensionContext) => Promise<AgentToolResult<OutlineToolDetails>>;
+export declare const executeExpandUi: (_toolCallId: string, params: ExpandUiParams, signal: AbortSignal | undefined, _onUpdate: AgentToolUpdateCallback<OutlineToolDetails> | undefined, ctx: ExtensionContext) => Promise<AgentToolResult<OutlineToolDetails>>;
+export declare const executeInspectUi: (_toolCallId: string, params: InspectUiParams, signal: AbortSignal | undefined, _onUpdate: AgentToolUpdateCallback<OutlineToolDetails> | undefined, ctx: ExtensionContext) => Promise<AgentToolResult<OutlineToolDetails>>;
+export declare const executeAct: (_toolCallId: string, params: ActParams, signal: AbortSignal | undefined, _onUpdate: AgentToolUpdateCallback<BrowserObservationDetails | ComputerUseDetails | TerminalDesktopActionDetails> | undefined, ctx: ExtensionContext) => Promise<AgentToolResult<BrowserObservationDetails | ComputerUseDetails | TerminalDesktopActionDetails>>;
+export declare const executeNavigateBrowser: (_toolCallId: string, params: NavigateBrowserParams, signal: AbortSignal | undefined, _onUpdate: AgentToolUpdateCallback<BrowserObservationDetails> | undefined, ctx: ExtensionContext) => Promise<AgentToolResult<BrowserObservationDetails>>;
+export declare const executeEvaluateBrowser: (_toolCallId: string, params: EvaluateBrowserParams, signal: AbortSignal | undefined, _onUpdate: AgentToolUpdateCallback<EvaluateBrowserDetails> | undefined, ctx: ExtensionContext) => Promise<AgentToolResult<EvaluateBrowserDetails>>;
+export declare const executeLaunchBrowser: (_toolCallId: string, params: LaunchBrowserParams, signal: AbortSignal | undefined, _onUpdate: AgentToolUpdateCallback<BrowserObservationDetails> | undefined, ctx: ExtensionContext) => Promise<AgentToolResult<BrowserObservationDetails>>;
+export declare function reconstructStateFromBranch(ctx: ExtensionContext): void;
+//# sourceMappingURL=bridge.d.ts.map