@moxxy/plugin-computer-control 0.41.2 → 0.42.0

package/dist/contract/outcome.d.ts

@@ -0,0 +1,61 @@
+import { z } from 'zod';
+/** Why an action did not go through, with the model's next step. One table for every platform. */
+declare const hints: {
+    readonly app_not_allowed: "Ask for this app with computer_request_access, or finish the task without it.";
+    readonly app_not_found: "Check the name with computer_list_apps and use the bundle identifier or app id it returns.";
+    readonly ambiguous_app: "Several apps match that name; pick one id from computer_list_apps.";
+    readonly tier_insufficient: "The grant for this app does not allow this action; ask the user for a higher level with computer_request_access or use a read-only path.";
+    readonly permissions_not_granted: "The operating system has not granted Accessibility or Screen Recording; ask the user to allow it in System Settings, then try again.";
+    readonly permissions_pending: "The user has not answered the permission prompt yet; wait for the answer before acting.";
+    readonly user_stopped: "The user stopped Computer Use; do not act again until they ask.";
+    readonly user_intervened: "The user took over the mouse or keyboard; the action was paused. Observe again once they resume.";
+    readonly screen_locked: "The screen is locked; ask the user to unlock it.";
+    readonly not_frontmost: "Another app came to the front; call computer_get_app_state for the target app and retry only if it is still needed.";
+    readonly target_blocked: "A dialog or sheet covers the target; handle it first, based on a fresh computer_get_app_state.";
+    readonly hit_test_mismatch: "Something else is at that point now; call computer_get_app_state and target by element_index.";
+    readonly point_outside_frame: "Use coordinates inside the latest screenshot of this window.";
+    readonly screen_changed: "The pixels at the target changed since the last screenshot; call computer_get_app_state before clicking.";
+    readonly stale_state: "That index or point belongs to an older state; call computer_get_app_state and use its indices.";
+    readonly no_state: "Call computer_get_app_state for this app first; actions use its indices and screenshot coordinates.";
+    readonly unsupported_action: "This element does not support that action; use an action listed for it or click by coordinates.";
+    readonly canvas: "The point is on a canvas without accessibility; use computer_click or computer_drag on screenshot coordinates, or keyboard shortcuts.";
+    readonly would_replace_content: "Typing would replace the whole field; select the insertion point first or confirm with the user.";
+    readonly system_key_combo: "System key combinations need the systemKeyCombos grant from computer_request_access.";
+    readonly clipboard_not_granted: "Clipboard access needs the clipboardRead or clipboardWrite grant from computer_request_access.";
+    readonly own_window: "That target belongs to Moxxy itself; act only on the granted app.";
+    readonly protected_path: "A save dialog points at a protected location; choose a different folder or ask the user.";
+    readonly no_progress: "The same action left the app unchanged twice; try another method (element action, keyboard shortcut, menu or coordinates).";
+    readonly invalid_key: "Use xdotool key syntax such as \"Return\", \"Tab\", \"super+c\", \"ctrl+shift+Tab\", \"Page_Down\" or \"KP_0\".";
+    readonly timeout: "The app did not respond in time; the action was not retried. Observe again before deciding.";
+    readonly helper_failed: "The native helper failed or answered unexpectedly; the action was not retried. Observe again before deciding.";
+    readonly page_loading: "The click followed a link to another page, and after several seconds the page has not changed: it is probably still loading. Look again with computer_get_app_state before you act; clicking the link again, double-clicking it or pressing Return on it can start the same thing twice (a second new design, a second order).";
+};
+export type ErrorCode = keyof typeof hints;
+export declare const errorCodes: ErrorCode[];
+export declare function isErrorCode(value: string): value is ErrorCode;
+export declare function hintFor(code: ErrorCode): string;
+/** Technical result of one action; `delivered` means sent, never verified. */
+export declare const actionResultSchema: z.ZodObject<{
+    outcome: z.ZodEnum<["delivered", "ineffective", "unsupported", "blocked"]>;
+    code: z.ZodOptional<z.ZodEnum<["app_not_allowed" | "app_not_found" | "ambiguous_app" | "tier_insufficient" | "permissions_not_granted" | "permissions_pending" | "user_stopped" | "user_intervened" | "screen_locked" | "not_frontmost" | "target_blocked" | "hit_test_mismatch" | "point_outside_frame" | "screen_changed" | "stale_state" | "no_state" | "unsupported_action" | "canvas" | "would_replace_content" | "system_key_combo" | "clipboard_not_granted" | "own_window" | "protected_path" | "no_progress" | "invalid_key" | "timeout" | "helper_failed" | "page_loading", ...("app_not_allowed" | "app_not_found" | "ambiguous_app" | "tier_insufficient" | "permissions_not_granted" | "permissions_pending" | "user_stopped" | "user_intervened" | "screen_locked" | "not_frontmost" | "target_blocked" | "hit_test_mismatch" | "point_outside_frame" | "screen_changed" | "stale_state" | "no_state" | "unsupported_action" | "canvas" | "would_replace_content" | "system_key_combo" | "clipboard_not_granted" | "own_window" | "protected_path" | "no_progress" | "invalid_key" | "timeout" | "helper_failed" | "page_loading")[]]>>;
+    hint: z.ZodOptional<z.ZodString>;
+    method: z.ZodOptional<z.ZodEnum<["ax", "input", "background"]>>;
+}, "strict", z.ZodTypeAny, {
+    outcome: "delivered" | "ineffective" | "unsupported" | "blocked";
+    code?: "app_not_allowed" | "app_not_found" | "ambiguous_app" | "tier_insufficient" | "permissions_not_granted" | "permissions_pending" | "user_stopped" | "user_intervened" | "screen_locked" | "not_frontmost" | "target_blocked" | "hit_test_mismatch" | "point_outside_frame" | "screen_changed" | "stale_state" | "no_state" | "unsupported_action" | "canvas" | "would_replace_content" | "system_key_combo" | "clipboard_not_granted" | "own_window" | "protected_path" | "no_progress" | "invalid_key" | "timeout" | "helper_failed" | "page_loading" | undefined;
+    hint?: string | undefined;
+    method?: "ax" | "input" | "background" | undefined;
+}, {
+    outcome: "delivered" | "ineffective" | "unsupported" | "blocked";
+    code?: "app_not_allowed" | "app_not_found" | "ambiguous_app" | "tier_insufficient" | "permissions_not_granted" | "permissions_pending" | "user_stopped" | "user_intervened" | "screen_locked" | "not_frontmost" | "target_blocked" | "hit_test_mismatch" | "point_outside_frame" | "screen_changed" | "stale_state" | "no_state" | "unsupported_action" | "canvas" | "would_replace_content" | "system_key_combo" | "clipboard_not_granted" | "own_window" | "protected_path" | "no_progress" | "invalid_key" | "timeout" | "helper_failed" | "page_loading" | undefined;
+    hint?: string | undefined;
+    method?: "ax" | "input" | "background" | undefined;
+}>;
+export type ActionResult = z.infer<typeof actionResultSchema>;
+export declare function describeResult(result: ActionResult): string;
+export declare class ComputerUseError extends Error {
+    readonly code: ErrorCode;
+    constructor(code: ErrorCode, message: string);
+}
+export {};
+//# sourceMappingURL=outcome.d.ts.map

package/dist/contract/outcome.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"outcome.d.ts","sourceRoot":"","sources":["../../src/contract/outcome.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,kGAAkG;AAClG,QAAA,MAAM,KAAK;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA6BD,CAAC;AAEX,MAAM,MAAM,SAAS,GAAG,MAAM,OAAO,KAAK,CAAC;AAC3C,eAAO,MAAM,UAAU,EAAyB,SAAS,EAAE,CAAC;AAG5D,wBAAgB,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,KAAK,IAAI,SAAS,CAE7D;AAED,wBAAgB,OAAO,CAAC,IAAI,EAAE,SAAS,GAAG,MAAM,CAE/C;AAED,8EAA8E;AAC9E,eAAO,MAAM,kBAAkB;;;;;;;;;;;;;;;EAKpB,CAAC;AACZ,MAAM,MAAM,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,CAAC;AAE9D,wBAAgB,cAAc,CAAC,MAAM,EAAE,YAAY,GAAG,MAAM,CAM3D;AAED,qBAAa,gBAAiB,SAAQ,KAAK;IACzC,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC;gBAEb,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM;CAK7C"}

package/dist/contract/outcome.js

@@ -0,0 +1,63 @@
+import { z } from 'zod';
+/** Why an action did not go through, with the model's next step. One table for every platform. */
+const hints = {
+    app_not_allowed: 'Ask for this app with computer_request_access, or finish the task without it.',
+    app_not_found: 'Check the name with computer_list_apps and use the bundle identifier or app id it returns.',
+    ambiguous_app: 'Several apps match that name; pick one id from computer_list_apps.',
+    tier_insufficient: 'The grant for this app does not allow this action; ask the user for a higher level with computer_request_access or use a read-only path.',
+    permissions_not_granted: 'The operating system has not granted Accessibility or Screen Recording; ask the user to allow it in System Settings, then try again.',
+    permissions_pending: 'The user has not answered the permission prompt yet; wait for the answer before acting.',
+    user_stopped: 'The user stopped Computer Use; do not act again until they ask.',
+    user_intervened: 'The user took over the mouse or keyboard; the action was paused. Observe again once they resume.',
+    screen_locked: 'The screen is locked; ask the user to unlock it.',
+    not_frontmost: 'Another app came to the front; call computer_get_app_state for the target app and retry only if it is still needed.',
+    target_blocked: 'A dialog or sheet covers the target; handle it first, based on a fresh computer_get_app_state.',
+    hit_test_mismatch: 'Something else is at that point now; call computer_get_app_state and target by element_index.',
+    point_outside_frame: 'Use coordinates inside the latest screenshot of this window.',
+    screen_changed: 'The pixels at the target changed since the last screenshot; call computer_get_app_state before clicking.',
+    stale_state: 'That index or point belongs to an older state; call computer_get_app_state and use its indices.',
+    no_state: 'Call computer_get_app_state for this app first; actions use its indices and screenshot coordinates.',
+    unsupported_action: 'This element does not support that action; use an action listed for it or click by coordinates.',
+    canvas: 'The point is on a canvas without accessibility; use computer_click or computer_drag on screenshot coordinates, or keyboard shortcuts.',
+    would_replace_content: 'Typing would replace the whole field; select the insertion point first or confirm with the user.',
+    system_key_combo: 'System key combinations need the systemKeyCombos grant from computer_request_access.',
+    clipboard_not_granted: 'Clipboard access needs the clipboardRead or clipboardWrite grant from computer_request_access.',
+    own_window: 'That target belongs to Moxxy itself; act only on the granted app.',
+    protected_path: 'A save dialog points at a protected location; choose a different folder or ask the user.',
+    no_progress: 'The same action left the app unchanged twice; try another method (element action, keyboard shortcut, menu or coordinates).',
+    invalid_key: 'Use xdotool key syntax such as "Return", "Tab", "super+c", "ctrl+shift+Tab", "Page_Down" or "KP_0".',
+    timeout: 'The app did not respond in time; the action was not retried. Observe again before deciding.',
+    helper_failed: 'The native helper failed or answered unexpectedly; the action was not retried. Observe again before deciding.',
+    page_loading: 'The click followed a link to another page, and after several seconds the page has not changed: it is probably still loading. Look again with computer_get_app_state before you act; clicking the link again, double-clicking it or pressing Return on it can start the same thing twice (a second new design, a second order).',
+};
+export const errorCodes = Object.keys(hints);
+const errorCodeSchema = z.enum(errorCodes);
+export function isErrorCode(value) {
+    return Object.hasOwn(hints, value);
+}
+export function hintFor(code) {
+    return hints[code];
+}
+/** Technical result of one action; `delivered` means sent, never verified. */
+export const actionResultSchema = z.object({
+    outcome: z.enum(['delivered', 'ineffective', 'unsupported', 'blocked']),
+    code: errorCodeSchema.optional(),
+    hint: z.string().max(1000).optional(),
+    method: z.enum(['ax', 'input', 'background']).optional(),
+}).strict();
+export function describeResult(result) {
+    if (result.outcome === 'delivered') {
+        return ['Action delivered; verify its effect in the new state before relying on it.', result.code ? hintFor(result.code) : undefined].filter(Boolean).join(' ');
+    }
+    const hint = result.hint ?? (result.code ? hintFor(result.code) : undefined);
+    return [`Action ${result.outcome}${result.code ? ` (${result.code})` : ''}.`, hint].filter(Boolean).join(' ');
+}
+export class ComputerUseError extends Error {
+    code;
+    constructor(code, message) {
+        super(`${message}. ${hintFor(code)}`);
+        this.name = 'ComputerUseError';
+        this.code = code;
+    }
+}
+//# sourceMappingURL=outcome.js.map

package/dist/contract/outcome.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"outcome.js","sourceRoot":"","sources":["../../src/contract/outcome.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,kGAAkG;AAClG,MAAM,KAAK,GAAG;IACZ,eAAe,EAAE,+EAA+E;IAChG,aAAa,EAAE,4FAA4F;IAC3G,aAAa,EAAE,oEAAoE;IACnF,iBAAiB,EAAE,0IAA0I;IAC7J,uBAAuB,EAAE,sIAAsI;IAC/J,mBAAmB,EAAE,yFAAyF;IAC9G,YAAY,EAAE,iEAAiE;IAC/E,eAAe,EAAE,kGAAkG;IACnH,aAAa,EAAE,kDAAkD;IACjE,aAAa,EAAE,qHAAqH;IACpI,cAAc,EAAE,gGAAgG;IAChH,iBAAiB,EAAE,+FAA+F;IAClH,mBAAmB,EAAE,8DAA8D;IACnF,cAAc,EAAE,0GAA0G;IAC1H,WAAW,EAAE,iGAAiG;IAC9G,QAAQ,EAAE,qGAAqG;IAC/G,kBAAkB,EAAE,iGAAiG;IACrH,MAAM,EAAE,uIAAuI;IAC/I,qBAAqB,EAAE,kGAAkG;IACzH,gBAAgB,EAAE,sFAAsF;IACxG,qBAAqB,EAAE,gGAAgG;IACvH,UAAU,EAAE,mEAAmE;IAC/E,cAAc,EAAE,0FAA0F;IAC1G,WAAW,EAAE,4HAA4H;IACzI,WAAW,EAAE,qGAAqG;IAClH,OAAO,EAAE,6FAA6F;IACtG,aAAa,EAAE,+GAA+G;IAC9H,YAAY,EAAE,gUAAgU;CACtU,CAAC;AAGX,MAAM,CAAC,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAgB,CAAC;AAC5D,MAAM,eAAe,GAAG,CAAC,CAAC,IAAI,CAAC,UAAyC,CAAC,CAAC;AAE1E,MAAM,UAAU,WAAW,CAAC,KAAa;IACvC,OAAO,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;AACrC,CAAC;AAED,MAAM,UAAU,OAAO,CAAC,IAAe;IACrC,OAAO,KAAK,CAAC,IAAI,CAAC,CAAC;AACrB,CAAC;AAED,8EAA8E;AAC9E,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,CAAC,MAAM,CAAC;IACzC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,aAAa,EAAE,aAAa,EAAE,SAAS,CAAC,CAAC;IACvE,IAAI,EAAE,eAAe,CAAC,QAAQ,EAAE;IAChC,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE;IACrC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,OAAO,EAAE,YAAY,CAAC,CAAC,CAAC,QAAQ,EAAE;CACzD,CAAC,CAAC,MAAM,EAAE,CAAC;AAGZ,MAAM,UAAU,cAAc,CAAC,MAAoB;IACjD,IAAI,MAAM,CAAC,OAAO,KAAK,WAAW,EAAE,CAAC;QACnC,OAAO,CAAC,4EAA4E,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAClK,CAAC;IACD,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IAC7E,OAAO,CAAC,UAAU,MAAM,CAAC,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAChH,CAAC;AAED,MAAM,OAAO,gBAAiB,SAAQ,KAAK;IAChC,IAAI,CAAY;IAEzB,YAAY,IAAe,EAAE,OAAe;QAC1C,KAAK,CAAC,GAAG,OAAO,KAAK,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACtC,IAAI,CAAC,IAAI,GAAG,kBAAkB,CAAC;QAC/B,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;IACnB,CAAC;CACF"}

package/dist/contract/progress.d.ts

@@ -0,0 +1,27 @@
+import type { AppTree } from '@moxxy/jev';
+/** What the model can see of an app: its tree and its screenshot. Pixels count, so canvas work is progress. */
+export declare function fingerprint(tree: AppTree, image?: {
+    base64: string;
+}): string;
+type Look = {
+    readonly tree: AppTree;
+    readonly screenshot?: {
+        readonly base64: string;
+    };
+};
+/**
+ * Whether the window changed between two looks: by elements and picture when both have a picture, else by the
+ * elements alone, without their places (which come with a picture).
+ */
+export declare function looksDifferent(a: Look, b: Look): boolean;
+/** Each app's last action, and how many times in a row it left the app looking the same. */
+export declare class ProgressTracker {
+    private readonly last;
+    /** The same action a third time after two that changed nothing is refused before it is sent. */
+    check(app: string, signature: string): void;
+    /** Returns how many times in a row `signature` has now left `app` unchanged (0 after a change). */
+    record(app: string, signature: string, changed: boolean): number;
+    forget(app: string): void;
+}
+export {};
+//# sourceMappingURL=progress.d.ts.map

package/dist/contract/progress.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"progress.d.ts","sourceRoot":"","sources":["../../src/contract/progress.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAE1C,+GAA+G;AAC/G,wBAAgB,WAAW,CAAC,IAAI,EAAE,OAAO,EAAE,KAAK,CAAC,EAAE;IAAE,MAAM,EAAE,MAAM,CAAA;CAAE,GAAG,MAAM,CAE7E;AAED,KAAK,IAAI,GAAG;IAAE,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC;IAAC,QAAQ,CAAC,UAAU,CAAC,EAAE;QAAE,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAA;CAAE,CAAC;AAE1F;;;GAGG;AACH,wBAAgB,cAAc,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,IAAI,GAAG,OAAO,CAIxD;AAED,4FAA4F;AAC5F,qBAAa,eAAe;IAC1B,OAAO,CAAC,QAAQ,CAAC,IAAI,CAA+D;IAEpF,gGAAgG;IAChG,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,IAAI;IAO3C,mGAAmG;IACnG,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,GAAG,MAAM;IAOhE,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI;CAC1B"}

package/dist/contract/progress.js

@@ -0,0 +1,36 @@
+import { createHash } from 'node:crypto';
+import { ComputerUseError } from './outcome.js';
+/** What the model can see of an app: its tree and its screenshot. Pixels count, so canvas work is progress. */
+export function fingerprint(tree, image) {
+    return createHash('sha256').update(JSON.stringify(tree)).update('\0').update(image ? image.base64 : '-').digest('hex');
+}
+/**
+ * Whether the window changed between two looks: by elements and picture when both have a picture, else by the
+ * elements alone, without their places (which come with a picture).
+ */
+export function looksDifferent(a, b) {
+    if (a.screenshot && b.screenshot)
+        return fingerprint(a.tree, a.screenshot) !== fingerprint(b.tree, b.screenshot);
+    const placeless = (tree) => ({ ...tree, elements: tree.elements.map(({ frame: _, ...element }) => element) });
+    return fingerprint(placeless(a.tree)) !== fingerprint(placeless(b.tree));
+}
+/** Each app's last action, and how many times in a row it left the app looking the same. */
+export class ProgressTracker {
+    last = new Map();
+    /** The same action a third time after two that changed nothing is refused before it is sent. */
+    check(app, signature) {
+        const last = this.last.get(app);
+        if (last && last.signature === signature && last.unchanged >= 2) {
+            throw new ComputerUseError('no_progress', 'This exact action already left the app unchanged twice and was not sent again');
+        }
+    }
+    /** Returns how many times in a row `signature` has now left `app` unchanged (0 after a change). */
+    record(app, signature, changed) {
+        const last = this.last.get(app);
+        const unchanged = changed ? 0 : (last?.signature === signature ? last.unchanged : 0) + 1;
+        this.last.set(app, { signature, unchanged });
+        return unchanged;
+    }
+    forget(app) { this.last.delete(app); }
+}
+//# sourceMappingURL=progress.js.map

package/dist/contract/progress.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"progress.js","sourceRoot":"","sources":["../../src/contract/progress.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAGhD,+GAA+G;AAC/G,MAAM,UAAU,WAAW,CAAC,IAAa,EAAE,KAA0B;IACnE,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AACzH,CAAC;AAID;;;GAGG;AACH,MAAM,UAAU,cAAc,CAAC,CAAO,EAAE,CAAO;IAC7C,IAAI,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC,UAAU;QAAE,OAAO,WAAW,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,UAAU,CAAC,KAAK,WAAW,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,UAAU,CAAC,CAAC;IACjH,MAAM,SAAS,GAAG,CAAC,IAAa,EAAW,EAAE,CAAC,CAAC,EAAE,GAAG,IAAI,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,OAAO,EAAE,EAAE,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAChI,OAAO,WAAW,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,WAAW,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;AAC3E,CAAC;AAED,4FAA4F;AAC5F,MAAM,OAAO,eAAe;IACT,IAAI,GAAG,IAAI,GAAG,EAAoD,CAAC;IAEpF,gGAAgG;IAChG,KAAK,CAAC,GAAW,EAAE,SAAiB;QAClC,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAChC,IAAI,IAAI,IAAI,IAAI,CAAC,SAAS,KAAK,SAAS,IAAI,IAAI,CAAC,SAAS,IAAI,CAAC,EAAE,CAAC;YAChE,MAAM,IAAI,gBAAgB,CAAC,aAAa,EAAE,+EAA+E,CAAC,CAAC;QAC7H,CAAC;IACH,CAAC;IAED,mGAAmG;IACnG,MAAM,CAAC,GAAW,EAAE,SAAiB,EAAE,OAAgB;QACrD,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAChC,MAAM,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QACzF,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC,CAAC;QAC7C,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,MAAM,CAAC,GAAW,IAAU,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;CACrD"}

package/dist/contract/tools.d.ts

@@ -0,0 +1,338 @@
+import { pointsAtFocus, wordsIn } from '@moxxy/jev';
+import { z } from 'zod';
+type TargetInput = {
+    element_index?: number;
+    x?: number;
+    y?: number;
+};
+export type Target = {
+    kind: 'element';
+    index: number;
+} | {
+    kind: 'point';
+    x: number;
+    y: number;
+} | {
+    kind: 'focused';
+};
+export declare function resolveTarget(input: TargetInput): Target;
+type TargetRule = 'required' | 'optional' | undefined;
+interface ActionDef<S extends z.ZodRawShape> {
+    readonly shape: S;
+    readonly target: TargetRule;
+}
+/** Actions on one app, without the `app` field. */
+declare const actions: {
+    click: ActionDef<{
+        element_index: z.ZodOptional<z.ZodNumber>;
+        x: z.ZodOptional<z.ZodNumber>;
+        y: z.ZodOptional<z.ZodNumber>;
+    } & {
+        mouse_button: z.ZodDefault<z.ZodEnum<["left", "right", "middle"]>>;
+        click_count: z.ZodDefault<z.ZodNumber>;
+        modifiers: z.ZodOptional<z.ZodEffects<z.ZodString, string, string>>;
+    }>;
+    type_text: ActionDef<{
+        element_index: z.ZodOptional<z.ZodNumber>;
+        text: z.ZodString;
+    }>;
+    press_key: ActionDef<{
+        key: z.ZodEffects<z.ZodString, string, string>;
+        repeat: z.ZodDefault<z.ZodNumber>;
+    }>;
+    scroll: ActionDef<{
+        element_index: z.ZodOptional<z.ZodNumber>;
+        x: z.ZodOptional<z.ZodNumber>;
+        y: z.ZodOptional<z.ZodNumber>;
+    } & {
+        direction: z.ZodEnum<["up", "down", "left", "right"]>;
+        pages: z.ZodDefault<z.ZodNumber>;
+    }>;
+    drag: ActionDef<{
+        from_x: z.ZodNumber;
+        from_y: z.ZodNumber;
+        to_x: z.ZodNumber;
+        to_y: z.ZodNumber;
+    }>;
+    set_value: ActionDef<{
+        element_index: z.ZodNumber;
+        value: z.ZodString;
+    }>;
+    perform_secondary_action: ActionDef<{
+        element_index: z.ZodNumber;
+        secondary_action: z.ZodString;
+    }>;
+};
+type ActionName = keyof typeof actions;
+type ActionOutput<N extends ActionName> = z.output<z.ZodObject<(typeof actions)[N]['shape'], 'strict'>>;
+export declare const actionNames: ActionName[];
+/** One action as the backend hands it on: its name and its parsed fields. */
+export type ComputerAction = {
+    [N in ActionName]: {
+        action: N;
+    } & ActionOutput<N>;
+}[ActionName];
+/** One step of computer_run: what to do, the element in words, and what must show afterwards. */
+export declare const runStepShape: z.ZodObject<{
+    do: z.ZodEnum<["click", "type", "set_value", "key", "scroll"]>;
+    target: z.ZodOptional<z.ZodString>;
+    text: z.ZodOptional<z.ZodString>;
+    key: z.ZodOptional<z.ZodEffects<z.ZodString, string, string>>;
+    direction: z.ZodOptional<z.ZodEnum<["up", "down", "left", "right"]>>;
+    expect: z.ZodOptional<z.ZodString>;
+}, "strict", z.ZodTypeAny, {
+    do: "type" | "key" | "click" | "scroll" | "set_value";
+    key?: string | undefined;
+    text?: string | undefined;
+    direction?: "up" | "down" | "left" | "right" | undefined;
+    target?: string | undefined;
+    expect?: string | undefined;
+}, {
+    do: "type" | "key" | "click" | "scroll" | "set_value";
+    key?: string | undefined;
+    text?: string | undefined;
+    direction?: "up" | "down" | "left" | "right" | undefined;
+    target?: string | undefined;
+    expect?: string | undefined;
+}>;
+export type RunStep = z.infer<typeof runStepShape>;
+export { pointsAtFocus, wordsIn };
+export interface ComputerToolSpec<I = unknown> {
+    readonly description: string;
+    readonly input: z.ZodType<I, z.ZodTypeDef, unknown>;
+}
+export declare const computerTools: {
+    computer_status: {
+        description: string;
+        input: z.ZodObject<{
+            open_settings: z.ZodOptional<z.ZodEnum<["accessibility", "screen_recording"]>>;
+        }, "strict", z.ZodTypeAny, {
+            open_settings?: "accessibility" | "screen_recording" | undefined;
+        }, {
+            open_settings?: "accessibility" | "screen_recording" | undefined;
+        }>;
+    };
+    computer_list_apps: {
+        description: string;
+        input: z.ZodObject<{
+            query: z.ZodOptional<z.ZodString>;
+            limit: z.ZodDefault<z.ZodNumber>;
+        }, "strict", z.ZodTypeAny, {
+            limit: number;
+            query?: string | undefined;
+        }, {
+            query?: string | undefined;
+            limit?: number | undefined;
+        }>;
+    };
+    computer_request_access: {
+        description: string;
+        input: z.ZodEffects<z.ZodObject<{
+            apps: z.ZodArray<z.ZodString, "many">;
+            reason: z.ZodString;
+            clipboard_read: z.ZodOptional<z.ZodBoolean>;
+            clipboard_write: z.ZodOptional<z.ZodBoolean>;
+            system_key_combos: z.ZodOptional<z.ZodBoolean>;
+            full_access: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+        }, "strict", z.ZodTypeAny, {
+            apps: string[];
+            reason: string;
+            clipboard_read?: boolean | undefined;
+            clipboard_write?: boolean | undefined;
+            system_key_combos?: boolean | undefined;
+            full_access?: string[] | undefined;
+        }, {
+            apps: string[];
+            reason: string;
+            clipboard_read?: boolean | undefined;
+            clipboard_write?: boolean | undefined;
+            system_key_combos?: boolean | undefined;
+            full_access?: string[] | undefined;
+        }>, {
+            apps: string[];
+            reason: string;
+            clipboard_read?: boolean | undefined;
+            clipboard_write?: boolean | undefined;
+            system_key_combos?: boolean | undefined;
+            full_access?: string[] | undefined;
+        }, {
+            apps: string[];
+            reason: string;
+            clipboard_read?: boolean | undefined;
+            clipboard_write?: boolean | undefined;
+            system_key_combos?: boolean | undefined;
+            full_access?: string[] | undefined;
+        }>;
+    };
+    computer_get_app_state: {
+        description: string;
+        input: z.ZodEffects<z.ZodObject<{
+            app: z.ZodString;
+            window_id: z.ZodOptional<z.ZodString>;
+            disable_diff: z.ZodDefault<z.ZodBoolean>;
+        }, "strict", z.ZodTypeAny, {
+            app: string;
+            disable_diff: boolean;
+            window_id?: string | undefined;
+        }, {
+            app: string;
+            window_id?: string | undefined;
+            disable_diff?: boolean | undefined;
+        }>, {
+            app: string;
+            disable_diff: boolean;
+            window_id?: string | undefined;
+        }, unknown>;
+    };
+    computer_click: {
+        description: string;
+        input: z.ZodType<{
+            mouse_button: "left" | "right" | "middle";
+            click_count: number;
+            x?: number | undefined;
+            y?: number | undefined;
+            modifiers?: string | undefined;
+            element_index?: number | undefined;
+        } & {
+            app: string;
+        }, z.ZodTypeDef, unknown>;
+    };
+    computer_type_text: {
+        description: string;
+        input: z.ZodType<{
+            text: string;
+            element_index?: number | undefined;
+        } & {
+            app: string;
+        }, z.ZodTypeDef, unknown>;
+    };
+    computer_press_key: {
+        description: string;
+        input: z.ZodType<{
+            key: string;
+            repeat: number;
+        } & {
+            app: string;
+        }, z.ZodTypeDef, unknown>;
+    };
+    computer_scroll: {
+        description: string;
+        input: z.ZodType<{
+            direction: "up" | "down" | "left" | "right";
+            pages: number;
+            x?: number | undefined;
+            y?: number | undefined;
+            element_index?: number | undefined;
+        } & {
+            app: string;
+        }, z.ZodTypeDef, unknown>;
+    };
+    computer_drag: {
+        description: string;
+        input: z.ZodType<{
+            from_x: number;
+            from_y: number;
+            to_x: number;
+            to_y: number;
+        } & {
+            app: string;
+        }, z.ZodTypeDef, unknown>;
+    };
+    computer_set_value: {
+        description: string;
+        input: z.ZodType<{
+            value: string;
+            element_index: number;
+        } & {
+            app: string;
+        }, z.ZodTypeDef, unknown>;
+    };
+    computer_perform_secondary_action: {
+        description: string;
+        input: z.ZodType<{
+            element_index: number;
+            secondary_action: string;
+        } & {
+            app: string;
+        }, z.ZodTypeDef, unknown>;
+    };
+    computer_run: {
+        description: string;
+        input: z.ZodObject<{
+            app: z.ZodString;
+            goal: z.ZodString;
+            steps: z.ZodArray<z.ZodEffects<z.ZodEffects<z.ZodObject<{
+                do: z.ZodEnum<["click", "type", "set_value", "key", "scroll"]>;
+                target: z.ZodOptional<z.ZodString>;
+                text: z.ZodOptional<z.ZodString>;
+                key: z.ZodOptional<z.ZodEffects<z.ZodString, string, string>>;
+                direction: z.ZodOptional<z.ZodEnum<["up", "down", "left", "right"]>>;
+                expect: z.ZodOptional<z.ZodString>;
+            }, "strict", z.ZodTypeAny, {
+                do: "type" | "key" | "click" | "scroll" | "set_value";
+                key?: string | undefined;
+                text?: string | undefined;
+                direction?: "up" | "down" | "left" | "right" | undefined;
+                target?: string | undefined;
+                expect?: string | undefined;
+            }, {
+                do: "type" | "key" | "click" | "scroll" | "set_value";
+                key?: string | undefined;
+                text?: string | undefined;
+                direction?: "up" | "down" | "left" | "right" | undefined;
+                target?: string | undefined;
+                expect?: string | undefined;
+            }>, {
+                do: "type" | "key" | "click" | "scroll" | "set_value";
+                key?: string | undefined;
+                text?: string | undefined;
+                direction?: "up" | "down" | "left" | "right" | undefined;
+                target?: string | undefined;
+                expect?: string | undefined;
+            }, {
+                do: "type" | "key" | "click" | "scroll" | "set_value";
+                key?: string | undefined;
+                text?: string | undefined;
+                direction?: "up" | "down" | "left" | "right" | undefined;
+                target?: string | undefined;
+                expect?: string | undefined;
+            }>, {
+                do: "type" | "key" | "click" | "scroll" | "set_value";
+                key?: string | undefined;
+                text?: string | undefined;
+                direction?: "up" | "down" | "left" | "right" | undefined;
+                target?: string | undefined;
+                expect?: string | undefined;
+            }, unknown>, "many">;
+        }, "strict", z.ZodTypeAny, {
+            app: string;
+            goal: string;
+            steps: {
+                do: "type" | "key" | "click" | "scroll" | "set_value";
+                key?: string | undefined;
+                text?: string | undefined;
+                direction?: "up" | "down" | "left" | "right" | undefined;
+                target?: string | undefined;
+                expect?: string | undefined;
+            }[];
+        }, {
+            app: string;
+            goal: string;
+            steps: unknown[];
+        }>;
+    };
+    computer_zoom: {
+        description: string;
+        input: z.ZodObject<{
+            app: z.ZodString;
+            region: z.ZodEffects<z.ZodArray<z.ZodNumber, "many">, number[], number[]>;
+        }, "strict", z.ZodTypeAny, {
+            app: string;
+            region: number[];
+        }, {
+            app: string;
+            region: number[];
+        }>;
+    };
+};
+//# sourceMappingURL=tools.d.ts.map

package/dist/contract/tools.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tools.d.ts","sourceRoot":"","sources":["../../src/contract/tools.ts"],"names":[],"mappings":"AAAA,OAAO,EAAgB,aAAa,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAClE,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAmCxB,KAAK,WAAW,GAAG;IAAE,aAAa,CAAC,EAAE,MAAM,CAAC;IAAC,CAAC,CAAC,EAAE,MAAM,CAAC;IAAC,CAAC,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC;AACtE,MAAM,MAAM,MAAM,GAAG;IAAE,IAAI,EAAE,SAAS,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,GAAG;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,CAAC,EAAE,MAAM,CAAC;IAAC,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG;IAAE,IAAI,EAAE,SAAS,CAAA;CAAE,CAAC;AAExH,wBAAgB,aAAa,CAAC,KAAK,EAAE,WAAW,GAAG,MAAM,CAIxD;AAeD,KAAK,UAAU,GAAG,UAAU,GAAG,UAAU,GAAG,SAAS,CAAC;AACtD,UAAU,SAAS,CAAC,CAAC,SAAS,CAAC,CAAC,WAAW;IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,UAAU,CAAA;CAAE;AAyB/F,mDAAmD;AACnD,QAAA,MAAM,OAAO;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAWZ,CAAC;AAEF,KAAK,UAAU,GAAG,MAAM,OAAO,OAAO,CAAC;AACvC,KAAK,YAAY,CAAC,CAAC,SAAS,UAAU,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,OAAO,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAC;AACxG,eAAO,MAAM,WAAW,EAA2B,UAAU,EAAE,CAAC;AAChE,6EAA6E;AAC7E,MAAM,MAAM,cAAc,GAAG;KAAG,CAAC,IAAI,UAAU,GAAG;QAAE,MAAM,EAAE,CAAC,CAAA;KAAE,GAAG,YAAY,CAAC,CAAC,CAAC;CAAE,CAAC,UAAU,CAAC,CAAC;AAMhG,iGAAiG;AACjG,eAAO,MAAM,YAAY;;;;;;;;;;;;;;;;;;;;;EASd,CAAC;AACZ,MAAM,MAAM,OAAO,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,YAAY,CAAC,CAAC;AAWnD,OAAO,EAAE,aAAa,EAAE,OAAO,EAAE,CAAC;AA2BlC,MAAM,WAAW,gBAAgB,CAAC,CAAC,GAAG,OAAO;IAAI,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC,CAAC,UAAU,EAAE,OAAO,CAAC,CAAA;CAAE;AAIpI,eAAO,MAAM,aAAa;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAvDsF,MAAM;;;;;;;;;iBAAN,MAAM;;;;;;;;;iBAAN,MAAM;;;;;;;;;;;;iBAAN,MAAM;;;;;;;;;;;iBAAN,MAAM;;;;;;;;;iBAAN,MAAM;;;;;;;;;iBAAN,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAyG1E,CAAC"}