ai-remote 0.6.0 → 0.6.1

package/dist/{cli-window-AWXZ5NJX.mjs → cli-window-G5ESQTF5.mjs}

@@ -4,8 +4,8 @@ import {
   openWindow,
   readCurrentDisplaySize,
   runWindow
-} from "./cli-chunk-P4645KKB.mjs";
-import "./cli-chunk-TKKENOSJ.mjs";
+} from "./cli-chunk-CD47NHZT.mjs";
+import "./cli-chunk-MURUADW4.mjs";
 export {
   detectCurrentDisplaySize,
   nativeWindowAvailable,

package/dist/cli.mjs

@@ -13,7 +13,7 @@ import {
   metaPath,
   sessionName,
   socketPath
-} from "./cli-chunk-TKKENOSJ.mjs";
+} from "./cli-chunk-MURUADW4.mjs";
 
 // src/cli/cli.ts
 import { resolve as resolvePath } from "node:path";
@@ -62,7 +62,14 @@ var VALUE_FLAGS = /* @__PURE__ */ new Set([
   "vnc-port",
   "port",
   "keys",
-  "region"
+  "region",
+  // How an input is performed rather than where: how long a button or chord is
+  // held, how many times, what is held down with it, and the floor between one
+  // action and the next.
+  "hold",
+  "count",
+  "with",
+  "pace"
 ]);
 var OPTIONAL_VALUE_FLAGS = { shot: /\.(png|jpe?g)$/i };
 function parse(argv) {
@@ -105,6 +112,22 @@ var shotFile = (args) => {
   if (value === void 0) return void 0;
   return typeof value === "string" && value || flag(args, "o", "out") || "screen.png";
 };
+function clickAction(args) {
+  const spec = args.rest[0];
+  if (!spec) throw new CliError("click needs a position, e.g. click 640,400", 12);
+  const [x, y] = spec.split(",").map(Number);
+  return {
+    type: "click",
+    x,
+    y,
+    mouse_button: flag(args, "button") ?? "left",
+    count: has(args, "double") ? 2 : Number(flag(args, "count") ?? 1),
+    intervalMs: Number(flag(args, "pace") ?? 0),
+    duration: Number(flag(args, "hold") ?? 0),
+    ...flag(args, "with") ? { key: flag(args, "with") } : {}
+  };
+}
+var audioEnabled = (args, mode) => mode === "rdp" && !has(args, "no-view", "headless", "no-audio");
 
 // src/cli/prompt.ts
 function canPrompt() {
@@ -250,6 +273,7 @@ The rest drive that session, and take no host:
 
   ${NAME} shot                     screenshot the desktop
   ${NAME} click X,Y                click, --button right|middle, --double
+                                   --hold MS, --count N, --with Shift
   ${NAME} type  "text"             type into whatever has focus
   ${NAME} key   <Code>             press keys, e.g. MetaLeft, ControlLeft+KeyA
   ${NAME} exec  "cmd"              run a command in the terminal (SSH)
@@ -261,6 +285,7 @@ The rest drive that session, and take no host:
   ${NAME} batch actions.json       structured actions; - reads JSON from stdin
   ${NAME} vnc                      give this session a screen over VNC
                                    (--vnc-port N, default 5900)
+  ${NAME} input off                take the pointer from the window (on gives it back)
   ${NAME} view                     open the window on the session
   ${NAME} view --watch-only        a window that watches rather than drives
   ${NAME} view --close             close the window, session keeps running
@@ -307,7 +332,7 @@ Options for open
   -H, --height N         desktop height                (default: 800)
       --fullscreen       fill the display; RDP uses its current resolution
       --side-panel       open the SSH terminal docked on the right
-      --audio            play the remote computer's sound in the window (RDP)
+      --no-audio         disable remote sound (enabled by default for windowed RDP)
       --no-clipboard     do not share this machine's clipboard (shared by default)
       --insecure-host-key
                          accept an SSH host key that does not match the one on
@@ -342,6 +367,29 @@ Options for the rest
       --region X,Y,W,H   capture a desktop region; --json reports its origin
       --json             machine-readable result on stdout
 
+Performing an input rather than merely sending it
+      --hold MS          for click and key: how long the button or chord stays
+                         down. A press and a release with nothing between is
+                         not a held key, and a control that measures how long
+                         it was held cannot be worked without this
+      --count N          for click: repeat it, --pace between repeats
+      --with CHORD       for click: a modifier held for that one action,
+                         e.g. --with ShiftLeft
+      --animate          for click, key and batch: draw the pointer travelling
+                         to each target in the window, and hold the input until
+                         it lands. Slower on purpose, and does nothing at all
+                         with no window open
+      --pace MS          a floor between one action and the next. ChatGPT's
+                         computer use waits 100ms after every action; this
+                         waits none unless asked, because a batch exists to
+                         spend one turn on many inputs
+      --exclusive        take the pointer from the window for the length of
+                         this command and give it back after. A desktop has one
+                         pointer: a hand that moves it mid-batch moves what the
+                         agent is aiming at. \`${NAME} input off\` holds it
+                         open instead, and either way the window's control
+                         button takes it back
+
 Signing in
   A terminal signs in with a key where it can: with no --identity, ssh-agent is
   asked for what it is holding and ~/.ssh/id_ed25519, id_ecdsa and id_rsa are
@@ -473,7 +521,7 @@ async function startSession(args, name, host, port, secret, account = "") {
   const explicitHeight = flag(args, "H", "height");
   let fullscreenSize = null;
   if (mode === "rdp" && has(args, "fullscreen") && (!explicitWidth || !explicitHeight)) {
-    const { detectCurrentDisplaySize } = await import("./cli-window-AWXZ5NJX.mjs");
+    const { detectCurrentDisplaySize } = await import("./cli-window-G5ESQTF5.mjs");
     const display = await detectCurrentDisplaySize();
     if (display) fullscreenSize = fullscreenRdpSize(display, has(args, "side-panel"));
   }
@@ -523,13 +571,14 @@ async function startSession(args, name, host, port, secret, account = "") {
   if (has(args, "tab")) daemonArgs.push("--tab");
   if (has(args, "fullscreen")) daemonArgs.push("--fullscreen");
   if (has(args, "side-panel") && mode !== "ssh") daemonArgs.push("--side-panel");
-  if (has(args, "audio", "sound")) {
+  if (has(args, "no-audio")) daemonArgs.push("--no-audio");
+  if (audioEnabled(args, mode) || has(args, "audio", "sound")) {
     if (mode !== "rdp") {
       process.stderr.write(`--audio is RDP only; a ${mode.toUpperCase()} session has no audio channel.
 `);
     } else if (has(args, "no-view", "headless")) {
       process.stderr.write("--audio needs a window to play into; ignoring it for a headless session.\n");
-    } else {
+    } else if (audioEnabled(args, mode)) {
       daemonArgs.push("--audio");
     }
   }
@@ -597,7 +646,7 @@ async function main() {
   }
   const json = has(args, "json");
   if (args.command === "__display-size") {
-    const { readCurrentDisplaySize } = await import("./cli-window-AWXZ5NJX.mjs");
+    const { readCurrentDisplaySize } = await import("./cli-window-G5ESQTF5.mjs");
     const size = await readCurrentDisplaySize();
     if (size) process.stdout.write(`${JSON.stringify(size)}
 `);
@@ -606,7 +655,7 @@ async function main() {
   if (args.command === "__session") {
     const mode = modeFor(args, args.rest[0]);
     const { host, port } = splitTarget(args.rest[0] ?? "", defaultPort(mode));
-    const { runDaemon } = await import("./cli-daemon-3L36TBIB.mjs");
+    const { runDaemon } = await import("./cli-daemon-SD2AS34J.mjs");
     await runDaemon({
       name: sessionName(host, port, flag(args, "name", "session")),
       host,
@@ -631,7 +680,7 @@ async function main() {
       watchOnly: has(args, "watch-only", "observe"),
       // RDP is the only protocol here with an audio channel, and a session with
       // no window has nowhere to play what the channel would carry.
-      audio: has(args, "audio", "sound") && mode === "rdp" && !has(args, "no-view", "headless"),
+      audio: audioEnabled(args, mode),
       idleMs: Math.max(0, Number(flag(args, "idle") ?? 0)) * 6e4,
       keys: flag(args, "keys") === "literal" ? "literal" : "adapt",
       clipboard: !has(args, "no-clipboard"),
@@ -640,7 +689,7 @@ async function main() {
     return;
   }
   if (args.command === "__window") {
-    const { runWindow } = await import("./cli-window-AWXZ5NJX.mjs");
+    const { runWindow } = await import("./cli-window-G5ESQTF5.mjs");
     await runWindow(process.argv.slice(3));
     return;
   }
@@ -984,6 +1033,12 @@ Open the host again without --ssh for a desktop.`,
     region: flag(args, "region"),
     maxEdge: Number(flag(args, "max-edge") ?? 0)
   };
+  const pacing = () => ({
+    animate: has(args, "animate"),
+    postActionSleepMs: Number(flag(args, "pace") ?? 0),
+    // Hold the pointer for the length of this command and give it back after.
+    exclusive: has(args, "exclusive")
+  });
   switch (args.command) {
     case "status": {
       Object.assign(summary, await tell(session, "status", {}, 1e4));
@@ -1012,8 +1067,11 @@ Open the host again without --ssh for a desktop.`,
       if (!spec) throw new CliError(`click needs a position, e.g. \`${NAME} click 640,400\``, 12);
       const [x, y] = spec.split(",").map(Number);
       const button = flag(args, "button") ?? "left";
-      const step = `click ${x},${y}${button === "left" ? "" : `,${button}`}`;
-      Object.assign(summary, await runSteps(has(args, "double") ? `${step}; ${step}` : step));
+      Object.assign(summary, await tell(session, "batch", {
+        actions: [clickAction(args)],
+        ...pacing(),
+        ...observation(shotFile(args))
+      }));
       Object.assign(summary, { clicked: { x, y, button } });
       break;
     }
@@ -1023,6 +1081,7 @@ Open the host again without --ssh for a desktop.`,
       if (!text) throw new CliError("type needs some text", 12);
       Object.assign(summary, await tell(session, "batch", {
         actions: [{ type: "type", text }],
+        ...pacing(),
         ...observation(shotFile(args))
       }));
       Object.assign(summary, { typed: text.length });
@@ -1030,7 +1089,12 @@ Open the host again without --ssh for a desktop.`,
     }
     case "key": {
       if (!args.rest.length) throw new CliError("key needs at least one code, e.g. MetaLeft", 12);
-      Object.assign(summary, await runSteps(args.rest.map((code) => `key ${code}`).join("; ")));
+      const duration = Number(flag(args, "hold") ?? 0);
+      Object.assign(summary, await tell(session, "batch", {
+        actions: args.rest.map((chord) => ({ type: "key", chord, duration })),
+        ...pacing(),
+        ...observation(shotFile(args))
+      }));
       Object.assign(summary, { keys: args.rest });
       break;
     }
@@ -1051,7 +1115,27 @@ Open the host again without --ssh for a desktop.`,
       } catch (error) {
         throw new CliError(`${file === "-" ? "stdin" : file} is not valid JSON: ${error.message}`, 12);
       }
-      Object.assign(summary, await tell(session, "batch", { actions, ...observation(shotFile(args)) }));
+      Object.assign(summary, await tell(session, "batch", {
+        actions,
+        ...pacing(),
+        ...observation(shotFile(args))
+      }));
+      break;
+    }
+    /**
+     * Who may drive: the people watching, or nobody but the agent.
+     *
+     * A remote desktop has one pointer, so an agent working while a hand rests
+     * on the trackpad is two things pulling at the same cursor. `off` takes it
+     * away from the windows; the button in the window still takes it back, and
+     * the next batch is told that it happened.
+     */
+    case "input": {
+      const word = args.rest[0];
+      if (word !== void 0 && word !== "on" && word !== "off") {
+        throw new CliError(`input takes on or off, e.g. \`${NAME} input off\``, 12);
+      }
+      Object.assign(summary, await tell(session, "input", word === void 0 ? {} : { on: word === "on" }));
       break;
     }
     case "exec": {
@@ -1370,6 +1454,14 @@ The password is saved in ${summary.saved}.` : ""
   if (command === "type") return `Typed ${summary.typed} characters.${looked}`;
   if (command === "batch") return `Ran ${summary.actions} actions (${summary.clicks} clicks) in ${summary.dispatchMs} ms of dispatch.${looked}`;
   if (command === "key") return `Pressed ${summary.keys.join(", ")}.${looked}`;
+  if (command === "input") {
+    const count = Number(summary.viewers ?? 0);
+    const windows = `${count} window${count === 1 ? "" : "s"}`;
+    if (summary.humanInput) {
+      return count ? `The keyboard and pointer are shared: ${windows} may drive.` : "The keyboard and pointer are shared with whoever is watching. No window is open.";
+    }
+    return count ? `The agent has the keyboard and pointer; ${windows} can only watch. The control button in the window takes them back.` : "The agent has the keyboard and pointer. No window is open, and one that opens will start watching.";
+  }
   if (command === "do") return `Ran ${summary.steps.length} steps: ${summary.steps.join(" \u2192 ")}${looked}`;
   return "Done.";
 }

package/dist/computer.d.ts

@@ -2,34 +2,102 @@ export type ComputerPoint = {
     x: number;
     y: number;
 };
+/** The wire's 0/1/2, or the name for it. */
+export type ComputerButton = 0 | 1 | 2 | 'left' | 'middle' | 'right' | 'l' | 'm' | 'r';
+/** A modifier chord held for the length of one action, e.g. `ShiftLeft`. */
+type Modified = {
+    key?: string;
+};
 export type ComputerAction = ({
     type: 'click';
-    button?: number;
+    button?: ComputerButton;
+    /** ChatGPT's spelling of `button`. */
+    mouse_button?: ComputerButton;
     count?: number;
+    /** ChatGPT's spelling of `count`. */
+    click_count?: number;
     intervalMs?: number;
-} & ComputerPoint) | ({
+    /** Milliseconds to hold the button down for each click. */
+    duration?: number;
+} & ComputerPoint & Modified) | ({
     type: 'move';
-} & ComputerPoint) | ({
+} & ComputerPoint & Modified) | ({
     type: 'scroll';
+    /** Wheel notches, positive right and down. */
     dx?: number;
     dy?: number;
-} & ComputerPoint) | {
+    /** ChatGPT's spelling: a direction and a distance in pixels. */
+    direction?: 'up' | 'down' | 'left' | 'right' | 'u' | 'd' | 'l' | 'r';
+    pixels?: number;
+} & ComputerPoint & Modified) | {
     type: 'type';
     text: string;
     chunkSize?: number;
     intervalMs?: number;
 } | {
     type: 'key';
-    chord: string;
+    chord?: string;
+    /** ChatGPT's spelling of `chord`. */
+    key?: string;
+    /** Milliseconds to hold the chord before releasing it. */
+    duration?: number;
 } | {
     type: 'wait';
     ms: number;
-} | {
+} | ({
     type: 'drag';
     path: ComputerPoint[];
-    button?: number;
+    button?: ComputerButton;
     intervalMs?: number;
+} & Modified)
+/**
+ * A drag held open across calls, which is ChatGPT's `drag_handle()`.
+ *
+ * `drag` is one gesture with a known shape. These three are for the drag
+ * whose ending depends on what the drag itself reveals: press, take a
+ * screenshot, decide, and only then release. The button stays down on the
+ * host between calls, so something has to end it -- the session lets go if it
+ * closes first, but nothing else will.
+ */
+ | ({
+    type: 'drag-start';
+    button?: ComputerButton;
+} & ComputerPoint) | ({
+    type: 'drag-move';
+} & ComputerPoint) | {
+    type: 'drag-end';
 };
+export interface BatchOptions {
+    /**
+     * Wait for the drawn cursor to reach each target before the input goes out.
+     *
+     * Only means anything while a viewer is attached; there is no cursor to watch
+     * otherwise and nothing to wait for. It makes a batch slower on purpose, in
+     * exchange for a click that a person watching sees land.
+     */
+    animate?: boolean;
+    /**
+     * A floor between consecutive actions, ChatGPT's `post_action_sleep_ms`.
+     *
+     * Theirs is 100ms and always on. This defaults to none: a batch exists to
+     * spend one model turn on many inputs, and `dispatchMs` is a published
+     * measurement. Ask for it when the far side needs the breathing room.
+     */
+    postActionSleepMs?: number;
+    /**
+     * Take the keyboard and pointer from the viewer for the length of this batch.
+     *
+     * A remote desktop has one pointer. An agent does not get its own -- neither
+     * protocol has a second one to give it -- so a hand resting on a trackpad is
+     * moving the same cursor the agent is aiming, and a click that lands where
+     * the hand left it is worse than one that does not land at all. This holds
+     * the input for the batch and gives it back afterwards.
+     *
+     * Not a lock. The control button in the window takes it back at once, and
+     * `interference` in the next result says that it happened.
+     */
+    exclusive?: boolean;
+}
 export interface ObservationOptions {
     region?: {
         x: number;
@@ -62,6 +130,21 @@ export interface ComputerBatchResult {
     clicks: number;
     /** Input dispatch and requested pacing; not proof of application completion. */
     dispatchMs: number;
+    /** A `drag-start` is still holding a button down on the host. */
+    dragging: boolean;
+    /**
+     * Input events that arrived from a viewer while this batch ran.
+     *
+     * Zero is the ordinary answer. Anything else is somebody using the machine at
+     * the same time, and their pointer is the same pointer: treat the actions in
+     * this batch as having possibly landed somewhere else, look before carrying
+     * on, and consider `exclusive` or `takeInput()` if it keeps happening.
+     * Refused input counts, so this stays a useful signal after the input has
+     * been taken away.
+     */
+    interference: number;
+    /** Whether the people watching may drive right now. */
+    humanInput: boolean;
 }
 /** Map a point in a returned image back to full-desktop coordinates. */
 export declare function desktopPoint(image: ComputerScreenshot, point: ComputerPoint): ComputerPoint;
@@ -71,7 +154,23 @@ export declare class RemoteComputer {
     private call;
     screenshot(options?: ObservationOptions): Promise<ComputerScreenshot>;
     /** One IPC call, no reconnect or model invocation between actions. */
-    act(actions: ComputerAction[], options?: ObservationOptions): Promise<ComputerBatchResult & ComputerScreenshot>;
+    act(actions: ComputerAction[], options?: ObservationOptions & BatchOptions): Promise<ComputerBatchResult & ComputerScreenshot>;
+    /**
+     * Take the keyboard and pointer from every window, until given back.
+     *
+     * For an agent that has seen `interference` and decided to work alone.
+     * `releaseInput()` undoes it, and so does the control button in the window --
+     * which is the point: the person at the machine is never locked out of it.
+     */
+    takeInput(): Promise<{
+        humanInput: boolean;
+        viewers: number;
+    }>;
+    /** Give the keyboard and pointer back to whoever is watching. */
+    releaseInput(): Promise<{
+        humanInput: boolean;
+        viewers: number;
+    }>;
     /** Explicitly skip observation for a known sequence; inspect when needed. */
-    dispatch(actions: ComputerAction[]): Promise<ComputerBatchResult>;
+    dispatch(actions: ComputerAction[], options?: BatchOptions): Promise<ComputerBatchResult>;
 }

package/dist/computer.js

@@ -1,3 +1,3 @@
-import g from"node:net";import{StringDecoder as f}from"node:string_decoder";function w(t,e){let r=new f("utf8"),n="";t.on("data",u=>{for(n+=r.write(u);;){let s=n.indexOf(`
-`);if(s===-1)return;let o=n.slice(0,s);if(n=n.slice(s+1),!!o.trim())try{e(JSON.parse(o))}catch{}}})}var x=(t,e)=>{t.write(`${JSON.stringify(e)}
-`)};function p(t,e,r={},n=12e4){return new Promise((u,s)=>{let o=g.connect(t),a=!1,c=(i,l)=>{a||(a=!0,clearTimeout(b),o.destroy(),i?s(i):u(l))},b=setTimeout(()=>c(new Error(`The session did not answer "${e}" in time.`)),n);o.on("connect",()=>x(o,{id:1,op:e,args:r})),o.on("error",i=>{c(Object.assign(i,{notRunning:i.code==="ENOENT"||i.code==="ECONNREFUSED"}))}),w(o,i=>c(null,i))})}import{homedir as y}from"node:os";import{join as m}from"node:path";var k=m(y(),".ai-remote"),P=m(k,"run");var h=t=>m(P,`${t}.sock`);function T(t,e){if(!Number.isFinite(e.x)||!Number.isFinite(e.y)||e.x<0||e.y<0||e.x>=t.width||e.y>=t.height)throw new Error("Point must be inside the screenshot.");let r=t.viewport??{x:0,y:0,width:t.desktopWidth,height:t.desktopHeight};return{x:Math.min(r.x+r.width-1,Math.floor(r.x+e.x*r.width/t.width)),y:Math.min(r.y+r.height-1,Math.floor(r.y+e.y*r.height/t.height))}}var d=class{session;constructor(e){if(!/^[\w.-]+$/.test(e)||e==="."||e==="..")throw new Error("Use a session name from ai-remote status --json.");this.session=e}async call(e,r){let n=await p(h(this.session),e,r);if(!n.ok)throw new Error(n.error??"Remote operation failed.");return n.result}screenshot(e={}){return this.call("shot",{...e})}act(e,r={}){return this.call("batch",{...r,actions:e})}dispatch(e){return this.call("batch",{actions:e,observe:!1})}};export{d as RemoteComputer,T as desktopPoint};
+import g from"node:net";import{StringDecoder as f}from"node:string_decoder";function w(r,e){let t=new f("utf8"),n="";r.on("data",u=>{for(n+=t.write(u);;){let s=n.indexOf(`
+`);if(s===-1)return;let o=n.slice(0,s);if(n=n.slice(s+1),!!o.trim())try{e(JSON.parse(o))}catch{}}})}var x=(r,e)=>{r.write(`${JSON.stringify(e)}
+`)};function p(r,e,t={},n=12e4){return new Promise((u,s)=>{let o=g.connect(r),a=!1,m=(i,b)=>{a||(a=!0,clearTimeout(l),o.destroy(),i?s(i):u(b))},l=setTimeout(()=>m(new Error(`The session did not answer "${e}" in time.`)),n);o.on("connect",()=>x(o,{id:1,op:e,args:t})),o.on("error",i=>{m(Object.assign(i,{notRunning:i.code==="ENOENT"||i.code==="ECONNREFUSED"}))}),w(o,i=>m(null,i))})}import{homedir as y}from"node:os";import{join as c}from"node:path";var k=c(y(),".ai-remote"),C=c(k,"run");var h=r=>c(C,`${r}.sock`);function N(r,e){if(!Number.isFinite(e.x)||!Number.isFinite(e.y)||e.x<0||e.y<0||e.x>=r.width||e.y>=r.height)throw new Error("Point must be inside the screenshot.");let t=r.viewport??{x:0,y:0,width:r.desktopWidth,height:r.desktopHeight};return{x:Math.min(t.x+t.width-1,Math.floor(t.x+e.x*t.width/r.width)),y:Math.min(t.y+t.height-1,Math.floor(t.y+e.y*t.height/r.height))}}var d=class{session;constructor(e){if(!/^[\w.-]+$/.test(e)||e==="."||e==="..")throw new Error("Use a session name from ai-remote status --json.");this.session=e}async call(e,t){let n=await p(h(this.session),e,t);if(!n.ok)throw new Error(n.error??"Remote operation failed.");return n.result}screenshot(e={}){return this.call("shot",{...e})}act(e,t={}){return this.call("batch",{...t,actions:e})}takeInput(){return this.call("input",{on:!1})}releaseInput(){return this.call("input",{on:!0})}dispatch(e,t={}){return this.call("batch",{...t,actions:e,observe:!1})}};export{d as RemoteComputer,N as desktopPoint};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ai-remote",
-  "version": "0.6.0",
+  "version": "0.6.1",
   "description": "RDP, VNC and SSH protocol engines that run in Node, a Worker or a browser -- plus `npx ai-remote`, a CLI that drives a machine and shows you what it is doing",
   "type": "module",
   "bin": {