humanish 0.101.0 → 0.103.0

package/README.md

@@ -52,8 +52,10 @@ your lab; installing Codex does not change an `openai-computer-use` lab.
 For local browsers, install only `humanish` and follow the
 [local study setup](docs/architecture/local-browser-runtime.md). The first live
 run downloads a verified runtime image; `npx humanish runtime setup` prepares it
-ahead of time. Supported Macs use Lima instead of Docker Desktop. Local inboxes
-and local camera/microphone support remain follow-ups.
+ahead of time. Supported Macs use Lima instead of Docker Desktop.
+[Optional camera and spoken conversation](docs/architecture/participant-media.md)
+use a separate media runtime. [Optional desktop video/audio](docs/architecture/desktop-recording.md)
+adds continuous Observer playback; screenshots remain the default. Local inboxes remain a follow-up.
 
 For a new local study, initialize with your app URL and task:
 

package/dist/affordance.d.ts

@@ -10,6 +10,8 @@ export type AffordanceClass =
 "pointer"
 /** Keyboard input into the page: typed text that is not a URL and not script. */
  | "keyboard"
+/** Spoken input sent through the participant's admitted microphone path. */
+ | "speech"
 /** Direct navigation by URL (the `nav` subset). A human affordance; see the note above. */
  | "url-navigation"
 /** Script execution via the address bar (a `javascript:` URL). Not a human affordance. */

package/dist/affordance.js

@@ -28,6 +28,7 @@ export const AFFORDANCE_CLASS_SCHEMA = "humanish.affordance-use.v1";
 export const NATURALISTIC_AFFORDANCE_CLASSES = [
     "pointer",
     "keyboard",
+    "speech",
     "url-navigation",
     "observation"
 ];
@@ -98,6 +99,8 @@ export function classifyCuaAction(action) {
             }
             return { affordance: "keyboard" };
         }
+        case "speak":
+            return { affordance: "speech" };
         default:
             // The wire->CuaAction mapper drops unrecognized action types before they reach here, so
             // this is unreachable today; classify defensively rather than throwing inside a run.

package/dist/affordance.js.map

@@ -1 +1 @@
-{"version":3,"file":"affordance.js","sourceRoot":"","sources":["../src/affordance.ts"],"names":[],"mappings":"AAAA,4FAA4F;AAC5F,EAAE;AACF,gGAAgG;AAChG,8FAA8F;AAC9F,gGAAgG;AAChG,iGAAiG;AACjG,iGAAiG;AACjG,+FAA+F;AAC/F,8FAA8F;AAC9F,yFAAyF;AACzF,EAAE;AACF,oCAAoC;AACpC,EAAE;AACF,mGAAmG;AACnG,kGAAkG;AAClG,kGAAkG;AAClG,sDAAsD;AACtD,mGAAmG;AACnG,8FAA8F;AAC9F,4CAA4C;AAC5C,EAAE;AACF,4FAA4F;AAC5F,gGAAgG;AAChG,2FAA2F;AAC3F,gGAAgG;AAIhG,MAAM,CAAC,MAAM,uBAAuB,GAAG,4BAA4B,CAAC;AAuBpE,2FAA2F;AAC3F,MAAM,CAAC,MAAM,+BAA+B,GAA+B;IACzE,SAAS;IACT,UAAU;IACV,gBAAgB;IAChB,aAAa;CACd,CAAC;AAEF,qFAAqF;AACrF,MAAM,CAAC,MAAM,2BAA2B,GAA+B,CAAC,kBAAkB,EAAE,UAAU,EAAE,kBAAkB,CAAC,CAAC;AAsB5H,+FAA+F;AAC/F,yEAAyE;AACzE,MAAM,UAAU,GAAG,2BAA2B,CAAC;AAC/C,2FAA2F;AAC3F,6CAA6C;AAC7C,MAAM,oBAAoB,GAAG,6DAA6D,CAAC;AAC3F,kFAAkF;AAClF,MAAM,QAAQ,GAAG,kBAAkB,CAAC;AACpC,MAAM,SAAS,GAAG,8DAA8D,CAAC;AACjF,+CAA+C;AAC/C,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC;IAC9B,KAAK;IACL,cAAc;IACd,cAAc;IACd,cAAc;IACd,WAAW;IACX,WAAW;IACX,WAAW;IACX,YAAY;IACZ,YAAY;IACZ,YAAY;CACb,CAAC,CAAC;AAEH;;;GAGG;AACH,MAAM,UAAU,iBAAiB,CAAC,MAAiB;IACjD,QAAQ,MAAM,CAAC,IAAI,EAAE,CAAC;QACpB,KAAK,OAAO,CAAC;QACb,KAAK,cAAc,CAAC;QACpB,KAAK,MAAM,CAAC;QACZ,KAAK,QAAQ;YACX,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,CAAC;QACnC,KAAK,MAAM,CAAC;QACZ,KAAK,YAAY,CAAC;QAClB,mFAAmF;QACnF,KAAK,MAAM;YACT,OAAO,EAAE,UAAU,EAAE,aAAa,EAAE,CAAC;QACvC,KAAK,UAAU,CAAC,CAAC,CAAC;YAChB,MAAM,KAAK,GAAG,cAAc,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YAC1C,IAAI,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC/B,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;YACnD,CAAC;YACD,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,CAAC;QACpC,CAAC;QACD,KAAK,MAAM,CAAC,CAAC,CAAC;YACZ,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC;YACzB,IAAI,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC1B,wFAAwF;gBACxF,iFAAiF;gBACjF,8DAA8D;gBAC9D,OAAO,EAAE,UAAU,EAAE,kBAAkB,EAAE,MAAM,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YACpE,CAAC;YACD,IAAI,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBACpC,OAAO,EAAE,UAAU,EAAE,kBAAkB,EAAE,MAAM,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YACpE,CAAC;YACD,IAAI,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBACxB,6EAA6E;gBAC7E,OAAO,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YAClE,CAAC;YACD,IAAI,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBACzB,OAAO,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC;YAC/D,CAAC;YACD,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,CAAC;QACpC,CAAC;QACD;YACE,wFAAwF;YACxF,qFAAqF;YACrF,OAAO,EAAE,UAAU,EAAE,aAAa,EAAE,CAAC;IACzC,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,SAAS,cAAc,CAAC,IAAuB;IAC7C,MAAM,KAAK,GAA2B;QACpC,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM;QAC5B,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK;QACpD,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK;KAC1B,CAAC;IACF,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;QAC7B,MAAM,OAAO,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QACzC,OAAO,KAAK,CAAC,OAAO,CAAC,IAAI,OAAO,CAAC;IACnC,CAAC,CAAC,CAAC;IACH,MAAM,KAAK,GAAG,CAAC,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;IAC9C,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;IACnH,MAAM,IAAI,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;IAC3D,OAAO,CAAC,GAAG,SAAS,EAAE,GAAG,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAC3C,CAAC;AAED,+FAA+F;AAC/F,SAAS,QAAQ,CAAC,IAAY;IAC5B,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAC5B,MAAM,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACnC,OAAO,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,WAAW,EAAE,GAAG,CAAC;AACzE,CAAC;AAED,kEAAkE;AAClE,MAAM,UAAU,sBAAsB,CAAC,YAA8C;IACnF,MAAM,MAAM,GAA6C,EAAE,CAAC;IAC5D,KAAK,MAAM,WAAW,IAAI,YAAY,EAAE,CAAC;QACvC,MAAM,CAAC,WAAW,CAAC,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,WAAW,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;IAC7E,CAAC;IACD,MAAM,aAAa,GAAG,2BAA2B,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACxG,OAAO,EAAE,MAAM,EAAE,uBAAuB,EAAE,MAAM,EAAE,KAAK,EAAE,YAAY,CAAC,MAAM,EAAE,aAAa,EAAE,CAAC;AAChG,CAAC"}
+{"version":3,"file":"affordance.js","sourceRoot":"","sources":["../src/affordance.ts"],"names":[],"mappings":"AAAA,4FAA4F;AAC5F,EAAE;AACF,gGAAgG;AAChG,8FAA8F;AAC9F,gGAAgG;AAChG,iGAAiG;AACjG,iGAAiG;AACjG,+FAA+F;AAC/F,8FAA8F;AAC9F,yFAAyF;AACzF,EAAE;AACF,oCAAoC;AACpC,EAAE;AACF,mGAAmG;AACnG,kGAAkG;AAClG,kGAAkG;AAClG,sDAAsD;AACtD,mGAAmG;AACnG,8FAA8F;AAC9F,4CAA4C;AAC5C,EAAE;AACF,4FAA4F;AAC5F,gGAAgG;AAChG,2FAA2F;AAC3F,gGAAgG;AAIhG,MAAM,CAAC,MAAM,uBAAuB,GAAG,4BAA4B,CAAC;AAyBpE,2FAA2F;AAC3F,MAAM,CAAC,MAAM,+BAA+B,GAA+B;IACzE,SAAS;IACT,UAAU;IACV,QAAQ;IACR,gBAAgB;IAChB,aAAa;CACd,CAAC;AAEF,qFAAqF;AACrF,MAAM,CAAC,MAAM,2BAA2B,GAA+B,CAAC,kBAAkB,EAAE,UAAU,EAAE,kBAAkB,CAAC,CAAC;AAsB5H,+FAA+F;AAC/F,yEAAyE;AACzE,MAAM,UAAU,GAAG,2BAA2B,CAAC;AAC/C,2FAA2F;AAC3F,6CAA6C;AAC7C,MAAM,oBAAoB,GAAG,6DAA6D,CAAC;AAC3F,kFAAkF;AAClF,MAAM,QAAQ,GAAG,kBAAkB,CAAC;AACpC,MAAM,SAAS,GAAG,8DAA8D,CAAC;AACjF,+CAA+C;AAC/C,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC;IAC9B,KAAK;IACL,cAAc;IACd,cAAc;IACd,cAAc;IACd,WAAW;IACX,WAAW;IACX,WAAW;IACX,YAAY;IACZ,YAAY;IACZ,YAAY;CACb,CAAC,CAAC;AAEH;;;GAGG;AACH,MAAM,UAAU,iBAAiB,CAAC,MAAiB;IACjD,QAAQ,MAAM,CAAC,IAAI,EAAE,CAAC;QACpB,KAAK,OAAO,CAAC;QACb,KAAK,cAAc,CAAC;QACpB,KAAK,MAAM,CAAC;QACZ,KAAK,QAAQ;YACX,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,CAAC;QACnC,KAAK,MAAM,CAAC;QACZ,KAAK,YAAY,CAAC;QAClB,mFAAmF;QACnF,KAAK,MAAM;YACT,OAAO,EAAE,UAAU,EAAE,aAAa,EAAE,CAAC;QACvC,KAAK,UAAU,CAAC,CAAC,CAAC;YAChB,MAAM,KAAK,GAAG,cAAc,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YAC1C,IAAI,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC/B,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;YACnD,CAAC;YACD,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,CAAC;QACpC,CAAC;QACD,KAAK,MAAM,CAAC,CAAC,CAAC;YACZ,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC;YACzB,IAAI,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC1B,wFAAwF;gBACxF,iFAAiF;gBACjF,8DAA8D;gBAC9D,OAAO,EAAE,UAAU,EAAE,kBAAkB,EAAE,MAAM,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YACpE,CAAC;YACD,IAAI,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBACpC,OAAO,EAAE,UAAU,EAAE,kBAAkB,EAAE,MAAM,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YACpE,CAAC;YACD,IAAI,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBACxB,6EAA6E;gBAC7E,OAAO,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YAClE,CAAC;YACD,IAAI,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBACzB,OAAO,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC;YAC/D,CAAC;YACD,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,CAAC;QACpC,CAAC;QACD,KAAK,OAAO;YACV,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,CAAC;QAClC;YACE,wFAAwF;YACxF,qFAAqF;YACrF,OAAO,EAAE,UAAU,EAAE,aAAa,EAAE,CAAC;IACzC,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,SAAS,cAAc,CAAC,IAAuB;IAC7C,MAAM,KAAK,GAA2B;QACpC,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM;QAC5B,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK;QACpD,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK;KAC1B,CAAC;IACF,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;QAC7B,MAAM,OAAO,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QACzC,OAAO,KAAK,CAAC,OAAO,CAAC,IAAI,OAAO,CAAC;IACnC,CAAC,CAAC,CAAC;IACH,MAAM,KAAK,GAAG,CAAC,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;IAC9C,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;IACnH,MAAM,IAAI,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;IAC3D,OAAO,CAAC,GAAG,SAAS,EAAE,GAAG,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAC3C,CAAC;AAED,+FAA+F;AAC/F,SAAS,QAAQ,CAAC,IAAY;IAC5B,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAC5B,MAAM,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACnC,OAAO,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,WAAW,EAAE,GAAG,CAAC;AACzE,CAAC;AAED,kEAAkE;AAClE,MAAM,UAAU,sBAAsB,CAAC,YAA8C;IACnF,MAAM,MAAM,GAA6C,EAAE,CAAC;IAC5D,KAAK,MAAM,WAAW,IAAI,YAAY,EAAE,CAAC;QACvC,MAAM,CAAC,WAAW,CAAC,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,WAAW,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;IAC7E,CAAC;IACD,MAAM,aAAa,GAAG,2BAA2B,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACxG,OAAO,EAAE,MAAM,EAAE,uBAAuB,EAAE,MAAM,EAAE,KAAK,EAAE,YAAY,CAAC,MAAM,EAAE,aAAa,EAAE,CAAC;AAChG,CAAC"}

package/dist/browser-control-client.d.ts

@@ -1,14 +1,18 @@
-import type { Duplex } from "node:stream";
+import type { Duplex, Writable } from "node:stream";
 import type { CuaExecutor } from "./computer-use.js";
 import { type BrowserControlIdentity } from "./browser-control-protocol.js";
+import type { DesktopRecordingMetadata } from "./desktop-recording-types.js";
 export interface BrowserControlClientOptions {
     transport: Duplex;
     identity: BrowserControlIdentity;
     requestTimeoutMs?: number;
+    /** Set only after the optional media runtime was admitted for this desktop. */
+    speechEnabled?: boolean;
 }
 export interface BrowserControlClient {
     executor: CuaExecutor;
     ready(): Promise<void>;
+    finishRecording(destination: Writable): Promise<DesktopRecordingMetadata>;
     close(): void;
 }
 export declare function createBrowserControlClient(options: BrowserControlClientOptions): BrowserControlClient;

package/dist/browser-control-client.js

@@ -1,12 +1,13 @@
 import { CuaExecutorError } from "./cua-executor-error.js";
 import { BROWSER_CONTROL_LIMITS, BROWSER_CONTROL_VERSION, decodeBrowserControlObservation, parseBrowserControlReply, sameBrowserControlIdentity, validateBrowserControlAction, validateBrowserControlIdentity } from "./browser-control-protocol.js";
 import { BrowserControlTransport } from "./browser-control-transport.js";
+import { receiveBrowserControlRecording } from "./browser-control-recording-transfer.js";
 export function createBrowserControlClient(options) {
     const identity = validateBrowserControlIdentity(options.identity);
     const timeoutMs = options.requestTimeoutMs ?? BROWSER_CONTROL_LIMITS.requestTimeoutMs;
     if (!Number.isFinite(timeoutMs) || timeoutMs < 1 || timeoutMs > BROWSER_CONTROL_LIMITS.maxRequestTimeoutMs)
         throw new CuaExecutorError("invalid_request", "not_dispatched");
-    let seq = 0, busy = false, ready = false, closed = false;
+    let seq = 0, busy = false, ready = false, closed = false, rawStream;
     let pending;
     const finish = () => {
         if (!pending?.reply || !pending.writeComplete)
@@ -14,7 +15,10 @@ export function createBrowserControlClient(options) {
         const operation = pending;
         pending = undefined;
         operation.dispose();
-        if (operation.reply.ok)
+        if (operation.reply.ok && operation.raw) {
+            void operation.raw.then(() => operation.resolve(operation.reply), operation.reject);
+        }
+        else if (operation.reply.ok)
             operation.resolve(operation.reply);
         else {
             const { code, disposition } = operation.reply.error;
@@ -42,6 +46,22 @@ export function createBrowserControlClient(options) {
             transport.close("protocol_mismatch");
             return;
         }
+        if (request.operation === "FINISH_RECORDING" && reply.ok) {
+            if (!reply.recording || !pending.destination) {
+                transport.close("invalid_response");
+                return;
+            }
+            try {
+                rawStream = transport.handoff();
+                closed = true;
+                pending.raw = receiveBrowserControlRecording(rawStream, pending.destination, reply.recording.bytes);
+                void pending.raw.catch(() => { });
+            }
+            catch {
+                transport.close("transport_failed");
+                return;
+            }
+        }
         pending.reply = reply;
         finish();
     }, code => {
@@ -53,7 +73,7 @@ export function createBrowserControlClient(options) {
             operation.reject(new CuaExecutorError(code, operation.written ? "outcome_uncertain" : "not_dispatched"));
         }
     });
-    const exchange = (operation, action, signal) => {
+    const exchange = (operation, action, signal, destination) => {
         if (closed)
             return Promise.reject(new CuaExecutorError("executor_closed", "not_dispatched"));
         if (signal?.aborted)
@@ -69,7 +89,7 @@ export function createBrowserControlClient(options) {
         return new Promise((resolve, reject) => {
             const abort = () => transport.close("cancelled");
             const timer = setTimeout(() => transport.close("deadline_exceeded"), timeoutMs);
-            const slot = { request, written: false, writeComplete: false, resolve, reject,
+            const slot = { request, written: false, writeComplete: false, resolve, reject, ...(destination ? { destination } : {}),
                 dispose: () => { clearTimeout(timer); signal?.removeEventListener("abort", abort); } };
             pending = slot;
             signal?.addEventListener("abort", abort, { once: true });
@@ -103,6 +123,7 @@ export function createBrowserControlClient(options) {
     };
     const executor = {
         stallRecovery: "fail_closed",
+        ...(options.speechEnabled === true ? { speechEnabled: true } : {}),
         observe: () => withOperation(async () => {
             await ensureReady();
             const reply = await exchange("OBSERVE");
@@ -118,6 +139,9 @@ export function createBrowserControlClient(options) {
         }),
         execute: (action, signal) => withOperation(async () => {
             const validAction = validateBrowserControlAction(action);
+            if (validAction.kind === "speak" && options.speechEnabled !== true) {
+                throw new CuaExecutorError("action_rejected", "not_dispatched");
+            }
             if (signal?.aborted)
                 throw new CuaExecutorError("cancelled", "not_dispatched");
             try {
@@ -131,6 +155,22 @@ export function createBrowserControlClient(options) {
             await exchange("EXECUTE", validAction, signal);
         })
     };
-    return { executor, ready: () => withOperation(ensureReady), close: () => transport.close() };
+    return { executor, ready: () => withOperation(ensureReady),
+        finishRecording: destination => withOperation(async () => {
+            if (!destination || destination.destroyed)
+                throw new CuaExecutorError("invalid_request", "not_dispatched");
+            try {
+                await ensureReady();
+                const reply = await exchange("FINISH_RECORDING", undefined, undefined, destination);
+                if (!reply.ok || !reply.recording)
+                    throw new CuaExecutorError("invalid_response", "outcome_uncertain");
+                return reply.recording;
+            }
+            catch (error) {
+                destination.destroy();
+                throw error;
+            }
+        }),
+        close: () => { rawStream?.destroy(); transport.close(); } };
 }
 //# sourceMappingURL=browser-control-client.js.map

package/dist/browser-control-client.js.map

@@ -1 +1 @@
-{"version":3,"file":"browser-control-client.js","sourceRoot":"","sources":["../src/browser-control-client.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAC3D,OAAO,EACL,sBAAsB,EAAE,uBAAuB,EAAE,+BAA+B,EAChF,wBAAwB,EAAE,0BAA0B,EAAE,4BAA4B,EAAE,8BAA8B,EAEnH,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAAE,uBAAuB,EAAE,MAAM,gCAAgC,CAAC;AAKzE,MAAM,UAAU,0BAA0B,CAAC,OAAoC;IAC7E,MAAM,QAAQ,GAAG,8BAA8B,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAClE,MAAM,SAAS,GAAG,OAAO,CAAC,gBAAgB,IAAI,sBAAsB,CAAC,gBAAgB,CAAC;IACtF,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,SAAS,GAAG,CAAC,IAAI,SAAS,GAAG,sBAAsB,CAAC,mBAAmB;QAAE,MAAM,IAAI,gBAAgB,CAAC,iBAAiB,EAAE,gBAAgB,CAAC,CAAC;IAC5K,IAAI,GAAG,GAAG,CAAC,EAAE,IAAI,GAAG,KAAK,EAAE,KAAK,GAAG,KAAK,EAAE,MAAM,GAAG,KAAK,CAAC;IACzD,IAAI,OACyH,CAAC;IAC9H,MAAM,MAAM,GAAG,GAAS,EAAE;QACxB,IAAI,CAAC,OAAO,EAAE,KAAK,IAAI,CAAC,OAAO,CAAC,aAAa;YAAE,OAAO;QACtD,MAAM,SAAS,GAAG,OAAO,CAAC;QAAC,OAAO,GAAG,SAAS,CAAC;QAAC,SAAS,CAAC,OAAO,EAAE,CAAC;QACpE,IAAI,SAAS,CAAC,KAAM,CAAC,EAAE;YAAE,SAAS,CAAC,OAAO,CAAC,SAAS,CAAC,KAAM,CAAC,CAAC;aACxD,CAAC;YACJ,MAAM,EAAE,IAAI,EAAE,WAAW,EAAE,GAAG,SAAS,CAAC,KAAM,CAAC,KAAK,CAAC;YACrD,IAAI,SAAS,CAAC,OAAO,CAAC,SAAS,KAAK,SAAS,IAAI,IAAI,KAAK,iBAAiB,IAAI,WAAW,KAAK,gBAAgB;gBAAE,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YACvI,SAAS,CAAC,MAAM,CAAC,IAAI,gBAAgB,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC,CAAC;QAC5D,CAAC;IACH,CAAC,CAAC;IACF,MAAM,SAAS,GAAG,IAAI,uBAAuB,CAAC,OAAO,CAAC,SAAS,EAAE,KAAK,CAAC,EAAE;QACvE,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;YAAC,SAAS,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;YAAC,OAAO;QAAC,CAAC;QAC/E,IAAI,KAA0B,CAAC;QAC/B,IAAI,CAAC;YAAC,KAAK,GAAG,wBAAwB,CAAC,KAAK,CAAC,CAAC;QAAC,CAAC;QAAC,MAAM,CAAC;YAAC,SAAS,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;YAAC,OAAO;QAAC,CAAC;QACvG,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;QAChC,IAAI,CAAC,0BAA0B,CAAC,QAAQ,EAAE,KAAK,CAAC,QAAQ,CAAC,IAAI,OAAO,CAAC,GAAG,KAAK,KAAK,CAAC,GAAG,IAAI,OAAO,CAAC,SAAS,KAAK,KAAK,CAAC,SAAS;eAC1H,OAAO,CAAC,SAAS,KAAK,KAAK,CAAC,SAAS,IAAI,CAAC,OAAO,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,KAAK,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,KAAK,SAAS,CAAC,EAAE,CAAC;YACrJ,SAAS,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC;YAAC,OAAO;QAC/C,CAAC;QACD,OAAO,CAAC,KAAK,GAAG,KAAK,CAAC;QAAC,MAAM,EAAE,CAAC;IAClC,CAAC,EAAE,IAAI,CAAC,EAAE;QACR,MAAM,GAAG,IAAI,CAAC;QACd,MAAM,SAAS,GAAG,OAAO,CAAC;QAAC,OAAO,GAAG,SAAS,CAAC;QAC/C,IAAI,SAAS,EAAE,CAAC;YAAC,SAAS,CAAC,OAAO,EAAE,CAAC;YAAC,SAAS,CAAC,MAAM,CAAC,IAAI,gBAAgB,CAAC,IAAI,EAAE,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC;QAAC,CAAC;IACnJ,CAAC,CAAC,CAAC;IACH,MAAM,QAAQ,GAAG,CAAC,SAA0C,EAAE,MAAkB,EAAE,MAAoB,EAAgC,EAAE;QACtI,IAAI,MAAM;YAAE,OAAO,OAAO,CAAC,MAAM,CAAC,IAAI,gBAAgB,CAAC,iBAAiB,EAAE,gBAAgB,CAAC,CAAC,CAAC;QAC7F,IAAI,MAAM,EAAE,OAAO;YAAE,OAAO,OAAO,CAAC,MAAM,CAAC,IAAI,gBAAgB,CAAC,WAAW,EAAE,gBAAgB,CAAC,CAAC,CAAC;QAChG,IAAI,GAAG,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;YAAC,SAAS,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC;YAAC,OAAO,OAAO,CAAC,MAAM,CAAC,IAAI,gBAAgB,CAAC,mBAAmB,EAAE,gBAAgB,CAAC,CAAC,CAAC;QAAC,CAAC;QACjK,GAAG,IAAI,CAAC,CAAC;QACT,MAAM,IAAI,GAAG,EAAE,OAAO,EAAE,uBAAyD,EAAE,IAAI,EAAE,SAAkB,EAAE,QAAQ,EAAE,GAAG,EAAE,SAAS,EAAE,WAAW,GAAG,EAAE,EAAE,CAAC;QAC1J,MAAM,OAAO,GAA0B,SAAS,KAAK,SAAS;YAC5D,CAAC,CAAC,EAAE,GAAG,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,UAAU,GAAG,EAAE,EAAE,MAAM,EAAE,MAAO,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,EAAE,SAAS,EAAE,CAAC;QAChG,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,MAAM,KAAK,GAAG,GAAS,EAAE,CAAC,SAAS,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;YACvD,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,KAAK,CAAC,mBAAmB,CAAC,EAAE,SAAS,CAAC,CAAC;YAChF,MAAM,IAAI,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,aAAa,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM;gBAC3E,OAAO,EAAE,GAAG,EAAE,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,mBAAmB,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YACzF,OAAO,GAAG,IAAI,CAAC;YAAC,MAAM,EAAE,gBAAgB,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;YACzE,KAAK,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE;gBACrE,IAAI,OAAO,KAAK,IAAI;oBAAE,OAAO;gBAC7B,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;gBAAC,MAAM,EAAE,CAAC;YACtC,CAAC,EAAE,GAAG,EAAE,GAAG,IAAI,OAAO,KAAK,IAAI;gBAAE,SAAS,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC3E,CAAC,CAAC,CAAC;IACL,CAAC,CAAC;IACF,MAAM,aAAa,GAAG,KAAK,EAAK,IAAsB,EAAc,EAAE;QACpE,IAAI,MAAM;YAAE,MAAM,IAAI,gBAAgB,CAAC,iBAAiB,EAAE,gBAAgB,CAAC,CAAC;QAC5E,IAAI,IAAI;YAAE,MAAM,IAAI,gBAAgB,CAAC,eAAe,EAAE,gBAAgB,CAAC,CAAC;QACxE,IAAI,GAAG,IAAI,CAAC;QACZ,IAAI,CAAC;YAAC,OAAO,MAAM,IAAI,EAAE,CAAC;QAAC,CAAC;gBAAS,CAAC;YAAC,IAAI,GAAG,KAAK,CAAC;QAAC,CAAC;IACxD,CAAC,CAAC;IACF,MAAM,WAAW,GAAG,KAAK,EAAE,MAAoB,EAAiB,EAAE;QAChE,IAAI,KAAK;YAAE,OAAO;QAClB,MAAM,QAAQ,CAAC,OAAO,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;QAAC,KAAK,GAAG,IAAI,CAAC;IAC3D,CAAC,CAAC;IACF,MAAM,QAAQ,GAA4D;QACxE,aAAa,EAAE,aAAa;QAC5B,OAAO,EAAE,GAAG,EAAE,CAAC,aAAa,CAAC,KAAK,IAA6B,EAAE;YAC/D,MAAM,WAAW,EAAE,CAAC;YACpB,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,SAAS,CAAC,CAAC;YACxC,IAAI,CAAC;gBACH,IAAI,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,WAAW;oBAAE,MAAM,IAAI,KAAK,EAAE,CAAC;gBACvD,OAAO,+BAA+B,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;YAC5D,CAAC;YAAC,MAAM,CAAC;gBAAC,SAAS,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;gBAAC,MAAM,IAAI,gBAAgB,CAAC,kBAAkB,EAAE,mBAAmB,CAAC,CAAC;YAAC,CAAC;QACvH,CAAC,CAAC;QACF,OAAO,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,CAAC,aAAa,CAAC,KAAK,IAAI,EAAE;YACpD,MAAM,WAAW,GAAG,4BAA4B,CAAC,MAAM,CAAC,CAAC;YACzD,IAAI,MAAM,EAAE,OAAO;gBAAE,MAAM,IAAI,gBAAgB,CAAC,WAAW,EAAE,gBAAgB,CAAC,CAAC;YAC/E,IAAI,CAAC;gBAAC,MAAM,WAAW,CAAC,MAAM,CAAC,CAAC;YAAC,CAAC;YAClC,OAAO,KAAK,EAAE,CAAC;gBACb,IAAI,KAAK,YAAY,gBAAgB;oBAAE,MAAM,IAAI,gBAAgB,CAAC,KAAK,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAAC;gBAChG,MAAM,KAAK,CAAC;YACd,CAAC;YACD,MAAM,QAAQ,CAAC,SAAS,EAAE,WAAW,EAAE,MAAM,CAAC,CAAC;QACjD,CAAC,CAAC;KACH,CAAC;IACF,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,aAAa,CAAC,WAAW,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,SAAS,CAAC,KAAK,EAAE,EAAE,CAAC;AAC/F,CAAC"}
+{"version":3,"file":"browser-control-client.js","sourceRoot":"","sources":["../src/browser-control-client.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAC3D,OAAO,EACL,sBAAsB,EAAE,uBAAuB,EAAE,+BAA+B,EAChF,wBAAwB,EAAE,0BAA0B,EAAE,4BAA4B,EAAE,8BAA8B,EAEnH,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAAE,uBAAuB,EAAE,MAAM,gCAAgC,CAAC;AACzE,OAAO,EAAE,8BAA8B,EAAE,MAAM,yCAAyC,CAAC;AAiBzF,MAAM,UAAU,0BAA0B,CAAC,OAAoC;IAC7E,MAAM,QAAQ,GAAG,8BAA8B,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAClE,MAAM,SAAS,GAAG,OAAO,CAAC,gBAAgB,IAAI,sBAAsB,CAAC,gBAAgB,CAAC;IACtF,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,SAAS,GAAG,CAAC,IAAI,SAAS,GAAG,sBAAsB,CAAC,mBAAmB;QAAE,MAAM,IAAI,gBAAgB,CAAC,iBAAiB,EAAE,gBAAgB,CAAC,CAAC;IAC5K,IAAI,GAAG,GAAG,CAAC,EAAE,IAAI,GAAG,KAAK,EAAE,KAAK,GAAG,KAAK,EAAE,MAAM,GAAG,KAAK,EAAE,SAA6B,CAAC;IACxF,IAAI,OAEyH,CAAC;IAC9H,MAAM,MAAM,GAAG,GAAS,EAAE;QACxB,IAAI,CAAC,OAAO,EAAE,KAAK,IAAI,CAAC,OAAO,CAAC,aAAa;YAAE,OAAO;QACtD,MAAM,SAAS,GAAG,OAAO,CAAC;QAAC,OAAO,GAAG,SAAS,CAAC;QAAC,SAAS,CAAC,OAAO,EAAE,CAAC;QACpE,IAAI,SAAS,CAAC,KAAM,CAAC,EAAE,IAAI,SAAS,CAAC,GAAG,EAAE,CAAC;YACzC,KAAK,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,OAAO,CAAC,SAAS,CAAC,KAAM,CAAC,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC;QACvF,CAAC;aAAM,IAAI,SAAS,CAAC,KAAM,CAAC,EAAE;YAAE,SAAS,CAAC,OAAO,CAAC,SAAS,CAAC,KAAM,CAAC,CAAC;aAC/D,CAAC;YACJ,MAAM,EAAE,IAAI,EAAE,WAAW,EAAE,GAAG,SAAS,CAAC,KAAM,CAAC,KAAK,CAAC;YACrD,IAAI,SAAS,CAAC,OAAO,CAAC,SAAS,KAAK,SAAS,IAAI,IAAI,KAAK,iBAAiB,IAAI,WAAW,KAAK,gBAAgB;gBAAE,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YACvI,SAAS,CAAC,MAAM,CAAC,IAAI,gBAAgB,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC,CAAC;QAC5D,CAAC;IACH,CAAC,CAAC;IACF,MAAM,SAAS,GAAG,IAAI,uBAAuB,CAAC,OAAO,CAAC,SAAS,EAAE,KAAK,CAAC,EAAE;QACvE,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;YAAC,SAAS,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;YAAC,OAAO;QAAC,CAAC;QAC/E,IAAI,KAA0B,CAAC;QAC/B,IAAI,CAAC;YAAC,KAAK,GAAG,wBAAwB,CAAC,KAAK,CAAC,CAAC;QAAC,CAAC;QAAC,MAAM,CAAC;YAAC,SAAS,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;YAAC,OAAO;QAAC,CAAC;QACvG,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;QAChC,IAAI,CAAC,0BAA0B,CAAC,QAAQ,EAAE,KAAK,CAAC,QAAQ,CAAC,IAAI,OAAO,CAAC,GAAG,KAAK,KAAK,CAAC,GAAG,IAAI,OAAO,CAAC,SAAS,KAAK,KAAK,CAAC,SAAS;eAC1H,OAAO,CAAC,SAAS,KAAK,KAAK,CAAC,SAAS,IAAI,CAAC,OAAO,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,KAAK,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,KAAK,SAAS,CAAC,EAAE,CAAC;YACrJ,SAAS,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC;YAAC,OAAO;QAC/C,CAAC;QACD,IAAI,OAAO,CAAC,SAAS,KAAK,kBAAkB,IAAI,KAAK,CAAC,EAAE,EAAE,CAAC;YACzD,IAAI,CAAC,KAAK,CAAC,SAAS,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;gBAAC,SAAS,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;gBAAC,OAAO;YAAC,CAAC;YAC9F,IAAI,CAAC;gBACH,SAAS,GAAG,SAAS,CAAC,OAAO,EAAE,CAAC;gBAAC,MAAM,GAAG,IAAI,CAAC;gBAC/C,OAAO,CAAC,GAAG,GAAG,8BAA8B,CAAC,SAAS,EAAE,OAAO,CAAC,WAAW,EAAE,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;gBACpG,KAAK,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;YACnC,CAAC;YAAC,MAAM,CAAC;gBAAC,SAAS,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;gBAAC,OAAO;YAAC,CAAC;QAC1D,CAAC;QACD,OAAO,CAAC,KAAK,GAAG,KAAK,CAAC;QAAC,MAAM,EAAE,CAAC;IAClC,CAAC,EAAE,IAAI,CAAC,EAAE;QACR,MAAM,GAAG,IAAI,CAAC;QACd,MAAM,SAAS,GAAG,OAAO,CAAC;QAAC,OAAO,GAAG,SAAS,CAAC;QAC/C,IAAI,SAAS,EAAE,CAAC;YAAC,SAAS,CAAC,OAAO,EAAE,CAAC;YAAC,SAAS,CAAC,MAAM,CAAC,IAAI,gBAAgB,CAAC,IAAI,EAAE,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC;QAAC,CAAC;IACnJ,CAAC,CAAC,CAAC;IACH,MAAM,QAAQ,GAAG,CAAC,SAA+D,EAAE,MAAkB,EACnG,MAAoB,EAAE,WAAsB,EAAgC,EAAE;QAC9E,IAAI,MAAM;YAAE,OAAO,OAAO,CAAC,MAAM,CAAC,IAAI,gBAAgB,CAAC,iBAAiB,EAAE,gBAAgB,CAAC,CAAC,CAAC;QAC7F,IAAI,MAAM,EAAE,OAAO;YAAE,OAAO,OAAO,CAAC,MAAM,CAAC,IAAI,gBAAgB,CAAC,WAAW,EAAE,gBAAgB,CAAC,CAAC,CAAC;QAChG,IAAI,GAAG,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;YAAC,SAAS,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC;YAAC,OAAO,OAAO,CAAC,MAAM,CAAC,IAAI,gBAAgB,CAAC,mBAAmB,EAAE,gBAAgB,CAAC,CAAC,CAAC;QAAC,CAAC;QACjK,GAAG,IAAI,CAAC,CAAC;QACT,MAAM,IAAI,GAAG,EAAE,OAAO,EAAE,uBAAyD,EAAE,IAAI,EAAE,SAAkB,EAAE,QAAQ,EAAE,GAAG,EAAE,SAAS,EAAE,WAAW,GAAG,EAAE,EAAE,CAAC;QAC1J,MAAM,OAAO,GAA0B,SAAS,KAAK,SAAS;YAC5D,CAAC,CAAC,EAAE,GAAG,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,UAAU,GAAG,EAAE,EAAE,MAAM,EAAE,MAAO,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,EAAE,SAAS,EAAE,CAAC;QAChG,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,MAAM,KAAK,GAAG,GAAS,EAAE,CAAC,SAAS,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;YACvD,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,KAAK,CAAC,mBAAmB,CAAC,EAAE,SAAS,CAAC,CAAC;YAChF,MAAM,IAAI,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,aAAa,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACpH,OAAO,EAAE,GAAG,EAAE,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,mBAAmB,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YACzF,OAAO,GAAG,IAAI,CAAC;YAAC,MAAM,EAAE,gBAAgB,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;YACzE,KAAK,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE;gBACrE,IAAI,OAAO,KAAK,IAAI;oBAAE,OAAO;gBAC7B,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;gBAAC,MAAM,EAAE,CAAC;YACtC,CAAC,EAAE,GAAG,EAAE,GAAG,IAAI,OAAO,KAAK,IAAI;gBAAE,SAAS,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC3E,CAAC,CAAC,CAAC;IACL,CAAC,CAAC;IACF,MAAM,aAAa,GAAG,KAAK,EAAK,IAAsB,EAAc,EAAE;QACpE,IAAI,MAAM;YAAE,MAAM,IAAI,gBAAgB,CAAC,iBAAiB,EAAE,gBAAgB,CAAC,CAAC;QAC5E,IAAI,IAAI;YAAE,MAAM,IAAI,gBAAgB,CAAC,eAAe,EAAE,gBAAgB,CAAC,CAAC;QACxE,IAAI,GAAG,IAAI,CAAC;QACZ,IAAI,CAAC;YAAC,OAAO,MAAM,IAAI,EAAE,CAAC;QAAC,CAAC;gBAAS,CAAC;YAAC,IAAI,GAAG,KAAK,CAAC;QAAC,CAAC;IACxD,CAAC,CAAC;IACF,MAAM,WAAW,GAAG,KAAK,EAAE,MAAoB,EAAiB,EAAE;QAChE,IAAI,KAAK;YAAE,OAAO;QAClB,MAAM,QAAQ,CAAC,OAAO,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;QAAC,KAAK,GAAG,IAAI,CAAC;IAC3D,CAAC,CAAC;IACF,MAAM,QAAQ,GAA4D;QACxE,aAAa,EAAE,aAAa;QAC5B,GAAG,CAAC,OAAO,CAAC,aAAa,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,IAAa,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC3E,OAAO,EAAE,GAAG,EAAE,CAAC,aAAa,CAAC,KAAK,IAA6B,EAAE;YAC/D,MAAM,WAAW,EAAE,CAAC;YACpB,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,SAAS,CAAC,CAAC;YACxC,IAAI,CAAC;gBACH,IAAI,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,WAAW;oBAAE,MAAM,IAAI,KAAK,EAAE,CAAC;gBACvD,OAAO,+BAA+B,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;YAC5D,CAAC;YAAC,MAAM,CAAC;gBAAC,SAAS,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;gBAAC,MAAM,IAAI,gBAAgB,CAAC,kBAAkB,EAAE,mBAAmB,CAAC,CAAC;YAAC,CAAC;QACvH,CAAC,CAAC;QACF,OAAO,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,CAAC,aAAa,CAAC,KAAK,IAAI,EAAE;YACpD,MAAM,WAAW,GAAG,4BAA4B,CAAC,MAAM,CAAC,CAAC;YACzD,IAAI,WAAW,CAAC,IAAI,KAAK,OAAO,IAAI,OAAO,CAAC,aAAa,KAAK,IAAI,EAAE,CAAC;gBACnE,MAAM,IAAI,gBAAgB,CAAC,iBAAiB,EAAE,gBAAgB,CAAC,CAAC;YAClE,CAAC;YACD,IAAI,MAAM,EAAE,OAAO;gBAAE,MAAM,IAAI,gBAAgB,CAAC,WAAW,EAAE,gBAAgB,CAAC,CAAC;YAC/E,IAAI,CAAC;gBAAC,MAAM,WAAW,CAAC,MAAM,CAAC,CAAC;YAAC,CAAC;YAClC,OAAO,KAAK,EAAE,CAAC;gBACb,IAAI,KAAK,YAAY,gBAAgB;oBAAE,MAAM,IAAI,gBAAgB,CAAC,KAAK,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAAC;gBAChG,MAAM,KAAK,CAAC;YACd,CAAC;YACD,MAAM,QAAQ,CAAC,SAAS,EAAE,WAAW,EAAE,MAAM,CAAC,CAAC;QACjD,CAAC,CAAC;KACH,CAAC;IACF,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,aAAa,CAAC,WAAW,CAAC;QACxD,eAAe,EAAE,WAAW,CAAC,EAAE,CAAC,aAAa,CAAC,KAAK,IAAI,EAAE;YACvD,IAAI,CAAC,WAAW,IAAI,WAAW,CAAC,SAAS;gBAAE,MAAM,IAAI,gBAAgB,CAAC,iBAAiB,EAAE,gBAAgB,CAAC,CAAC;YAC3G,IAAI,CAAC;gBACH,MAAM,WAAW,EAAE,CAAC;gBACpB,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,kBAAkB,EAAE,SAAS,EAAE,SAAS,EAAE,WAAW,CAAC,CAAC;gBACpF,IAAI,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS;oBAAE,MAAM,IAAI,gBAAgB,CAAC,kBAAkB,EAAE,mBAAmB,CAAC,CAAC;gBACvG,OAAO,KAAK,CAAC,SAAS,CAAC;YACzB,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBAAC,WAAW,CAAC,OAAO,EAAE,CAAC;gBAAC,MAAM,KAAK,CAAC;YAAC,CAAC;QACzD,CAAC,CAAC;QACF,KAAK,EAAE,GAAG,EAAE,GAAG,SAAS,EAAE,OAAO,EAAE,CAAC,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;AAChE,CAAC"}

package/dist/browser-control-dispatcher.d.ts

@@ -1,12 +1,17 @@
-import type { Duplex } from "node:stream";
+import type { Duplex, Readable } from "node:stream";
 import type { CuaExecutor } from "./computer-use.js";
 import { type BrowserControlIdentity } from "./browser-control-protocol.js";
+import { type DesktopRecordingMetadata } from "./desktop-recording-types.js";
 export interface BrowserControlDispatcherOptions {
     transport: Duplex;
     identity: BrowserControlIdentity;
     executor: CuaExecutor;
     isAuthorized: () => boolean;
     authoritySignal: AbortSignal;
+    finishRecording?: () => Promise<{
+        metadata: DesktopRecordingMetadata;
+        stream: Readable;
+    }>;
 }
 /** Caller retains physical runtime ownership. Closing this channel does not claim the browser stopped. */
 export declare function attachBrowserControlDispatcher(options: BrowserControlDispatcherOptions): {

package/dist/browser-control-dispatcher.js

@@ -1,10 +1,12 @@
 import { CuaExecutorError } from "./cua-executor-error.js";
 import { BROWSER_CONTROL_LIMITS, BROWSER_CONTROL_VERSION, encodeBrowserControlObservation, parseBrowserControlRequest, safeBrowserControlFailure, sameBrowserControlIdentity, validateBrowserControlIdentity } from "./browser-control-protocol.js";
 import { BrowserControlTransport } from "./browser-control-transport.js";
+import { sendBrowserControlRecording } from "./browser-control-recording-transfer.js";
+import { desktopRecordingMetadataSchema } from "./desktop-recording-types.js";
 /** Caller retains physical runtime ownership. Closing this channel does not claim the browser stopped. */
 export function attachBrowserControlDispatcher(options) {
     const identity = validateBrowserControlIdentity(options.identity);
-    let lastSeq = 0, busy = false, ready = false, closed = false;
+    let lastSeq = 0, busy = false, ready = false, closed = false, rawStream;
     let active;
     let timer;
     const dispose = () => { clearTimeout(timer); timer = undefined; active?.abort(); active = undefined; };
@@ -69,7 +71,10 @@ export function attachBrowserControlDispatcher(options) {
                 }
                 reply = { ...common, ok: true, observation: encodeBrowserControlObservation(observation) };
             }
-            else {
+            else if (request.operation === "EXECUTE") {
+                if (request.action.kind === "speak" && options.executor.speechEnabled !== true) {
+                    throw new CuaExecutorError("action_rejected", "not_dispatched");
+                }
                 // No await between the owner gate above and invocation. The physical driver must
                 // check this signal again immediately before each actual input after preparation.
                 invoked = true;
@@ -80,6 +85,39 @@ export function attachBrowserControlDispatcher(options) {
                 }
                 reply = { ...common, ok: true };
             }
+            else {
+                invoked = true;
+                if (!options.finishRecording)
+                    throw new CuaExecutorError("action_rejected", "not_dispatched");
+                const recording = await options.finishRecording();
+                let metadata;
+                try {
+                    metadata = desktopRecordingMetadataSchema.parse(recording.metadata);
+                }
+                catch (error) {
+                    recording.stream.destroy();
+                    throw error;
+                }
+                if (closed || signal.aborted || !authorized()) {
+                    recording.stream.destroy();
+                    transport.close("session_revoked");
+                    return;
+                }
+                reply = { ...common, ok: true, recording: metadata };
+                try {
+                    await transport.send(reply);
+                    rawStream = transport.handoff();
+                }
+                catch (error) {
+                    recording.stream.destroy();
+                    throw error;
+                }
+                closed = true;
+                dispose();
+                options.authoritySignal.removeEventListener("abort", revoke);
+                await sendBrowserControlRecording(recording.stream, rawStream, metadata.bytes).catch(() => { });
+                return;
+            }
         }
         catch (error) {
             reply = { ...common, ok: false, error: safeBrowserControlFailure(error, invoked) };
@@ -104,6 +142,6 @@ export function attachBrowserControlDispatcher(options) {
     options.authoritySignal.addEventListener("abort", revoke, { once: true });
     if (!authorized())
         transport.close("session_revoked");
-    return { close: () => transport.close() };
+    return { close: () => { rawStream?.destroy(); transport.close(); } };
 }
 //# sourceMappingURL=browser-control-dispatcher.js.map

package/dist/browser-control-dispatcher.js.map

@@ -1 +1 @@
-{"version":3,"file":"browser-control-dispatcher.js","sourceRoot":"","sources":["../src/browser-control-dispatcher.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAC3D,OAAO,EACL,sBAAsB,EAAE,uBAAuB,EAAE,+BAA+B,EAAE,0BAA0B,EAC5G,yBAAyB,EAAE,0BAA0B,EAAE,8BAA8B,EAEtF,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAAE,uBAAuB,EAAE,MAAM,gCAAgC,CAAC;AAMzE,0GAA0G;AAC1G,MAAM,UAAU,8BAA8B,CAAC,OAAwC;IACrF,MAAM,QAAQ,GAAG,8BAA8B,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAClE,IAAI,OAAO,GAAG,CAAC,EAAE,IAAI,GAAG,KAAK,EAAE,KAAK,GAAG,KAAK,EAAE,MAAM,GAAG,KAAK,CAAC;IAC7D,IAAI,MAAmC,CAAC;IACxC,IAAI,KAAgD,CAAC;IACrD,MAAM,OAAO,GAAG,GAAS,EAAE,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,GAAG,SAAS,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;IAC7G,MAAM,UAAU,GAAG,GAAY,EAAE;QAC/B,IAAI,CAAC;YAAC,OAAO,CAAC,OAAO,CAAC,eAAe,CAAC,OAAO,IAAI,OAAO,CAAC,YAAY,EAAE,KAAK,IAAI,CAAC;QAAC,CAAC;QAAC,MAAM,CAAC;YAAC,OAAO,KAAK,CAAC;QAAC,CAAC;IAC7G,CAAC,CAAC;IACF,MAAM,SAAS,GAAG,IAAI,uBAAuB,CAAC,OAAO,CAAC,SAAS,EAAE,KAAK,CAAC,EAAE;QACvE,IAAI,MAAM;YAAE,OAAO;QACnB,IAAI,IAAI,EAAE,CAAC;YAAC,SAAS,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;YAAC,OAAO;QAAC,CAAC;QACvD,IAAI,OAA8B,CAAC;QACnC,IAAI,CAAC;YAAC,OAAO,GAAG,0BAA0B,CAAC,KAAK,CAAC,CAAC;QAAC,CAAC;QAAC,MAAM,CAAC;YAAC,SAAS,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;YAAC,OAAO;QAAC,CAAC;QAC1G,IAAI,CAAC,0BAA0B,CAAC,QAAQ,EAAE,OAAO,CAAC,QAAQ,CAAC,IAAI,OAAO,CAAC,GAAG,KAAK,OAAO,GAAG,CAAC;eACrF,CAAC,OAAO,CAAC,SAAS,KAAK,OAAO,CAAC,CAAC,CAAC,KAAK,IAAI,OAAO,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC;YAAC,SAAS,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC;YAAC,OAAO;QAAC,CAAC;QAC7H,sFAAsF;QACtF,IAAI,GAAG,IAAI,CAAC;QAAC,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC;QAAC,MAAM,GAAG,IAAI,eAAe,EAAE,CAAC;QACnE,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,KAAK,CAAC,mBAAmB,CAAC,EAAE,sBAAsB,CAAC,gBAAgB,CAAC,CAAC;QACxG,KAAK,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;IACxC,CAAC,EAAE,GAAG,EAAE;QACN,MAAM,GAAG,IAAI,CAAC;QAAC,OAAO,EAAE,CAAC;QAAC,OAAO,CAAC,eAAe,CAAC,mBAAmB,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IACzF,CAAC,CAAC,CAAC;IACH,SAAS,MAAM,KAAW,SAAS,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC;IAC/D,KAAK,UAAU,QAAQ,CAAC,OAA8B,EAAE,MAAmB;QACzE,MAAM,MAAM,GAAG,EAAE,OAAO,EAAE,uBAAyD,EAAE,IAAI,EAAE,OAAgB,EAAE,QAAQ,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG;YACrI,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS;YAC1D,GAAG,CAAC,OAAO,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;QAC/E,IAAI,OAAO,GAAG,KAAK,EAAE,KAA0B,CAAC;QAChD,IAAI,CAAC;YACH,IAAI,CAAC,UAAU,EAAE,IAAI,MAAM,CAAC,OAAO;gBAAE,MAAM,IAAI,gBAAgB,CAAC,iBAAiB,EAAE,gBAAgB,CAAC,CAAC;YACrG,IAAI,OAAO,CAAC,SAAS,KAAK,OAAO,EAAE,CAAC;gBAAC,KAAK,GAAG,IAAI,CAAC;gBAAC,KAAK,GAAG,EAAE,GAAG,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC;YAAC,CAAC;iBAChF,IAAI,OAAO,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;gBACzC,OAAO,GAAG,IAAI,CAAC;gBACf,MAAM,WAAW,GAAG,MAAM,OAAO,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC;gBACrD,IAAI,MAAM,IAAI,MAAM,CAAC,OAAO,IAAI,CAAC,UAAU,EAAE,EAAE,CAAC;oBAAC,SAAS,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;oBAAC,OAAO;gBAAC,CAAC;gBAC9F,KAAK,GAAG,EAAE,GAAG,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,+BAA+B,CAAC,WAAW,CAAC,EAAE,CAAC;YAC7F,CAAC;iBAAM,CAAC;gBACN,iFAAiF;gBACjF,kFAAkF;gBAClF,OAAO,GAAG,IAAI,CAAC;gBACf,MAAM,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;gBACvD,IAAI,MAAM,IAAI,MAAM,CAAC,OAAO,IAAI,CAAC,UAAU,EAAE,EAAE,CAAC;oBAAC,SAAS,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;oBAAC,OAAO;gBAAC,CAAC;gBAC9F,KAAK,GAAG,EAAE,GAAG,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC;YAClC,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAAC,KAAK,GAAG,EAAE,GAAG,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,yBAAyB,CAAC,KAAK,EAAE,OAAO,CAAC,EAAE,CAAC;QAAC,CAAC;QACvG,IAAI,MAAM;YAAE,OAAO;QACnB,IAAI,CAAC;YACH,MAAM,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC5B,IAAI,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,SAAS,KAAK,SAAS,IAAI,KAAK,CAAC,KAAK,CAAC,IAAI,KAAK,iBAAiB,IAAI,KAAK,CAAC,KAAK,CAAC,WAAW,KAAK,gBAAgB,CAAC,EAAE,CAAC;gBAC7I,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBAAC,OAAO;YAC5C,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YAAC,SAAS,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;QAAC,CAAC;gBACxC,CAAC;YAAC,OAAO,EAAE,CAAC;YAAC,IAAI,GAAG,KAAK,CAAC;QAAC,CAAC;IACtC,CAAC;IACD,OAAO,CAAC,eAAe,CAAC,gBAAgB,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;IAC1E,IAAI,CAAC,UAAU,EAAE;QAAE,SAAS,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;IACtD,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,SAAS,CAAC,KAAK,EAAE,EAAE,CAAC;AAC5C,CAAC"}
+{"version":3,"file":"browser-control-dispatcher.js","sourceRoot":"","sources":["../src/browser-control-dispatcher.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAC3D,OAAO,EACL,sBAAsB,EAAE,uBAAuB,EAAE,+BAA+B,EAAE,0BAA0B,EAC5G,yBAAyB,EAAE,0BAA0B,EAAE,8BAA8B,EAEtF,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAAE,uBAAuB,EAAE,MAAM,gCAAgC,CAAC;AACzE,OAAO,EAAE,2BAA2B,EAAE,MAAM,yCAAyC,CAAC;AACtF,OAAO,EAAE,8BAA8B,EAAiC,MAAM,8BAA8B,CAAC;AAO7G,0GAA0G;AAC1G,MAAM,UAAU,8BAA8B,CAAC,OAAwC;IACrF,MAAM,QAAQ,GAAG,8BAA8B,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAClE,IAAI,OAAO,GAAG,CAAC,EAAE,IAAI,GAAG,KAAK,EAAE,KAAK,GAAG,KAAK,EAAE,MAAM,GAAG,KAAK,EAAE,SAA6B,CAAC;IAC5F,IAAI,MAAmC,CAAC;IACxC,IAAI,KAAgD,CAAC;IACrD,MAAM,OAAO,GAAG,GAAS,EAAE,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,GAAG,SAAS,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;IAC7G,MAAM,UAAU,GAAG,GAAY,EAAE;QAC/B,IAAI,CAAC;YAAC,OAAO,CAAC,OAAO,CAAC,eAAe,CAAC,OAAO,IAAI,OAAO,CAAC,YAAY,EAAE,KAAK,IAAI,CAAC;QAAC,CAAC;QAAC,MAAM,CAAC;YAAC,OAAO,KAAK,CAAC;QAAC,CAAC;IAC7G,CAAC,CAAC;IACF,MAAM,SAAS,GAAG,IAAI,uBAAuB,CAAC,OAAO,CAAC,SAAS,EAAE,KAAK,CAAC,EAAE;QACvE,IAAI,MAAM;YAAE,OAAO;QACnB,IAAI,IAAI,EAAE,CAAC;YAAC,SAAS,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;YAAC,OAAO;QAAC,CAAC;QACvD,IAAI,OAA8B,CAAC;QACnC,IAAI,CAAC;YAAC,OAAO,GAAG,0BAA0B,CAAC,KAAK,CAAC,CAAC;QAAC,CAAC;QAAC,MAAM,CAAC;YAAC,SAAS,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;YAAC,OAAO;QAAC,CAAC;QAC1G,IAAI,CAAC,0BAA0B,CAAC,QAAQ,EAAE,OAAO,CAAC,QAAQ,CAAC,IAAI,OAAO,CAAC,GAAG,KAAK,OAAO,GAAG,CAAC;eACrF,CAAC,OAAO,CAAC,SAAS,KAAK,OAAO,CAAC,CAAC,CAAC,KAAK,IAAI,OAAO,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC;YAAC,SAAS,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC;YAAC,OAAO;QAAC,CAAC;QAC7H,sFAAsF;QACtF,IAAI,GAAG,IAAI,CAAC;QAAC,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC;QAAC,MAAM,GAAG,IAAI,eAAe,EAAE,CAAC;QACnE,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,KAAK,CAAC,mBAAmB,CAAC,EAAE,sBAAsB,CAAC,gBAAgB,CAAC,CAAC;QACxG,KAAK,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;IACxC,CAAC,EAAE,GAAG,EAAE;QACN,MAAM,GAAG,IAAI,CAAC;QAAC,OAAO,EAAE,CAAC;QAAC,OAAO,CAAC,eAAe,CAAC,mBAAmB,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IACzF,CAAC,CAAC,CAAC;IACH,SAAS,MAAM,KAAW,SAAS,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC;IAC/D,KAAK,UAAU,QAAQ,CAAC,OAA8B,EAAE,MAAmB;QACzE,MAAM,MAAM,GAAG,EAAE,OAAO,EAAE,uBAAyD,EAAE,IAAI,EAAE,OAAgB,EAAE,QAAQ,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG;YACrI,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS;YAC1D,GAAG,CAAC,OAAO,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;QAC/E,IAAI,OAAO,GAAG,KAAK,EAAE,KAA0B,CAAC;QAChD,IAAI,CAAC;YACH,IAAI,CAAC,UAAU,EAAE,IAAI,MAAM,CAAC,OAAO;gBAAE,MAAM,IAAI,gBAAgB,CAAC,iBAAiB,EAAE,gBAAgB,CAAC,CAAC;YACrG,IAAI,OAAO,CAAC,SAAS,KAAK,OAAO,EAAE,CAAC;gBAAC,KAAK,GAAG,IAAI,CAAC;gBAAC,KAAK,GAAG,EAAE,GAAG,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC;YAAC,CAAC;iBAChF,IAAI,OAAO,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;gBACzC,OAAO,GAAG,IAAI,CAAC;gBACf,MAAM,WAAW,GAAG,MAAM,OAAO,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC;gBACrD,IAAI,MAAM,IAAI,MAAM,CAAC,OAAO,IAAI,CAAC,UAAU,EAAE,EAAE,CAAC;oBAAC,SAAS,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;oBAAC,OAAO;gBAAC,CAAC;gBAC9F,KAAK,GAAG,EAAE,GAAG,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,+BAA+B,CAAC,WAAW,CAAC,EAAE,CAAC;YAC7F,CAAC;iBAAM,IAAI,OAAO,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;gBAC3C,IAAI,OAAO,CAAC,MAAM,CAAC,IAAI,KAAK,OAAO,IAAI,OAAO,CAAC,QAAQ,CAAC,aAAa,KAAK,IAAI,EAAE,CAAC;oBAC/E,MAAM,IAAI,gBAAgB,CAAC,iBAAiB,EAAE,gBAAgB,CAAC,CAAC;gBAClE,CAAC;gBACD,iFAAiF;gBACjF,kFAAkF;gBAClF,OAAO,GAAG,IAAI,CAAC;gBACf,MAAM,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;gBACvD,IAAI,MAAM,IAAI,MAAM,CAAC,OAAO,IAAI,CAAC,UAAU,EAAE,EAAE,CAAC;oBAAC,SAAS,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;oBAAC,OAAO;gBAAC,CAAC;gBAC9F,KAAK,GAAG,EAAE,GAAG,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC;YAClC,CAAC;iBAAM,CAAC;gBACN,OAAO,GAAG,IAAI,CAAC;gBACf,IAAI,CAAC,OAAO,CAAC,eAAe;oBAAE,MAAM,IAAI,gBAAgB,CAAC,iBAAiB,EAAE,gBAAgB,CAAC,CAAC;gBAC9F,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,eAAe,EAAE,CAAC;gBAClD,IAAI,QAAkC,CAAC;gBACvC,IAAI,CAAC;oBAAC,QAAQ,GAAG,8BAA8B,CAAC,KAAK,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;gBAAC,CAAC;gBAC5E,OAAO,KAAK,EAAE,CAAC;oBAAC,SAAS,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;oBAAC,MAAM,KAAK,CAAC;gBAAC,CAAC;gBAC1D,IAAI,MAAM,IAAI,MAAM,CAAC,OAAO,IAAI,CAAC,UAAU,EAAE,EAAE,CAAC;oBAAC,SAAS,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;oBAAC,SAAS,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;oBAAC,OAAO;gBAAC,CAAC;gBAC1H,KAAK,GAAG,EAAE,GAAG,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC;gBACrD,IAAI,CAAC;oBAAC,MAAM,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;oBAAC,SAAS,GAAG,SAAS,CAAC,OAAO,EAAE,CAAC;gBAAC,CAAC;gBACrE,OAAO,KAAK,EAAE,CAAC;oBAAC,SAAS,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;oBAAC,MAAM,KAAK,CAAC;gBAAC,CAAC;gBAC1D,MAAM,GAAG,IAAI,CAAC;gBAAC,OAAO,EAAE,CAAC;gBACzB,OAAO,CAAC,eAAe,CAAC,mBAAmB,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;gBAC7D,MAAM,2BAA2B,CAAC,SAAS,CAAC,MAAM,EAAE,SAAS,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;gBAC/F,OAAO;YACT,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAAC,KAAK,GAAG,EAAE,GAAG,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,yBAAyB,CAAC,KAAK,EAAE,OAAO,CAAC,EAAE,CAAC;QAAC,CAAC;QACvG,IAAI,MAAM;YAAE,OAAO;QACnB,IAAI,CAAC;YACH,MAAM,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC5B,IAAI,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,SAAS,KAAK,SAAS,IAAI,KAAK,CAAC,KAAK,CAAC,IAAI,KAAK,iBAAiB,IAAI,KAAK,CAAC,KAAK,CAAC,WAAW,KAAK,gBAAgB,CAAC,EAAE,CAAC;gBAC7I,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBAAC,OAAO;YAC5C,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YAAC,SAAS,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;QAAC,CAAC;gBACxC,CAAC;YAAC,OAAO,EAAE,CAAC;YAAC,IAAI,GAAG,KAAK,CAAC;QAAC,CAAC;IACtC,CAAC;IACD,OAAO,CAAC,eAAe,CAAC,gBAAgB,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;IAC1E,IAAI,CAAC,UAAU,EAAE;QAAE,SAAS,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;IACtD,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,SAAS,EAAE,OAAO,EAAE,CAAC,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;AACvE,CAAC"}

package/dist/browser-control-protocol.d.ts

@@ -1,5 +1,6 @@
 import { z } from "zod";
 import type { CuaAction, CuaObservation } from "./computer-use.js";
+import { type HeardSpeech } from "./cua-speech.js";
 import { type CuaExecutorErrorCode } from "./cua-executor-error.js";
 export declare const BROWSER_CONTROL_VERSION = 1;
 export declare const BROWSER_CONTROL_LIMITS: Readonly<{
@@ -8,6 +9,10 @@ export declare const BROWSER_CONTROL_LIMITS: Readonly<{
     dimension: 4096;
     pixels: 16000000;
     textBytes: number;
+    speechCharacters: 400;
+    speechBytes: 1600;
+    heardSpeech: 4;
+    speechDurationMs: 120000;
     chordKeys: 16;
     keyCharacters: 64;
     dragPoints: 1024;
@@ -20,6 +25,47 @@ export interface BrowserControlIdentity {
     challenge: string;
     runtimeRevision: string;
 }
+export declare const browserOnlyControlActionSchema: z.ZodDiscriminatedUnion<readonly [z.ZodObject<{
+    button: z.ZodOptional<z.ZodEnum<{
+        left: "left";
+        right: "right";
+        middle: "middle";
+    }>>;
+    x: z.ZodNumber;
+    y: z.ZodNumber;
+    kind: z.ZodLiteral<"click">;
+}, z.core.$strict>, z.ZodObject<{
+    x: z.ZodNumber;
+    y: z.ZodNumber;
+    kind: z.ZodLiteral<"double_click">;
+}, z.core.$strict>, z.ZodObject<{
+    x: z.ZodNumber;
+    y: z.ZodNumber;
+    kind: z.ZodLiteral<"move">;
+}, z.core.$strict>, z.ZodObject<{
+    dx: z.ZodNumber;
+    dy: z.ZodNumber;
+    x: z.ZodNumber;
+    y: z.ZodNumber;
+    kind: z.ZodLiteral<"scroll">;
+}, z.core.$strict>, z.ZodObject<{
+    kind: z.ZodLiteral<"type">;
+    text: z.ZodString;
+}, z.core.$strict>, z.ZodObject<{
+    kind: z.ZodLiteral<"keypress">;
+    keys: z.ZodArray<z.ZodString>;
+}, z.core.$strict>, z.ZodObject<{
+    kind: z.ZodLiteral<"drag">;
+    path: z.ZodArray<z.ZodObject<{
+        x: z.ZodNumber;
+        y: z.ZodNumber;
+    }, z.core.$strict>>;
+}, z.core.$strict>, z.ZodObject<{
+    kind: z.ZodLiteral<"wait">;
+    ms: z.ZodOptional<z.ZodNumber>;
+}, z.core.$strict>, z.ZodObject<{
+    kind: z.ZodLiteral<"screenshot">;
+}, z.core.$strict>], "kind">;
 export declare const browserControlActionSchema: z.ZodPipe<z.ZodDiscriminatedUnion<[z.ZodObject<{
     button: z.ZodOptional<z.ZodEnum<{
         left: "left";
@@ -60,6 +106,9 @@ export declare const browserControlActionSchema: z.ZodPipe<z.ZodDiscriminatedUni
     ms: z.ZodOptional<z.ZodNumber>;
 }, z.core.$strict>, z.ZodObject<{
     kind: z.ZodLiteral<"screenshot">;
+}, z.core.$strict>, z.ZodObject<{
+    kind: z.ZodLiteral<"speak">;
+    text: z.ZodString;
 }, z.core.$strict>], "kind">, z.ZodTransform<{
     kind: "click";
     x: number;
@@ -94,6 +143,9 @@ export declare const browserControlActionSchema: z.ZodPipe<z.ZodDiscriminatedUni
 } | {
     kind: "wait";
     ms?: number;
+} | {
+    kind: "speak";
+    text: string;
 } | {
     kind: "screenshot";
 }, {
@@ -132,6 +184,9 @@ export declare const browserControlActionSchema: z.ZodPipe<z.ZodDiscriminatedUni
     ms?: number | undefined;
 } | {
     kind: "screenshot";
+} | {
+    kind: "speak";
+    text: string;
 }>>;
 declare const observationSchema: z.ZodObject<{
     png: z.ZodString;
@@ -140,6 +195,12 @@ declare const observationSchema: z.ZodObject<{
     title: z.ZodOptional<z.ZodString>;
     text: z.ZodOptional<z.ZodString>;
     scrollY: z.ZodOptional<z.ZodNumber>;
+    heardSpeech: z.ZodOptional<z.ZodArray<z.ZodObject<{
+        id: z.ZodString;
+        source: z.ZodLiteral<"speaker_audio">;
+        text: z.ZodString;
+        durationMs: z.ZodNumber;
+    }, z.core.$strict>>>;
 }, z.core.$strict>;
 type WireObservation = z.infer<typeof observationSchema>;
 declare const requestSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
@@ -208,6 +269,9 @@ declare const requestSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
         ms: z.ZodOptional<z.ZodNumber>;
     }, z.core.$strict>, z.ZodObject<{
         kind: z.ZodLiteral<"screenshot">;
+    }, z.core.$strict>, z.ZodObject<{
+        kind: z.ZodLiteral<"speak">;
+        text: z.ZodString;
     }, z.core.$strict>], "kind">, z.ZodTransform<{
         kind: "click";
         x: number;
@@ -242,6 +306,9 @@ declare const requestSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
     } | {
         kind: "wait";
         ms?: number;
+    } | {
+        kind: "speak";
+        text: string;
     } | {
         kind: "screenshot";
     }, {
@@ -280,6 +347,9 @@ declare const requestSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
         ms?: number | undefined;
     } | {
         kind: "screenshot";
+    } | {
+        kind: "speak";
+        text: string;
     }>>;
     version: z.ZodLiteral<1>;
     identity: z.ZodObject<{
@@ -289,6 +359,17 @@ declare const requestSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
     }, z.core.$strict>;
     seq: z.ZodNumber;
     requestId: z.ZodString;
+}, z.core.$strict>, z.ZodObject<{
+    type: z.ZodLiteral<"request">;
+    operation: z.ZodLiteral<"FINISH_RECORDING">;
+    version: z.ZodLiteral<1>;
+    identity: z.ZodObject<{
+        generation: z.ZodString;
+        challenge: z.ZodString;
+        runtimeRevision: z.ZodString;
+    }, z.core.$strict>;
+    seq: z.ZodNumber;
+    requestId: z.ZodString;
 }, z.core.$strict>], "operation">;
 declare const replySchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
     ok: z.ZodLiteral<true>;
@@ -299,12 +380,30 @@ declare const replySchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
         title: z.ZodOptional<z.ZodString>;
         text: z.ZodOptional<z.ZodString>;
         scrollY: z.ZodOptional<z.ZodNumber>;
+        heardSpeech: z.ZodOptional<z.ZodArray<z.ZodObject<{
+            id: z.ZodString;
+            source: z.ZodLiteral<"speaker_audio">;
+            text: z.ZodString;
+            durationMs: z.ZodNumber;
+        }, z.core.$strict>>>;
+    }, z.core.$strict>>;
+    recording: z.ZodOptional<z.ZodObject<{
+        mimeType: z.ZodLiteral<"video/mp4">;
+        startedAt: z.ZodISODateTime;
+        durationMs: z.ZodNumber;
+        bytes: z.ZodNumber;
+        audioSources: z.ZodArray<z.ZodEnum<{
+            "microphone-input": "microphone-input";
+            "speaker-output": "speaker-output";
+        }>>;
+        complete: z.ZodBoolean;
     }, z.core.$strict>>;
     type: z.ZodLiteral<"reply">;
     operation: z.ZodEnum<{
         HELLO: "HELLO";
         OBSERVE: "OBSERVE";
         EXECUTE: "EXECUTE";
+        FINISH_RECORDING: "FINISH_RECORDING";
     }>;
     actionId: z.ZodOptional<z.ZodString>;
     version: z.ZodLiteral<1>;
@@ -342,6 +441,7 @@ declare const replySchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
         HELLO: "HELLO";
         OBSERVE: "OBSERVE";
         EXECUTE: "EXECUTE";
+        FINISH_RECORDING: "FINISH_RECORDING";
     }>;
     actionId: z.ZodOptional<z.ZodString>;
     version: z.ZodLiteral<1>;
@@ -360,6 +460,7 @@ export declare function sameBrowserControlIdentity(a: BrowserControlIdentity, b:
 export declare function parseBrowserControlRequest(value: unknown): BrowserControlRequest;
 export declare function parseBrowserControlReply(value: unknown): BrowserControlReply;
 export declare function validateBrowserControlAction(value: unknown): CuaAction;
+export declare function validateHeardSpeech(value: unknown): HeardSpeech[];
 /** Check dimensions before allocating decoder output; v1 accepts browser-style 8-bit, noninterlaced PNG. */
 export declare function validateBrowserControlPng(bytes: Buffer): void;
 export declare function encodeBrowserControlObservation(observation: CuaObservation): WireObservation;