humanish 0.101.0 → 0.102.0

package/README.md

@@ -52,8 +52,9 @@ your lab; installing Codex does not change an `openai-computer-use` lab.
 For local browsers, install only `humanish` and follow the
 [local study setup](docs/architecture/local-browser-runtime.md). The first live
 run downloads a verified runtime image; `npx humanish runtime setup` prepares it
-ahead of time. Supported Macs use Lima instead of Docker Desktop. Local inboxes
-and local camera/microphone support remain follow-ups.
+ahead of time. Supported Macs use Lima instead of Docker Desktop.
+[Optional camera and spoken conversation](docs/architecture/participant-media.md)
+use a separate media runtime. Local inboxes remain a follow-up.
 
 For a new local study, initialize with your app URL and task:
 

package/dist/affordance.d.ts

@@ -10,6 +10,8 @@ export type AffordanceClass =
 "pointer"
 /** Keyboard input into the page: typed text that is not a URL and not script. */
  | "keyboard"
+/** Spoken input sent through the participant's admitted microphone path. */
+ | "speech"
 /** Direct navigation by URL (the `nav` subset). A human affordance; see the note above. */
  | "url-navigation"
 /** Script execution via the address bar (a `javascript:` URL). Not a human affordance. */

package/dist/affordance.js

@@ -28,6 +28,7 @@ export const AFFORDANCE_CLASS_SCHEMA = "humanish.affordance-use.v1";
 export const NATURALISTIC_AFFORDANCE_CLASSES = [
     "pointer",
     "keyboard",
+    "speech",
     "url-navigation",
     "observation"
 ];
@@ -98,6 +99,8 @@ export function classifyCuaAction(action) {
             }
             return { affordance: "keyboard" };
         }
+        case "speak":
+            return { affordance: "speech" };
         default:
             // The wire->CuaAction mapper drops unrecognized action types before they reach here, so
             // this is unreachable today; classify defensively rather than throwing inside a run.

package/dist/affordance.js.map

@@ -1 +1 @@
-{"version":3,"file":"affordance.js","sourceRoot":"","sources":["../src/affordance.ts"],"names":[],"mappings":"AAAA,4FAA4F;AAC5F,EAAE;AACF,gGAAgG;AAChG,8FAA8F;AAC9F,gGAAgG;AAChG,iGAAiG;AACjG,iGAAiG;AACjG,+FAA+F;AAC/F,8FAA8F;AAC9F,yFAAyF;AACzF,EAAE;AACF,oCAAoC;AACpC,EAAE;AACF,mGAAmG;AACnG,kGAAkG;AAClG,kGAAkG;AAClG,sDAAsD;AACtD,mGAAmG;AACnG,8FAA8F;AAC9F,4CAA4C;AAC5C,EAAE;AACF,4FAA4F;AAC5F,gGAAgG;AAChG,2FAA2F;AAC3F,gGAAgG;AAIhG,MAAM,CAAC,MAAM,uBAAuB,GAAG,4BAA4B,CAAC;AAuBpE,2FAA2F;AAC3F,MAAM,CAAC,MAAM,+BAA+B,GAA+B;IACzE,SAAS;IACT,UAAU;IACV,gBAAgB;IAChB,aAAa;CACd,CAAC;AAEF,qFAAqF;AACrF,MAAM,CAAC,MAAM,2BAA2B,GAA+B,CAAC,kBAAkB,EAAE,UAAU,EAAE,kBAAkB,CAAC,CAAC;AAsB5H,+FAA+F;AAC/F,yEAAyE;AACzE,MAAM,UAAU,GAAG,2BAA2B,CAAC;AAC/C,2FAA2F;AAC3F,6CAA6C;AAC7C,MAAM,oBAAoB,GAAG,6DAA6D,CAAC;AAC3F,kFAAkF;AAClF,MAAM,QAAQ,GAAG,kBAAkB,CAAC;AACpC,MAAM,SAAS,GAAG,8DAA8D,CAAC;AACjF,+CAA+C;AAC/C,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC;IAC9B,KAAK;IACL,cAAc;IACd,cAAc;IACd,cAAc;IACd,WAAW;IACX,WAAW;IACX,WAAW;IACX,YAAY;IACZ,YAAY;IACZ,YAAY;CACb,CAAC,CAAC;AAEH;;;GAGG;AACH,MAAM,UAAU,iBAAiB,CAAC,MAAiB;IACjD,QAAQ,MAAM,CAAC,IAAI,EAAE,CAAC;QACpB,KAAK,OAAO,CAAC;QACb,KAAK,cAAc,CAAC;QACpB,KAAK,MAAM,CAAC;QACZ,KAAK,QAAQ;YACX,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,CAAC;QACnC,KAAK,MAAM,CAAC;QACZ,KAAK,YAAY,CAAC;QAClB,mFAAmF;QACnF,KAAK,MAAM;YACT,OAAO,EAAE,UAAU,EAAE,aAAa,EAAE,CAAC;QACvC,KAAK,UAAU,CAAC,CAAC,CAAC;YAChB,MAAM,KAAK,GAAG,cAAc,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YAC1C,IAAI,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC/B,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;YACnD,CAAC;YACD,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,CAAC;QACpC,CAAC;QACD,KAAK,MAAM,CAAC,CAAC,CAAC;YACZ,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC;YACzB,IAAI,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC1B,wFAAwF;gBACxF,iFAAiF;gBACjF,8DAA8D;gBAC9D,OAAO,EAAE,UAAU,EAAE,kBAAkB,EAAE,MAAM,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YACpE,CAAC;YACD,IAAI,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBACpC,OAAO,EAAE,UAAU,EAAE,kBAAkB,EAAE,MAAM,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YACpE,CAAC;YACD,IAAI,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBACxB,6EAA6E;gBAC7E,OAAO,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YAClE,CAAC;YACD,IAAI,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBACzB,OAAO,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC;YAC/D,CAAC;YACD,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,CAAC;QACpC,CAAC;QACD;YACE,wFAAwF;YACxF,qFAAqF;YACrF,OAAO,EAAE,UAAU,EAAE,aAAa,EAAE,CAAC;IACzC,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,SAAS,cAAc,CAAC,IAAuB;IAC7C,MAAM,KAAK,GAA2B;QACpC,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM;QAC5B,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK;QACpD,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK;KAC1B,CAAC;IACF,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;QAC7B,MAAM,OAAO,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QACzC,OAAO,KAAK,CAAC,OAAO,CAAC,IAAI,OAAO,CAAC;IACnC,CAAC,CAAC,CAAC;IACH,MAAM,KAAK,GAAG,CAAC,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;IAC9C,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;IACnH,MAAM,IAAI,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;IAC3D,OAAO,CAAC,GAAG,SAAS,EAAE,GAAG,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAC3C,CAAC;AAED,+FAA+F;AAC/F,SAAS,QAAQ,CAAC,IAAY;IAC5B,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAC5B,MAAM,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACnC,OAAO,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,WAAW,EAAE,GAAG,CAAC;AACzE,CAAC;AAED,kEAAkE;AAClE,MAAM,UAAU,sBAAsB,CAAC,YAA8C;IACnF,MAAM,MAAM,GAA6C,EAAE,CAAC;IAC5D,KAAK,MAAM,WAAW,IAAI,YAAY,EAAE,CAAC;QACvC,MAAM,CAAC,WAAW,CAAC,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,WAAW,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;IAC7E,CAAC;IACD,MAAM,aAAa,GAAG,2BAA2B,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACxG,OAAO,EAAE,MAAM,EAAE,uBAAuB,EAAE,MAAM,EAAE,KAAK,EAAE,YAAY,CAAC,MAAM,EAAE,aAAa,EAAE,CAAC;AAChG,CAAC"}
+{"version":3,"file":"affordance.js","sourceRoot":"","sources":["../src/affordance.ts"],"names":[],"mappings":"AAAA,4FAA4F;AAC5F,EAAE;AACF,gGAAgG;AAChG,8FAA8F;AAC9F,gGAAgG;AAChG,iGAAiG;AACjG,iGAAiG;AACjG,+FAA+F;AAC/F,8FAA8F;AAC9F,yFAAyF;AACzF,EAAE;AACF,oCAAoC;AACpC,EAAE;AACF,mGAAmG;AACnG,kGAAkG;AAClG,kGAAkG;AAClG,sDAAsD;AACtD,mGAAmG;AACnG,8FAA8F;AAC9F,4CAA4C;AAC5C,EAAE;AACF,4FAA4F;AAC5F,gGAAgG;AAChG,2FAA2F;AAC3F,gGAAgG;AAIhG,MAAM,CAAC,MAAM,uBAAuB,GAAG,4BAA4B,CAAC;AAyBpE,2FAA2F;AAC3F,MAAM,CAAC,MAAM,+BAA+B,GAA+B;IACzE,SAAS;IACT,UAAU;IACV,QAAQ;IACR,gBAAgB;IAChB,aAAa;CACd,CAAC;AAEF,qFAAqF;AACrF,MAAM,CAAC,MAAM,2BAA2B,GAA+B,CAAC,kBAAkB,EAAE,UAAU,EAAE,kBAAkB,CAAC,CAAC;AAsB5H,+FAA+F;AAC/F,yEAAyE;AACzE,MAAM,UAAU,GAAG,2BAA2B,CAAC;AAC/C,2FAA2F;AAC3F,6CAA6C;AAC7C,MAAM,oBAAoB,GAAG,6DAA6D,CAAC;AAC3F,kFAAkF;AAClF,MAAM,QAAQ,GAAG,kBAAkB,CAAC;AACpC,MAAM,SAAS,GAAG,8DAA8D,CAAC;AACjF,+CAA+C;AAC/C,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC;IAC9B,KAAK;IACL,cAAc;IACd,cAAc;IACd,cAAc;IACd,WAAW;IACX,WAAW;IACX,WAAW;IACX,YAAY;IACZ,YAAY;IACZ,YAAY;CACb,CAAC,CAAC;AAEH;;;GAGG;AACH,MAAM,UAAU,iBAAiB,CAAC,MAAiB;IACjD,QAAQ,MAAM,CAAC,IAAI,EAAE,CAAC;QACpB,KAAK,OAAO,CAAC;QACb,KAAK,cAAc,CAAC;QACpB,KAAK,MAAM,CAAC;QACZ,KAAK,QAAQ;YACX,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,CAAC;QACnC,KAAK,MAAM,CAAC;QACZ,KAAK,YAAY,CAAC;QAClB,mFAAmF;QACnF,KAAK,MAAM;YACT,OAAO,EAAE,UAAU,EAAE,aAAa,EAAE,CAAC;QACvC,KAAK,UAAU,CAAC,CAAC,CAAC;YAChB,MAAM,KAAK,GAAG,cAAc,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YAC1C,IAAI,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC/B,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;YACnD,CAAC;YACD,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,CAAC;QACpC,CAAC;QACD,KAAK,MAAM,CAAC,CAAC,CAAC;YACZ,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC;YACzB,IAAI,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC1B,wFAAwF;gBACxF,iFAAiF;gBACjF,8DAA8D;gBAC9D,OAAO,EAAE,UAAU,EAAE,kBAAkB,EAAE,MAAM,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YACpE,CAAC;YACD,IAAI,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBACpC,OAAO,EAAE,UAAU,EAAE,kBAAkB,EAAE,MAAM,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YACpE,CAAC;YACD,IAAI,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBACxB,6EAA6E;gBAC7E,OAAO,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YAClE,CAAC;YACD,IAAI,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBACzB,OAAO,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC;YAC/D,CAAC;YACD,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,CAAC;QACpC,CAAC;QACD,KAAK,OAAO;YACV,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,CAAC;QAClC;YACE,wFAAwF;YACxF,qFAAqF;YACrF,OAAO,EAAE,UAAU,EAAE,aAAa,EAAE,CAAC;IACzC,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,SAAS,cAAc,CAAC,IAAuB;IAC7C,MAAM,KAAK,GAA2B;QACpC,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM;QAC5B,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK;QACpD,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK;KAC1B,CAAC;IACF,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;QAC7B,MAAM,OAAO,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QACzC,OAAO,KAAK,CAAC,OAAO,CAAC,IAAI,OAAO,CAAC;IACnC,CAAC,CAAC,CAAC;IACH,MAAM,KAAK,GAAG,CAAC,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;IAC9C,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;IACnH,MAAM,IAAI,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;IAC3D,OAAO,CAAC,GAAG,SAAS,EAAE,GAAG,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAC3C,CAAC;AAED,+FAA+F;AAC/F,SAAS,QAAQ,CAAC,IAAY;IAC5B,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAC5B,MAAM,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACnC,OAAO,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,WAAW,EAAE,GAAG,CAAC;AACzE,CAAC;AAED,kEAAkE;AAClE,MAAM,UAAU,sBAAsB,CAAC,YAA8C;IACnF,MAAM,MAAM,GAA6C,EAAE,CAAC;IAC5D,KAAK,MAAM,WAAW,IAAI,YAAY,EAAE,CAAC;QACvC,MAAM,CAAC,WAAW,CAAC,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,WAAW,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;IAC7E,CAAC;IACD,MAAM,aAAa,GAAG,2BAA2B,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACxG,OAAO,EAAE,MAAM,EAAE,uBAAuB,EAAE,MAAM,EAAE,KAAK,EAAE,YAAY,CAAC,MAAM,EAAE,aAAa,EAAE,CAAC;AAChG,CAAC"}

package/dist/browser-control-client.d.ts

@@ -5,6 +5,8 @@ export interface BrowserControlClientOptions {
     transport: Duplex;
     identity: BrowserControlIdentity;
     requestTimeoutMs?: number;
+    /** Set only after the optional media runtime was admitted for this desktop. */
+    speechEnabled?: boolean;
 }
 export interface BrowserControlClient {
     executor: CuaExecutor;

package/dist/browser-control-client.js

@@ -103,6 +103,7 @@ export function createBrowserControlClient(options) {
     };
     const executor = {
         stallRecovery: "fail_closed",
+        ...(options.speechEnabled === true ? { speechEnabled: true } : {}),
         observe: () => withOperation(async () => {
             await ensureReady();
             const reply = await exchange("OBSERVE");
@@ -118,6 +119,9 @@ export function createBrowserControlClient(options) {
         }),
         execute: (action, signal) => withOperation(async () => {
             const validAction = validateBrowserControlAction(action);
+            if (validAction.kind === "speak" && options.speechEnabled !== true) {
+                throw new CuaExecutorError("action_rejected", "not_dispatched");
+            }
             if (signal?.aborted)
                 throw new CuaExecutorError("cancelled", "not_dispatched");
             try {

package/dist/browser-control-client.js.map

@@ -1 +1 @@
-{"version":3,"file":"browser-control-client.js","sourceRoot":"","sources":["../src/browser-control-client.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAC3D,OAAO,EACL,sBAAsB,EAAE,uBAAuB,EAAE,+BAA+B,EAChF,wBAAwB,EAAE,0BAA0B,EAAE,4BAA4B,EAAE,8BAA8B,EAEnH,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAAE,uBAAuB,EAAE,MAAM,gCAAgC,CAAC;AAKzE,MAAM,UAAU,0BAA0B,CAAC,OAAoC;IAC7E,MAAM,QAAQ,GAAG,8BAA8B,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAClE,MAAM,SAAS,GAAG,OAAO,CAAC,gBAAgB,IAAI,sBAAsB,CAAC,gBAAgB,CAAC;IACtF,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,SAAS,GAAG,CAAC,IAAI,SAAS,GAAG,sBAAsB,CAAC,mBAAmB;QAAE,MAAM,IAAI,gBAAgB,CAAC,iBAAiB,EAAE,gBAAgB,CAAC,CAAC;IAC5K,IAAI,GAAG,GAAG,CAAC,EAAE,IAAI,GAAG,KAAK,EAAE,KAAK,GAAG,KAAK,EAAE,MAAM,GAAG,KAAK,CAAC;IACzD,IAAI,OACyH,CAAC;IAC9H,MAAM,MAAM,GAAG,GAAS,EAAE;QACxB,IAAI,CAAC,OAAO,EAAE,KAAK,IAAI,CAAC,OAAO,CAAC,aAAa;YAAE,OAAO;QACtD,MAAM,SAAS,GAAG,OAAO,CAAC;QAAC,OAAO,GAAG,SAAS,CAAC;QAAC,SAAS,CAAC,OAAO,EAAE,CAAC;QACpE,IAAI,SAAS,CAAC,KAAM,CAAC,EAAE;YAAE,SAAS,CAAC,OAAO,CAAC,SAAS,CAAC,KAAM,CAAC,CAAC;aACxD,CAAC;YACJ,MAAM,EAAE,IAAI,EAAE,WAAW,EAAE,GAAG,SAAS,CAAC,KAAM,CAAC,KAAK,CAAC;YACrD,IAAI,SAAS,CAAC,OAAO,CAAC,SAAS,KAAK,SAAS,IAAI,IAAI,KAAK,iBAAiB,IAAI,WAAW,KAAK,gBAAgB;gBAAE,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YACvI,SAAS,CAAC,MAAM,CAAC,IAAI,gBAAgB,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC,CAAC;QAC5D,CAAC;IACH,CAAC,CAAC;IACF,MAAM,SAAS,GAAG,IAAI,uBAAuB,CAAC,OAAO,CAAC,SAAS,EAAE,KAAK,CAAC,EAAE;QACvE,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;YAAC,SAAS,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;YAAC,OAAO;QAAC,CAAC;QAC/E,IAAI,KAA0B,CAAC;QAC/B,IAAI,CAAC;YAAC,KAAK,GAAG,wBAAwB,CAAC,KAAK,CAAC,CAAC;QAAC,CAAC;QAAC,MAAM,CAAC;YAAC,SAAS,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;YAAC,OAAO;QAAC,CAAC;QACvG,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;QAChC,IAAI,CAAC,0BAA0B,CAAC,QAAQ,EAAE,KAAK,CAAC,QAAQ,CAAC,IAAI,OAAO,CAAC,GAAG,KAAK,KAAK,CAAC,GAAG,IAAI,OAAO,CAAC,SAAS,KAAK,KAAK,CAAC,SAAS;eAC1H,OAAO,CAAC,SAAS,KAAK,KAAK,CAAC,SAAS,IAAI,CAAC,OAAO,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,KAAK,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,KAAK,SAAS,CAAC,EAAE,CAAC;YACrJ,SAAS,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC;YAAC,OAAO;QAC/C,CAAC;QACD,OAAO,CAAC,KAAK,GAAG,KAAK,CAAC;QAAC,MAAM,EAAE,CAAC;IAClC,CAAC,EAAE,IAAI,CAAC,EAAE;QACR,MAAM,GAAG,IAAI,CAAC;QACd,MAAM,SAAS,GAAG,OAAO,CAAC;QAAC,OAAO,GAAG,SAAS,CAAC;QAC/C,IAAI,SAAS,EAAE,CAAC;YAAC,SAAS,CAAC,OAAO,EAAE,CAAC;YAAC,SAAS,CAAC,MAAM,CAAC,IAAI,gBAAgB,CAAC,IAAI,EAAE,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC;QAAC,CAAC;IACnJ,CAAC,CAAC,CAAC;IACH,MAAM,QAAQ,GAAG,CAAC,SAA0C,EAAE,MAAkB,EAAE,MAAoB,EAAgC,EAAE;QACtI,IAAI,MAAM;YAAE,OAAO,OAAO,CAAC,MAAM,CAAC,IAAI,gBAAgB,CAAC,iBAAiB,EAAE,gBAAgB,CAAC,CAAC,CAAC;QAC7F,IAAI,MAAM,EAAE,OAAO;YAAE,OAAO,OAAO,CAAC,MAAM,CAAC,IAAI,gBAAgB,CAAC,WAAW,EAAE,gBAAgB,CAAC,CAAC,CAAC;QAChG,IAAI,GAAG,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;YAAC,SAAS,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC;YAAC,OAAO,OAAO,CAAC,MAAM,CAAC,IAAI,gBAAgB,CAAC,mBAAmB,EAAE,gBAAgB,CAAC,CAAC,CAAC;QAAC,CAAC;QACjK,GAAG,IAAI,CAAC,CAAC;QACT,MAAM,IAAI,GAAG,EAAE,OAAO,EAAE,uBAAyD,EAAE,IAAI,EAAE,SAAkB,EAAE,QAAQ,EAAE,GAAG,EAAE,SAAS,EAAE,WAAW,GAAG,EAAE,EAAE,CAAC;QAC1J,MAAM,OAAO,GAA0B,SAAS,KAAK,SAAS;YAC5D,CAAC,CAAC,EAAE,GAAG,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,UAAU,GAAG,EAAE,EAAE,MAAM,EAAE,MAAO,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,EAAE,SAAS,EAAE,CAAC;QAChG,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,MAAM,KAAK,GAAG,GAAS,EAAE,CAAC,SAAS,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;YACvD,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,KAAK,CAAC,mBAAmB,CAAC,EAAE,SAAS,CAAC,CAAC;YAChF,MAAM,IAAI,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,aAAa,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM;gBAC3E,OAAO,EAAE,GAAG,EAAE,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,mBAAmB,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YACzF,OAAO,GAAG,IAAI,CAAC;YAAC,MAAM,EAAE,gBAAgB,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;YACzE,KAAK,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE;gBACrE,IAAI,OAAO,KAAK,IAAI;oBAAE,OAAO;gBAC7B,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;gBAAC,MAAM,EAAE,CAAC;YACtC,CAAC,EAAE,GAAG,EAAE,GAAG,IAAI,OAAO,KAAK,IAAI;gBAAE,SAAS,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC3E,CAAC,CAAC,CAAC;IACL,CAAC,CAAC;IACF,MAAM,aAAa,GAAG,KAAK,EAAK,IAAsB,EAAc,EAAE;QACpE,IAAI,MAAM;YAAE,MAAM,IAAI,gBAAgB,CAAC,iBAAiB,EAAE,gBAAgB,CAAC,CAAC;QAC5E,IAAI,IAAI;YAAE,MAAM,IAAI,gBAAgB,CAAC,eAAe,EAAE,gBAAgB,CAAC,CAAC;QACxE,IAAI,GAAG,IAAI,CAAC;QACZ,IAAI,CAAC;YAAC,OAAO,MAAM,IAAI,EAAE,CAAC;QAAC,CAAC;gBAAS,CAAC;YAAC,IAAI,GAAG,KAAK,CAAC;QAAC,CAAC;IACxD,CAAC,CAAC;IACF,MAAM,WAAW,GAAG,KAAK,EAAE,MAAoB,EAAiB,EAAE;QAChE,IAAI,KAAK;YAAE,OAAO;QAClB,MAAM,QAAQ,CAAC,OAAO,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;QAAC,KAAK,GAAG,IAAI,CAAC;IAC3D,CAAC,CAAC;IACF,MAAM,QAAQ,GAA4D;QACxE,aAAa,EAAE,aAAa;QAC5B,OAAO,EAAE,GAAG,EAAE,CAAC,aAAa,CAAC,KAAK,IAA6B,EAAE;YAC/D,MAAM,WAAW,EAAE,CAAC;YACpB,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,SAAS,CAAC,CAAC;YACxC,IAAI,CAAC;gBACH,IAAI,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,WAAW;oBAAE,MAAM,IAAI,KAAK,EAAE,CAAC;gBACvD,OAAO,+BAA+B,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;YAC5D,CAAC;YAAC,MAAM,CAAC;gBAAC,SAAS,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;gBAAC,MAAM,IAAI,gBAAgB,CAAC,kBAAkB,EAAE,mBAAmB,CAAC,CAAC;YAAC,CAAC;QACvH,CAAC,CAAC;QACF,OAAO,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,CAAC,aAAa,CAAC,KAAK,IAAI,EAAE;YACpD,MAAM,WAAW,GAAG,4BAA4B,CAAC,MAAM,CAAC,CAAC;YACzD,IAAI,MAAM,EAAE,OAAO;gBAAE,MAAM,IAAI,gBAAgB,CAAC,WAAW,EAAE,gBAAgB,CAAC,CAAC;YAC/E,IAAI,CAAC;gBAAC,MAAM,WAAW,CAAC,MAAM,CAAC,CAAC;YAAC,CAAC;YAClC,OAAO,KAAK,EAAE,CAAC;gBACb,IAAI,KAAK,YAAY,gBAAgB;oBAAE,MAAM,IAAI,gBAAgB,CAAC,KAAK,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAAC;gBAChG,MAAM,KAAK,CAAC;YACd,CAAC;YACD,MAAM,QAAQ,CAAC,SAAS,EAAE,WAAW,EAAE,MAAM,CAAC,CAAC;QACjD,CAAC,CAAC;KACH,CAAC;IACF,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,aAAa,CAAC,WAAW,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,SAAS,CAAC,KAAK,EAAE,EAAE,CAAC;AAC/F,CAAC"}
+{"version":3,"file":"browser-control-client.js","sourceRoot":"","sources":["../src/browser-control-client.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAC3D,OAAO,EACL,sBAAsB,EAAE,uBAAuB,EAAE,+BAA+B,EAChF,wBAAwB,EAAE,0BAA0B,EAAE,4BAA4B,EAAE,8BAA8B,EAEnH,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAAE,uBAAuB,EAAE,MAAM,gCAAgC,CAAC;AAWzE,MAAM,UAAU,0BAA0B,CAAC,OAAoC;IAC7E,MAAM,QAAQ,GAAG,8BAA8B,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAClE,MAAM,SAAS,GAAG,OAAO,CAAC,gBAAgB,IAAI,sBAAsB,CAAC,gBAAgB,CAAC;IACtF,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,SAAS,GAAG,CAAC,IAAI,SAAS,GAAG,sBAAsB,CAAC,mBAAmB;QAAE,MAAM,IAAI,gBAAgB,CAAC,iBAAiB,EAAE,gBAAgB,CAAC,CAAC;IAC5K,IAAI,GAAG,GAAG,CAAC,EAAE,IAAI,GAAG,KAAK,EAAE,KAAK,GAAG,KAAK,EAAE,MAAM,GAAG,KAAK,CAAC;IACzD,IAAI,OACyH,CAAC;IAC9H,MAAM,MAAM,GAAG,GAAS,EAAE;QACxB,IAAI,CAAC,OAAO,EAAE,KAAK,IAAI,CAAC,OAAO,CAAC,aAAa;YAAE,OAAO;QACtD,MAAM,SAAS,GAAG,OAAO,CAAC;QAAC,OAAO,GAAG,SAAS,CAAC;QAAC,SAAS,CAAC,OAAO,EAAE,CAAC;QACpE,IAAI,SAAS,CAAC,KAAM,CAAC,EAAE;YAAE,SAAS,CAAC,OAAO,CAAC,SAAS,CAAC,KAAM,CAAC,CAAC;aACxD,CAAC;YACJ,MAAM,EAAE,IAAI,EAAE,WAAW,EAAE,GAAG,SAAS,CAAC,KAAM,CAAC,KAAK,CAAC;YACrD,IAAI,SAAS,CAAC,OAAO,CAAC,SAAS,KAAK,SAAS,IAAI,IAAI,KAAK,iBAAiB,IAAI,WAAW,KAAK,gBAAgB;gBAAE,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YACvI,SAAS,CAAC,MAAM,CAAC,IAAI,gBAAgB,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC,CAAC;QAC5D,CAAC;IACH,CAAC,CAAC;IACF,MAAM,SAAS,GAAG,IAAI,uBAAuB,CAAC,OAAO,CAAC,SAAS,EAAE,KAAK,CAAC,EAAE;QACvE,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;YAAC,SAAS,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;YAAC,OAAO;QAAC,CAAC;QAC/E,IAAI,KAA0B,CAAC;QAC/B,IAAI,CAAC;YAAC,KAAK,GAAG,wBAAwB,CAAC,KAAK,CAAC,CAAC;QAAC,CAAC;QAAC,MAAM,CAAC;YAAC,SAAS,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;YAAC,OAAO;QAAC,CAAC;QACvG,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;QAChC,IAAI,CAAC,0BAA0B,CAAC,QAAQ,EAAE,KAAK,CAAC,QAAQ,CAAC,IAAI,OAAO,CAAC,GAAG,KAAK,KAAK,CAAC,GAAG,IAAI,OAAO,CAAC,SAAS,KAAK,KAAK,CAAC,SAAS;eAC1H,OAAO,CAAC,SAAS,KAAK,KAAK,CAAC,SAAS,IAAI,CAAC,OAAO,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,KAAK,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,KAAK,SAAS,CAAC,EAAE,CAAC;YACrJ,SAAS,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC;YAAC,OAAO;QAC/C,CAAC;QACD,OAAO,CAAC,KAAK,GAAG,KAAK,CAAC;QAAC,MAAM,EAAE,CAAC;IAClC,CAAC,EAAE,IAAI,CAAC,EAAE;QACR,MAAM,GAAG,IAAI,CAAC;QACd,MAAM,SAAS,GAAG,OAAO,CAAC;QAAC,OAAO,GAAG,SAAS,CAAC;QAC/C,IAAI,SAAS,EAAE,CAAC;YAAC,SAAS,CAAC,OAAO,EAAE,CAAC;YAAC,SAAS,CAAC,MAAM,CAAC,IAAI,gBAAgB,CAAC,IAAI,EAAE,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC;QAAC,CAAC;IACnJ,CAAC,CAAC,CAAC;IACH,MAAM,QAAQ,GAAG,CAAC,SAA0C,EAAE,MAAkB,EAAE,MAAoB,EAAgC,EAAE;QACtI,IAAI,MAAM;YAAE,OAAO,OAAO,CAAC,MAAM,CAAC,IAAI,gBAAgB,CAAC,iBAAiB,EAAE,gBAAgB,CAAC,CAAC,CAAC;QAC7F,IAAI,MAAM,EAAE,OAAO;YAAE,OAAO,OAAO,CAAC,MAAM,CAAC,IAAI,gBAAgB,CAAC,WAAW,EAAE,gBAAgB,CAAC,CAAC,CAAC;QAChG,IAAI,GAAG,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;YAAC,SAAS,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC;YAAC,OAAO,OAAO,CAAC,MAAM,CAAC,IAAI,gBAAgB,CAAC,mBAAmB,EAAE,gBAAgB,CAAC,CAAC,CAAC;QAAC,CAAC;QACjK,GAAG,IAAI,CAAC,CAAC;QACT,MAAM,IAAI,GAAG,EAAE,OAAO,EAAE,uBAAyD,EAAE,IAAI,EAAE,SAAkB,EAAE,QAAQ,EAAE,GAAG,EAAE,SAAS,EAAE,WAAW,GAAG,EAAE,EAAE,CAAC;QAC1J,MAAM,OAAO,GAA0B,SAAS,KAAK,SAAS;YAC5D,CAAC,CAAC,EAAE,GAAG,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,UAAU,GAAG,EAAE,EAAE,MAAM,EAAE,MAAO,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,EAAE,SAAS,EAAE,CAAC;QAChG,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,MAAM,KAAK,GAAG,GAAS,EAAE,CAAC,SAAS,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;YACvD,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,KAAK,CAAC,mBAAmB,CAAC,EAAE,SAAS,CAAC,CAAC;YAChF,MAAM,IAAI,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,aAAa,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM;gBAC3E,OAAO,EAAE,GAAG,EAAE,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,mBAAmB,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YACzF,OAAO,GAAG,IAAI,CAAC;YAAC,MAAM,EAAE,gBAAgB,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;YACzE,KAAK,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE;gBACrE,IAAI,OAAO,KAAK,IAAI;oBAAE,OAAO;gBAC7B,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;gBAAC,MAAM,EAAE,CAAC;YACtC,CAAC,EAAE,GAAG,EAAE,GAAG,IAAI,OAAO,KAAK,IAAI;gBAAE,SAAS,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC3E,CAAC,CAAC,CAAC;IACL,CAAC,CAAC;IACF,MAAM,aAAa,GAAG,KAAK,EAAK,IAAsB,EAAc,EAAE;QACpE,IAAI,MAAM;YAAE,MAAM,IAAI,gBAAgB,CAAC,iBAAiB,EAAE,gBAAgB,CAAC,CAAC;QAC5E,IAAI,IAAI;YAAE,MAAM,IAAI,gBAAgB,CAAC,eAAe,EAAE,gBAAgB,CAAC,CAAC;QACxE,IAAI,GAAG,IAAI,CAAC;QACZ,IAAI,CAAC;YAAC,OAAO,MAAM,IAAI,EAAE,CAAC;QAAC,CAAC;gBAAS,CAAC;YAAC,IAAI,GAAG,KAAK,CAAC;QAAC,CAAC;IACxD,CAAC,CAAC;IACF,MAAM,WAAW,GAAG,KAAK,EAAE,MAAoB,EAAiB,EAAE;QAChE,IAAI,KAAK;YAAE,OAAO;QAClB,MAAM,QAAQ,CAAC,OAAO,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;QAAC,KAAK,GAAG,IAAI,CAAC;IAC3D,CAAC,CAAC;IACF,MAAM,QAAQ,GAA4D;QACxE,aAAa,EAAE,aAAa;QAC5B,GAAG,CAAC,OAAO,CAAC,aAAa,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,IAAa,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC3E,OAAO,EAAE,GAAG,EAAE,CAAC,aAAa,CAAC,KAAK,IAA6B,EAAE;YAC/D,MAAM,WAAW,EAAE,CAAC;YACpB,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,SAAS,CAAC,CAAC;YACxC,IAAI,CAAC;gBACH,IAAI,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,WAAW;oBAAE,MAAM,IAAI,KAAK,EAAE,CAAC;gBACvD,OAAO,+BAA+B,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;YAC5D,CAAC;YAAC,MAAM,CAAC;gBAAC,SAAS,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;gBAAC,MAAM,IAAI,gBAAgB,CAAC,kBAAkB,EAAE,mBAAmB,CAAC,CAAC;YAAC,CAAC;QACvH,CAAC,CAAC;QACF,OAAO,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,CAAC,aAAa,CAAC,KAAK,IAAI,EAAE;YACpD,MAAM,WAAW,GAAG,4BAA4B,CAAC,MAAM,CAAC,CAAC;YACzD,IAAI,WAAW,CAAC,IAAI,KAAK,OAAO,IAAI,OAAO,CAAC,aAAa,KAAK,IAAI,EAAE,CAAC;gBACnE,MAAM,IAAI,gBAAgB,CAAC,iBAAiB,EAAE,gBAAgB,CAAC,CAAC;YAClE,CAAC;YACD,IAAI,MAAM,EAAE,OAAO;gBAAE,MAAM,IAAI,gBAAgB,CAAC,WAAW,EAAE,gBAAgB,CAAC,CAAC;YAC/E,IAAI,CAAC;gBAAC,MAAM,WAAW,CAAC,MAAM,CAAC,CAAC;YAAC,CAAC;YAClC,OAAO,KAAK,EAAE,CAAC;gBACb,IAAI,KAAK,YAAY,gBAAgB;oBAAE,MAAM,IAAI,gBAAgB,CAAC,KAAK,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAAC;gBAChG,MAAM,KAAK,CAAC;YACd,CAAC;YACD,MAAM,QAAQ,CAAC,SAAS,EAAE,WAAW,EAAE,MAAM,CAAC,CAAC;QACjD,CAAC,CAAC;KACH,CAAC;IACF,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,aAAa,CAAC,WAAW,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,SAAS,CAAC,KAAK,EAAE,EAAE,CAAC;AAC/F,CAAC"}

package/dist/browser-control-dispatcher.js

@@ -70,6 +70,9 @@ export function attachBrowserControlDispatcher(options) {
                 reply = { ...common, ok: true, observation: encodeBrowserControlObservation(observation) };
             }
             else {
+                if (request.action.kind === "speak" && options.executor.speechEnabled !== true) {
+                    throw new CuaExecutorError("action_rejected", "not_dispatched");
+                }
                 // No await between the owner gate above and invocation. The physical driver must
                 // check this signal again immediately before each actual input after preparation.
                 invoked = true;

package/dist/browser-control-dispatcher.js.map

@@ -1 +1 @@
-{"version":3,"file":"browser-control-dispatcher.js","sourceRoot":"","sources":["../src/browser-control-dispatcher.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAC3D,OAAO,EACL,sBAAsB,EAAE,uBAAuB,EAAE,+BAA+B,EAAE,0BAA0B,EAC5G,yBAAyB,EAAE,0BAA0B,EAAE,8BAA8B,EAEtF,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAAE,uBAAuB,EAAE,MAAM,gCAAgC,CAAC;AAMzE,0GAA0G;AAC1G,MAAM,UAAU,8BAA8B,CAAC,OAAwC;IACrF,MAAM,QAAQ,GAAG,8BAA8B,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAClE,IAAI,OAAO,GAAG,CAAC,EAAE,IAAI,GAAG,KAAK,EAAE,KAAK,GAAG,KAAK,EAAE,MAAM,GAAG,KAAK,CAAC;IAC7D,IAAI,MAAmC,CAAC;IACxC,IAAI,KAAgD,CAAC;IACrD,MAAM,OAAO,GAAG,GAAS,EAAE,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,GAAG,SAAS,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;IAC7G,MAAM,UAAU,GAAG,GAAY,EAAE;QAC/B,IAAI,CAAC;YAAC,OAAO,CAAC,OAAO,CAAC,eAAe,CAAC,OAAO,IAAI,OAAO,CAAC,YAAY,EAAE,KAAK,IAAI,CAAC;QAAC,CAAC;QAAC,MAAM,CAAC;YAAC,OAAO,KAAK,CAAC;QAAC,CAAC;IAC7G,CAAC,CAAC;IACF,MAAM,SAAS,GAAG,IAAI,uBAAuB,CAAC,OAAO,CAAC,SAAS,EAAE,KAAK,CAAC,EAAE;QACvE,IAAI,MAAM;YAAE,OAAO;QACnB,IAAI,IAAI,EAAE,CAAC;YAAC,SAAS,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;YAAC,OAAO;QAAC,CAAC;QACvD,IAAI,OAA8B,CAAC;QACnC,IAAI,CAAC;YAAC,OAAO,GAAG,0BAA0B,CAAC,KAAK,CAAC,CAAC;QAAC,CAAC;QAAC,MAAM,CAAC;YAAC,SAAS,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;YAAC,OAAO;QAAC,CAAC;QAC1G,IAAI,CAAC,0BAA0B,CAAC,QAAQ,EAAE,OAAO,CAAC,QAAQ,CAAC,IAAI,OAAO,CAAC,GAAG,KAAK,OAAO,GAAG,CAAC;eACrF,CAAC,OAAO,CAAC,SAAS,KAAK,OAAO,CAAC,CAAC,CAAC,KAAK,IAAI,OAAO,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC;YAAC,SAAS,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC;YAAC,OAAO;QAAC,CAAC;QAC7H,sFAAsF;QACtF,IAAI,GAAG,IAAI,CAAC;QAAC,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC;QAAC,MAAM,GAAG,IAAI,eAAe,EAAE,CAAC;QACnE,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,KAAK,CAAC,mBAAmB,CAAC,EAAE,sBAAsB,CAAC,gBAAgB,CAAC,CAAC;QACxG,KAAK,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;IACxC,CAAC,EAAE,GAAG,EAAE;QACN,MAAM,GAAG,IAAI,CAAC;QAAC,OAAO,EAAE,CAAC;QAAC,OAAO,CAAC,eAAe,CAAC,mBAAmB,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IACzF,CAAC,CAAC,CAAC;IACH,SAAS,MAAM,KAAW,SAAS,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC;IAC/D,KAAK,UAAU,QAAQ,CAAC,OAA8B,EAAE,MAAmB;QACzE,MAAM,MAAM,GAAG,EAAE,OAAO,EAAE,uBAAyD,EAAE,IAAI,EAAE,OAAgB,EAAE,QAAQ,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG;YACrI,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS;YAC1D,GAAG,CAAC,OAAO,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;QAC/E,IAAI,OAAO,GAAG,KAAK,EAAE,KAA0B,CAAC;QAChD,IAAI,CAAC;YACH,IAAI,CAAC,UAAU,EAAE,IAAI,MAAM,CAAC,OAAO;gBAAE,MAAM,IAAI,gBAAgB,CAAC,iBAAiB,EAAE,gBAAgB,CAAC,CAAC;YACrG,IAAI,OAAO,CAAC,SAAS,KAAK,OAAO,EAAE,CAAC;gBAAC,KAAK,GAAG,IAAI,CAAC;gBAAC,KAAK,GAAG,EAAE,GAAG,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC;YAAC,CAAC;iBAChF,IAAI,OAAO,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;gBACzC,OAAO,GAAG,IAAI,CAAC;gBACf,MAAM,WAAW,GAAG,MAAM,OAAO,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC;gBACrD,IAAI,MAAM,IAAI,MAAM,CAAC,OAAO,IAAI,CAAC,UAAU,EAAE,EAAE,CAAC;oBAAC,SAAS,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;oBAAC,OAAO;gBAAC,CAAC;gBAC9F,KAAK,GAAG,EAAE,GAAG,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,+BAA+B,CAAC,WAAW,CAAC,EAAE,CAAC;YAC7F,CAAC;iBAAM,CAAC;gBACN,iFAAiF;gBACjF,kFAAkF;gBAClF,OAAO,GAAG,IAAI,CAAC;gBACf,MAAM,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;gBACvD,IAAI,MAAM,IAAI,MAAM,CAAC,OAAO,IAAI,CAAC,UAAU,EAAE,EAAE,CAAC;oBAAC,SAAS,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;oBAAC,OAAO;gBAAC,CAAC;gBAC9F,KAAK,GAAG,EAAE,GAAG,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC;YAClC,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAAC,KAAK,GAAG,EAAE,GAAG,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,yBAAyB,CAAC,KAAK,EAAE,OAAO,CAAC,EAAE,CAAC;QAAC,CAAC;QACvG,IAAI,MAAM;YAAE,OAAO;QACnB,IAAI,CAAC;YACH,MAAM,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC5B,IAAI,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,SAAS,KAAK,SAAS,IAAI,KAAK,CAAC,KAAK,CAAC,IAAI,KAAK,iBAAiB,IAAI,KAAK,CAAC,KAAK,CAAC,WAAW,KAAK,gBAAgB,CAAC,EAAE,CAAC;gBAC7I,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBAAC,OAAO;YAC5C,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YAAC,SAAS,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;QAAC,CAAC;gBACxC,CAAC;YAAC,OAAO,EAAE,CAAC;YAAC,IAAI,GAAG,KAAK,CAAC;QAAC,CAAC;IACtC,CAAC;IACD,OAAO,CAAC,eAAe,CAAC,gBAAgB,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;IAC1E,IAAI,CAAC,UAAU,EAAE;QAAE,SAAS,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;IACtD,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,SAAS,CAAC,KAAK,EAAE,EAAE,CAAC;AAC5C,CAAC"}
+{"version":3,"file":"browser-control-dispatcher.js","sourceRoot":"","sources":["../src/browser-control-dispatcher.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAC3D,OAAO,EACL,sBAAsB,EAAE,uBAAuB,EAAE,+BAA+B,EAAE,0BAA0B,EAC5G,yBAAyB,EAAE,0BAA0B,EAAE,8BAA8B,EAEtF,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAAE,uBAAuB,EAAE,MAAM,gCAAgC,CAAC;AAMzE,0GAA0G;AAC1G,MAAM,UAAU,8BAA8B,CAAC,OAAwC;IACrF,MAAM,QAAQ,GAAG,8BAA8B,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAClE,IAAI,OAAO,GAAG,CAAC,EAAE,IAAI,GAAG,KAAK,EAAE,KAAK,GAAG,KAAK,EAAE,MAAM,GAAG,KAAK,CAAC;IAC7D,IAAI,MAAmC,CAAC;IACxC,IAAI,KAAgD,CAAC;IACrD,MAAM,OAAO,GAAG,GAAS,EAAE,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,GAAG,SAAS,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;IAC7G,MAAM,UAAU,GAAG,GAAY,EAAE;QAC/B,IAAI,CAAC;YAAC,OAAO,CAAC,OAAO,CAAC,eAAe,CAAC,OAAO,IAAI,OAAO,CAAC,YAAY,EAAE,KAAK,IAAI,CAAC;QAAC,CAAC;QAAC,MAAM,CAAC;YAAC,OAAO,KAAK,CAAC;QAAC,CAAC;IAC7G,CAAC,CAAC;IACF,MAAM,SAAS,GAAG,IAAI,uBAAuB,CAAC,OAAO,CAAC,SAAS,EAAE,KAAK,CAAC,EAAE;QACvE,IAAI,MAAM;YAAE,OAAO;QACnB,IAAI,IAAI,EAAE,CAAC;YAAC,SAAS,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;YAAC,OAAO;QAAC,CAAC;QACvD,IAAI,OAA8B,CAAC;QACnC,IAAI,CAAC;YAAC,OAAO,GAAG,0BAA0B,CAAC,KAAK,CAAC,CAAC;QAAC,CAAC;QAAC,MAAM,CAAC;YAAC,SAAS,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;YAAC,OAAO;QAAC,CAAC;QAC1G,IAAI,CAAC,0BAA0B,CAAC,QAAQ,EAAE,OAAO,CAAC,QAAQ,CAAC,IAAI,OAAO,CAAC,GAAG,KAAK,OAAO,GAAG,CAAC;eACrF,CAAC,OAAO,CAAC,SAAS,KAAK,OAAO,CAAC,CAAC,CAAC,KAAK,IAAI,OAAO,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC;YAAC,SAAS,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC;YAAC,OAAO;QAAC,CAAC;QAC7H,sFAAsF;QACtF,IAAI,GAAG,IAAI,CAAC;QAAC,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC;QAAC,MAAM,GAAG,IAAI,eAAe,EAAE,CAAC;QACnE,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,KAAK,CAAC,mBAAmB,CAAC,EAAE,sBAAsB,CAAC,gBAAgB,CAAC,CAAC;QACxG,KAAK,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;IACxC,CAAC,EAAE,GAAG,EAAE;QACN,MAAM,GAAG,IAAI,CAAC;QAAC,OAAO,EAAE,CAAC;QAAC,OAAO,CAAC,eAAe,CAAC,mBAAmB,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IACzF,CAAC,CAAC,CAAC;IACH,SAAS,MAAM,KAAW,SAAS,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC;IAC/D,KAAK,UAAU,QAAQ,CAAC,OAA8B,EAAE,MAAmB;QACzE,MAAM,MAAM,GAAG,EAAE,OAAO,EAAE,uBAAyD,EAAE,IAAI,EAAE,OAAgB,EAAE,QAAQ,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG;YACrI,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS;YAC1D,GAAG,CAAC,OAAO,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;QAC/E,IAAI,OAAO,GAAG,KAAK,EAAE,KAA0B,CAAC;QAChD,IAAI,CAAC;YACH,IAAI,CAAC,UAAU,EAAE,IAAI,MAAM,CAAC,OAAO;gBAAE,MAAM,IAAI,gBAAgB,CAAC,iBAAiB,EAAE,gBAAgB,CAAC,CAAC;YACrG,IAAI,OAAO,CAAC,SAAS,KAAK,OAAO,EAAE,CAAC;gBAAC,KAAK,GAAG,IAAI,CAAC;gBAAC,KAAK,GAAG,EAAE,GAAG,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC;YAAC,CAAC;iBAChF,IAAI,OAAO,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;gBACzC,OAAO,GAAG,IAAI,CAAC;gBACf,MAAM,WAAW,GAAG,MAAM,OAAO,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC;gBACrD,IAAI,MAAM,IAAI,MAAM,CAAC,OAAO,IAAI,CAAC,UAAU,EAAE,EAAE,CAAC;oBAAC,SAAS,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;oBAAC,OAAO;gBAAC,CAAC;gBAC9F,KAAK,GAAG,EAAE,GAAG,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,+BAA+B,CAAC,WAAW,CAAC,EAAE,CAAC;YAC7F,CAAC;iBAAM,CAAC;gBACN,IAAI,OAAO,CAAC,MAAM,CAAC,IAAI,KAAK,OAAO,IAAI,OAAO,CAAC,QAAQ,CAAC,aAAa,KAAK,IAAI,EAAE,CAAC;oBAC/E,MAAM,IAAI,gBAAgB,CAAC,iBAAiB,EAAE,gBAAgB,CAAC,CAAC;gBAClE,CAAC;gBACD,iFAAiF;gBACjF,kFAAkF;gBAClF,OAAO,GAAG,IAAI,CAAC;gBACf,MAAM,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;gBACvD,IAAI,MAAM,IAAI,MAAM,CAAC,OAAO,IAAI,CAAC,UAAU,EAAE,EAAE,CAAC;oBAAC,SAAS,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;oBAAC,OAAO;gBAAC,CAAC;gBAC9F,KAAK,GAAG,EAAE,GAAG,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC;YAClC,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAAC,KAAK,GAAG,EAAE,GAAG,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,yBAAyB,CAAC,KAAK,EAAE,OAAO,CAAC,EAAE,CAAC;QAAC,CAAC;QACvG,IAAI,MAAM;YAAE,OAAO;QACnB,IAAI,CAAC;YACH,MAAM,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC5B,IAAI,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,SAAS,KAAK,SAAS,IAAI,KAAK,CAAC,KAAK,CAAC,IAAI,KAAK,iBAAiB,IAAI,KAAK,CAAC,KAAK,CAAC,WAAW,KAAK,gBAAgB,CAAC,EAAE,CAAC;gBAC7I,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBAAC,OAAO;YAC5C,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YAAC,SAAS,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;QAAC,CAAC;gBACxC,CAAC;YAAC,OAAO,EAAE,CAAC;YAAC,IAAI,GAAG,KAAK,CAAC;QAAC,CAAC;IACtC,CAAC;IACD,OAAO,CAAC,eAAe,CAAC,gBAAgB,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;IAC1E,IAAI,CAAC,UAAU,EAAE;QAAE,SAAS,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;IACtD,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,SAAS,CAAC,KAAK,EAAE,EAAE,CAAC;AAC5C,CAAC"}

package/dist/browser-control-protocol.d.ts

@@ -1,5 +1,6 @@
 import { z } from "zod";
 import type { CuaAction, CuaObservation } from "./computer-use.js";
+import { type HeardSpeech } from "./cua-speech.js";
 import { type CuaExecutorErrorCode } from "./cua-executor-error.js";
 export declare const BROWSER_CONTROL_VERSION = 1;
 export declare const BROWSER_CONTROL_LIMITS: Readonly<{
@@ -8,6 +9,10 @@ export declare const BROWSER_CONTROL_LIMITS: Readonly<{
     dimension: 4096;
     pixels: 16000000;
     textBytes: number;
+    speechCharacters: 400;
+    speechBytes: 1600;
+    heardSpeech: 4;
+    speechDurationMs: 120000;
     chordKeys: 16;
     keyCharacters: 64;
     dragPoints: 1024;
@@ -20,6 +25,47 @@ export interface BrowserControlIdentity {
     challenge: string;
     runtimeRevision: string;
 }
+export declare const browserOnlyControlActionSchema: z.ZodDiscriminatedUnion<readonly [z.ZodObject<{
+    button: z.ZodOptional<z.ZodEnum<{
+        left: "left";
+        right: "right";
+        middle: "middle";
+    }>>;
+    x: z.ZodNumber;
+    y: z.ZodNumber;
+    kind: z.ZodLiteral<"click">;
+}, z.core.$strict>, z.ZodObject<{
+    x: z.ZodNumber;
+    y: z.ZodNumber;
+    kind: z.ZodLiteral<"double_click">;
+}, z.core.$strict>, z.ZodObject<{
+    x: z.ZodNumber;
+    y: z.ZodNumber;
+    kind: z.ZodLiteral<"move">;
+}, z.core.$strict>, z.ZodObject<{
+    dx: z.ZodNumber;
+    dy: z.ZodNumber;
+    x: z.ZodNumber;
+    y: z.ZodNumber;
+    kind: z.ZodLiteral<"scroll">;
+}, z.core.$strict>, z.ZodObject<{
+    kind: z.ZodLiteral<"type">;
+    text: z.ZodString;
+}, z.core.$strict>, z.ZodObject<{
+    kind: z.ZodLiteral<"keypress">;
+    keys: z.ZodArray<z.ZodString>;
+}, z.core.$strict>, z.ZodObject<{
+    kind: z.ZodLiteral<"drag">;
+    path: z.ZodArray<z.ZodObject<{
+        x: z.ZodNumber;
+        y: z.ZodNumber;
+    }, z.core.$strict>>;
+}, z.core.$strict>, z.ZodObject<{
+    kind: z.ZodLiteral<"wait">;
+    ms: z.ZodOptional<z.ZodNumber>;
+}, z.core.$strict>, z.ZodObject<{
+    kind: z.ZodLiteral<"screenshot">;
+}, z.core.$strict>], "kind">;
 export declare const browserControlActionSchema: z.ZodPipe<z.ZodDiscriminatedUnion<[z.ZodObject<{
     button: z.ZodOptional<z.ZodEnum<{
         left: "left";
@@ -60,6 +106,9 @@ export declare const browserControlActionSchema: z.ZodPipe<z.ZodDiscriminatedUni
     ms: z.ZodOptional<z.ZodNumber>;
 }, z.core.$strict>, z.ZodObject<{
     kind: z.ZodLiteral<"screenshot">;
+}, z.core.$strict>, z.ZodObject<{
+    kind: z.ZodLiteral<"speak">;
+    text: z.ZodString;
 }, z.core.$strict>], "kind">, z.ZodTransform<{
     kind: "click";
     x: number;
@@ -94,6 +143,9 @@ export declare const browserControlActionSchema: z.ZodPipe<z.ZodDiscriminatedUni
 } | {
     kind: "wait";
     ms?: number;
+} | {
+    kind: "speak";
+    text: string;
 } | {
     kind: "screenshot";
 }, {
@@ -132,6 +184,9 @@ export declare const browserControlActionSchema: z.ZodPipe<z.ZodDiscriminatedUni
     ms?: number | undefined;
 } | {
     kind: "screenshot";
+} | {
+    kind: "speak";
+    text: string;
 }>>;
 declare const observationSchema: z.ZodObject<{
     png: z.ZodString;
@@ -140,6 +195,12 @@ declare const observationSchema: z.ZodObject<{
     title: z.ZodOptional<z.ZodString>;
     text: z.ZodOptional<z.ZodString>;
     scrollY: z.ZodOptional<z.ZodNumber>;
+    heardSpeech: z.ZodOptional<z.ZodArray<z.ZodObject<{
+        id: z.ZodString;
+        source: z.ZodLiteral<"speaker_audio">;
+        text: z.ZodString;
+        durationMs: z.ZodNumber;
+    }, z.core.$strict>>>;
 }, z.core.$strict>;
 type WireObservation = z.infer<typeof observationSchema>;
 declare const requestSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
@@ -208,6 +269,9 @@ declare const requestSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
         ms: z.ZodOptional<z.ZodNumber>;
     }, z.core.$strict>, z.ZodObject<{
         kind: z.ZodLiteral<"screenshot">;
+    }, z.core.$strict>, z.ZodObject<{
+        kind: z.ZodLiteral<"speak">;
+        text: z.ZodString;
     }, z.core.$strict>], "kind">, z.ZodTransform<{
         kind: "click";
         x: number;
@@ -242,6 +306,9 @@ declare const requestSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
     } | {
         kind: "wait";
         ms?: number;
+    } | {
+        kind: "speak";
+        text: string;
     } | {
         kind: "screenshot";
     }, {
@@ -280,6 +347,9 @@ declare const requestSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
         ms?: number | undefined;
     } | {
         kind: "screenshot";
+    } | {
+        kind: "speak";
+        text: string;
     }>>;
     version: z.ZodLiteral<1>;
     identity: z.ZodObject<{
@@ -299,6 +369,12 @@ declare const replySchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
         title: z.ZodOptional<z.ZodString>;
         text: z.ZodOptional<z.ZodString>;
         scrollY: z.ZodOptional<z.ZodNumber>;
+        heardSpeech: z.ZodOptional<z.ZodArray<z.ZodObject<{
+            id: z.ZodString;
+            source: z.ZodLiteral<"speaker_audio">;
+            text: z.ZodString;
+            durationMs: z.ZodNumber;
+        }, z.core.$strict>>>;
     }, z.core.$strict>>;
     type: z.ZodLiteral<"reply">;
     operation: z.ZodEnum<{
@@ -360,6 +436,7 @@ export declare function sameBrowserControlIdentity(a: BrowserControlIdentity, b:
 export declare function parseBrowserControlRequest(value: unknown): BrowserControlRequest;
 export declare function parseBrowserControlReply(value: unknown): BrowserControlReply;
 export declare function validateBrowserControlAction(value: unknown): CuaAction;
+export declare function validateHeardSpeech(value: unknown): HeardSpeech[];
 /** Check dimensions before allocating decoder output; v1 accepts browser-style 8-bit, noninterlaced PNG. */
 export declare function validateBrowserControlPng(bytes: Buffer): void;
 export declare function encodeBrowserControlObservation(observation: CuaObservation): WireObservation;

package/dist/browser-control-protocol.js

@@ -1,19 +1,24 @@
 import { PNG } from "pngjs";
 import { z } from "zod";
+import { CUA_SPEECH_LIMITS } from "./cua-speech.js";
 import { CuaExecutorError, isCuaExecutorError } from "./cua-executor-error.js";
 export const BROWSER_CONTROL_VERSION = 1;
 export const BROWSER_CONTROL_LIMITS = Object.freeze({
     frameBytes: 12 * 1024 * 1024, pngBytes: 8 * 1024 * 1024,
     dimension: 4096, pixels: 16_000_000, textBytes: 64 * 1024,
+    speechCharacters: CUA_SPEECH_LIMITS.characters, speechBytes: CUA_SPEECH_LIMITS.bytes,
+    heardSpeech: CUA_SPEECH_LIMITS.utterances, speechDurationMs: CUA_SPEECH_LIMITS.durationMs,
     chordKeys: 16, keyCharacters: 64, dragPoints: 1024, waitMs: 30_000,
     requestTimeoutMs: 35_000, maxRequestTimeoutMs: 60_000
 });
 const token = z.string().min(1).max(128).regex(/^[A-Za-z0-9._-]+$/);
 const identitySchema = z.strictObject({ generation: token, challenge: token, runtimeRevision: token });
 const text = z.string().max(BROWSER_CONTROL_LIMITS.textBytes).refine(value => Buffer.byteLength(value) <= BROWSER_CONTROL_LIMITS.textBytes);
+const speechText = z.string().min(1).max(BROWSER_CONTROL_LIMITS.speechCharacters)
+    .refine(value => value.trim().length > 0 && Buffer.byteLength(value) <= BROWSER_CONTROL_LIMITS.speechBytes);
 const coordinate = z.number().finite().min(-1_000_000).max(1_000_000);
 const point = { x: coordinate, y: coordinate };
-export const browserControlActionSchema = z.discriminatedUnion("kind", [
+const browserActionSchemas = [
     z.strictObject({ kind: z.literal("click"), ...point, button: z.enum(["left", "right", "middle"]).optional() }),
     z.strictObject({ kind: z.literal("double_click"), ...point }),
     z.strictObject({ kind: z.literal("move"), ...point }),
@@ -23,6 +28,11 @@ export const browserControlActionSchema = z.discriminatedUnion("kind", [
     z.strictObject({ kind: z.literal("drag"), path: z.array(z.strictObject(point)).min(1).max(BROWSER_CONTROL_LIMITS.dragPoints) }),
     z.strictObject({ kind: z.literal("wait"), ms: z.number().finite().min(0).max(BROWSER_CONTROL_LIMITS.waitMs).optional() }),
     z.strictObject({ kind: z.literal("screenshot") })
+];
+export const browserOnlyControlActionSchema = z.discriminatedUnion("kind", browserActionSchemas);
+export const browserControlActionSchema = z.discriminatedUnion("kind", [
+    ...browserActionSchemas,
+    z.strictObject({ kind: z.literal("speak"), text: speechText })
 ]).transform((action) => {
     if (action.kind === "click")
         return { kind: action.kind, x: action.x, y: action.y, ...(action.button !== undefined ? { button: action.button } : {}) };
@@ -30,9 +40,18 @@ export const browserControlActionSchema = z.discriminatedUnion("kind", [
         return { kind: action.kind, ...(action.ms !== undefined ? { ms: action.ms } : {}) };
     return action;
 });
+const heardSpeechSchema = z.strictObject({
+    id: token,
+    source: z.literal("speaker_audio"),
+    text: speechText,
+    durationMs: z.number().int().min(1).max(BROWSER_CONTROL_LIMITS.speechDurationMs)
+});
+const heardSpeechListSchema = z.array(heardSpeechSchema).min(1).max(BROWSER_CONTROL_LIMITS.heardSpeech)
+    .refine(rows => new Set(rows.map(row => row.id)).size === rows.length);
 const observationSchema = z.strictObject({
     png: z.string().min(1).max(4 * Math.ceil(BROWSER_CONTROL_LIMITS.pngBytes / 3)).regex(/^[A-Za-z0-9+/]*={0,2}$/).refine(value => value.length % 4 === 0),
-    stateSignature: text, url: text.optional(), title: text.optional(), text: text.optional(), scrollY: coordinate.optional()
+    stateSignature: text, url: text.optional(), title: text.optional(), text: text.optional(), scrollY: coordinate.optional(),
+    heardSpeech: heardSpeechListSchema.optional()
 });
 const common = {
     version: z.literal(BROWSER_CONTROL_VERSION), identity: identitySchema,
@@ -83,6 +102,12 @@ export function validateBrowserControlAction(value) {
         throw new CuaExecutorError("invalid_request", "not_dispatched");
     return parsed.data;
 }
+export function validateHeardSpeech(value) {
+    const parsed = heardSpeechListSchema.safeParse(value);
+    if (!parsed.success)
+        throw new CuaExecutorError("invalid_response", "outcome_uncertain");
+    return parsed.data;
+}
 /** Check dimensions before allocating decoder output; v1 accepts browser-style 8-bit, noninterlaced PNG. */
 export function validateBrowserControlPng(bytes) {
     const fail = () => { throw new CuaExecutorError("invalid_response", "outcome_uncertain"); };
@@ -129,7 +154,8 @@ export function encodeBrowserControlObservation(observation) {
     validateBrowserControlPng(observation.screenshot);
     const parsed = observationSchema.safeParse({ png: observation.screenshot.toString("base64"), stateSignature: observation.stateSignature,
         ...(observation.url !== undefined ? { url: observation.url } : {}), ...(observation.title !== undefined ? { title: observation.title } : {}),
-        ...(observation.text !== undefined ? { text: observation.text } : {}), ...(observation.scrollY !== undefined ? { scrollY: observation.scrollY } : {}) });
+        ...(observation.text !== undefined ? { text: observation.text } : {}), ...(observation.scrollY !== undefined ? { scrollY: observation.scrollY } : {}),
+        ...(observation.heardSpeech !== undefined ? { heardSpeech: observation.heardSpeech } : {}) });
     if (!parsed.success)
         throw new CuaExecutorError("invalid_response", "outcome_uncertain");
     return parsed.data;
@@ -145,7 +171,8 @@ export function decodeBrowserControlObservation(value) {
     validateBrowserControlPng(screenshot);
     return { screenshot, stateSignature: state.stateSignature,
         ...(state.url !== undefined ? { url: state.url } : {}), ...(state.title !== undefined ? { title: state.title } : {}),
-        ...(state.text !== undefined ? { text: state.text } : {}), ...(state.scrollY !== undefined ? { scrollY: state.scrollY } : {}) };
+        ...(state.text !== undefined ? { text: state.text } : {}), ...(state.scrollY !== undefined ? { scrollY: state.scrollY } : {}),
+        ...(state.heardSpeech !== undefined ? { heardSpeech: state.heardSpeech } : {}) };
 }
 export function safeBrowserControlFailure(error, dispatched) {
     if (isCuaExecutorError(error))

package/dist/browser-control-protocol.js.map

@@ -1 +1 @@
-{"version":3,"file":"browser-control-protocol.js","sourceRoot":"","sources":["../src/browser-control-protocol.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,OAAO,CAAC;AAC5B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,gBAAgB,EAAE,kBAAkB,EAA6B,MAAM,yBAAyB,CAAC;AAE1G,MAAM,CAAC,MAAM,uBAAuB,GAAG,CAAC,CAAC;AACzC,MAAM,CAAC,MAAM,sBAAsB,GAAG,MAAM,CAAC,MAAM,CAAC;IAClD,UAAU,EAAE,EAAE,GAAG,IAAI,GAAG,IAAI,EAAE,QAAQ,EAAE,CAAC,GAAG,IAAI,GAAG,IAAI;IACvD,SAAS,EAAE,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,EAAE,GAAG,IAAI;IACzD,SAAS,EAAE,EAAE,EAAE,aAAa,EAAE,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM;IAClE,gBAAgB,EAAE,MAAM,EAAE,mBAAmB,EAAE,MAAM;CACtD,CAAC,CAAC;AAEH,MAAM,KAAK,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC;AACpE,MAAM,cAAc,GAAG,CAAC,CAAC,YAAY,CAAC,EAAE,UAAU,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,eAAe,EAAE,KAAK,EAAE,CAAC,CAAC;AACvG,MAAM,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,sBAAsB,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,sBAAsB,CAAC,SAAS,CAAC,CAAC;AAC5I,MAAM,UAAU,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;AACtE,MAAM,KAAK,GAAG,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC;AAC/C,MAAM,CAAC,MAAM,0BAA0B,GAAG,CAAC,CAAC,kBAAkB,CAAC,MAAM,EAAE;IACrE,CAAC,CAAC,YAAY,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,GAAG,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC;IAC9G,CAAC,CAAC,YAAY,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,cAAc,CAAC,EAAE,GAAG,KAAK,EAAE,CAAC;IAC7D,CAAC,CAAC,YAAY,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,GAAG,KAAK,EAAE,CAAC;IACrD,CAAC,CAAC,YAAY,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,GAAG,KAAK,EAAE,EAAE,EAAE,UAAU,EAAE,EAAE,EAAE,UAAU,EAAE,CAAC;IACvF,CAAC,CAAC,YAAY,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC;IACjD,CAAC,CAAC,YAAY,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,sBAAsB,CAAC,aAAa,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,sBAAsB,CAAC,SAAS,CAAC,EAAE,CAAC;IACxK,CAAC,CAAC,YAAY,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,sBAAsB,CAAC,UAAU,CAAC,EAAE,CAAC;IAC/H,CAAC,CAAC,YAAY,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,sBAAsB,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC;IACzH,CAAC,CAAC,YAAY,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,EAAE,CAAC;CAClD,CAAC,CAAC,SAAS,CAAC,CAAC,MAAM,EAAa,EAAE;IACjC,IAAI,MAAM,CAAC,IAAI,KAAK,OAAO;QAAE,OAAO,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;IACvJ,IAAI,MAAM,CAAC,IAAI,KAAK,MAAM;QAAE,OAAO,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,GAAG,CAAC,MAAM,CAAC,EAAE,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;IAChH,OAAO,MAAM,CAAC;AAChB,CAAC,CAAC,CAAC;AACH,MAAM,iBAAiB,GAAG,CAAC,CAAC,YAAY,CAAC;IACvC,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,sBAAsB,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,wBAAwB,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,KAAK,CAAC,CAAC;IACtJ,cAAc,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,OAAO,EAAE,UAAU,CAAC,QAAQ,EAAE;CAC1H,CAAC,CAAC;AAEH,MAAM,MAAM,GAAG;IACb,OAAO,EAAE,CAAC,CAAC,OAAO,CAAC,uBAAuB,CAAC,EAAE,QAAQ,EAAE,cAAc;IACrE,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,gBAAgB,CAAC,EAAE,SAAS,EAAE,KAAK;CAC5E,CAAC;AACF,MAAM,aAAa,GAAG,CAAC,CAAC,kBAAkB,CAAC,WAAW,EAAE;IACtD,CAAC,CAAC,YAAY,CAAC,EAAE,GAAG,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;IACxF,CAAC,CAAC,YAAY,CAAC,EAAE,GAAG,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC;IAC1F,CAAC,CAAC,YAAY,CAAC,EAAE,GAAG,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,0BAA0B,EAAE,CAAC;CAChJ,CAAC,CAAC;AACH,MAAM,SAAS,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,iBAAiB,EAAE,oBAAoB,EAAE,eAAe,EAAE,WAAW,EAAE,iBAAiB,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,iBAAiB,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,iBAAiB,EAAE,kBAAkB,CAAC,CAAC,CAAC;AACjQ,MAAM,SAAS,GAAG,EAAE,GAAG,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,EAAE,CAAC;AAC1I,MAAM,WAAW,GAAG,CAAC,CAAC,kBAAkB,CAAC,IAAI,EAAE;IAC7C,CAAC,CAAC,YAAY,CAAC,EAAE,GAAG,SAAS,EAAE,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,WAAW,EAAE,iBAAiB,CAAC,QAAQ,EAAE,EAAE,CAAC;IAChG,CAAC,CAAC,YAAY,CAAC,EAAE,GAAG,SAAS,EAAE,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,YAAY,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,gBAAgB,EAAE,mBAAmB,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;CACjK,CAAC,CAAC;AAGH,MAAM,UAAU,8BAA8B,CAAC,KAAc;IAC3D,MAAM,MAAM,GAAG,cAAc,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;IAC/C,IAAI,CAAC,MAAM,CAAC,OAAO;QAAE,MAAM,IAAI,gBAAgB,CAAC,iBAAiB,EAAE,gBAAgB,CAAC,CAAC;IACrF,OAAO,MAAM,CAAC,IAAI,CAAC;AACrB,CAAC;AACD,MAAM,UAAU,0BAA0B,CAAC,CAAyB,EAAE,CAAyB;IAC7F,OAAO,CAAC,CAAC,UAAU,KAAK,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC,SAAS,KAAK,CAAC,CAAC,SAAS,IAAI,CAAC,CAAC,eAAe,KAAK,CAAC,CAAC,eAAe,CAAC;AACjH,CAAC;AACD,MAAM,UAAU,0BAA0B,CAAC,KAAc;IACvD,MAAM,MAAM,GAAG,aAAa,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;IAC9C,IAAI,CAAC,MAAM,CAAC,OAAO;QAAE,MAAM,IAAI,gBAAgB,CAAC,iBAAiB,EAAE,gBAAgB,CAAC,CAAC;IACrF,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC;IAC5B,IAAI,OAAO,CAAC,SAAS,KAAK,WAAW,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC,OAAO,CAAC,SAAS,KAAK,SAAS,IAAI,OAAO,CAAC,QAAQ,KAAK,UAAU,OAAO,CAAC,GAAG,EAAE,CAAC;QAAE,MAAM,IAAI,gBAAgB,CAAC,iBAAiB,EAAE,gBAAgB,CAAC,CAAC;IACzM,OAAO,OAAO,CAAC;AACjB,CAAC;AACD,MAAM,UAAU,wBAAwB,CAAC,KAAc;IACrD,MAAM,MAAM,GAAG,WAAW,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;IAC5C,IAAI,CAAC,MAAM,CAAC,OAAO;QAAE,MAAM,IAAI,gBAAgB,CAAC,kBAAkB,EAAE,mBAAmB,CAAC,CAAC;IACzF,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC;IAC1B,IAAI,CAAC,KAAK,CAAC,SAAS,KAAK,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,KAAK,SAAS,CAAC;WACjE,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS,KAAK,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC;QAAE,MAAM,IAAI,gBAAgB,CAAC,kBAAkB,EAAE,mBAAmB,CAAC,CAAC;IAC9J,OAAO,KAAK,CAAC;AACf,CAAC;AACD,MAAM,UAAU,4BAA4B,CAAC,KAAc;IACzD,MAAM,MAAM,GAAG,0BAA0B,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;IAC3D,IAAI,CAAC,MAAM,CAAC,OAAO;QAAE,MAAM,IAAI,gBAAgB,CAAC,iBAAiB,EAAE,gBAAgB,CAAC,CAAC;IACrF,OAAO,MAAM,CAAC,IAAI,CAAC;AACrB,CAAC;AAED,4GAA4G;AAC5G,MAAM,UAAU,yBAAyB,CAAC,KAAa;IACrD,MAAM,IAAI,GAAG,GAAU,EAAE,GAAG,MAAM,IAAI,gBAAgB,CAAC,kBAAkB,EAAE,mBAAmB,CAAC,CAAC,CAAC,CAAC,CAAC;IACnG,IAAI,KAAK,CAAC,MAAM,GAAG,EAAE,IAAI,KAAK,CAAC,MAAM,GAAG,sBAAsB,CAAC,QAAQ;WAClE,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC;WAC5E,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC,KAAK,EAAE,IAAI,KAAK,CAAC,QAAQ,CAAC,OAAO,EAAE,EAAE,EAAE,EAAE,CAAC,KAAK,MAAM;QAAE,IAAI,EAAE,CAAC;IACxF,MAAM,KAAK,GAAG,KAAK,CAAC,YAAY,CAAC,EAAE,CAAC,EAAE,MAAM,GAAG,KAAK,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC;IACtE,IAAI,KAAK,KAAK,CAAC,IAAI,MAAM,KAAK,CAAC,IAAI,KAAK,GAAG,sBAAsB,CAAC,SAAS,IAAI,MAAM,GAAG,sBAAsB,CAAC,SAAS;WACnH,KAAK,GAAG,MAAM,GAAG,sBAAsB,CAAC,MAAM,IAAI,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAE,CAAC;WAC1G,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC;QAAE,IAAI,EAAE,CAAC;IACnE,wFAAwF;IACxF,IAAI,MAAM,GAAG,CAAC,EAAE,OAAO,GAAG,KAAK,EAAE,MAAM,GAAG,KAAK,CAAC;IAChD,OAAO,MAAM,GAAG,EAAE,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;QACnC,MAAM,MAAM,GAAG,KAAK,CAAC,YAAY,CAAC,MAAM,CAAC,EAAE,IAAI,GAAG,KAAK,CAAC,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,CAAC,EAAE,MAAM,GAAG,CAAC,CAAC,CAAC;QAClG,IAAI,MAAM,GAAG,KAAK,CAAC,MAAM,GAAG,MAAM,GAAG,EAAE,IAAI,CAAC,IAAI,KAAK,MAAM,IAAI,MAAM,KAAK,CAAC,CAAC;YAAE,IAAI,EAAE,CAAC;QACrF,IAAI,IAAI,KAAK,MAAM;YAAE,OAAO,GAAG,IAAI,CAAC;QACpC,MAAM,IAAI,EAAE,GAAG,MAAM,CAAC;QACtB,IAAI,IAAI,KAAK,MAAM,EAAE,CAAC;YAAC,IAAI,MAAM,KAAK,CAAC,IAAI,MAAM,KAAK,KAAK,CAAC,MAAM;gBAAE,IAAI,EAAE,CAAC;YAAC,MAAM,GAAG,IAAI,CAAC;YAAC,MAAM;QAAC,CAAC;IACrG,CAAC;IACD,IAAI,CAAC,OAAO,IAAI,CAAC,MAAM;QAAE,IAAI,EAAE,CAAC;IAChC,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;QACzD,IAAI,OAAO,CAAC,KAAK,KAAK,KAAK,IAAI,OAAO,CAAC,MAAM,KAAK,MAAM;YAAE,IAAI,EAAE,CAAC;IACnE,CAAC;IAAC,MAAM,CAAC;QAAC,IAAI,EAAE,CAAC;IAAC,CAAC;AACrB,CAAC;AACD,MAAM,UAAU,+BAA+B,CAAC,WAA2B;IACzE,oFAAoF;IACpF,IAAI,CAAC,WAAW,IAAI,WAAW,CAAC,QAAQ,KAAK,SAAS,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,UAAU,CAAC;QAAE,MAAM,IAAI,gBAAgB,CAAC,kBAAkB,EAAE,mBAAmB,CAAC,CAAC;IACxK,yBAAyB,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;IAClD,MAAM,MAAM,GAAG,iBAAiB,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,WAAW,CAAC,UAAU,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,cAAc,EAAE,WAAW,CAAC,cAAc;QACrI,GAAG,CAAC,WAAW,CAAC,GAAG,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,WAAW,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,GAAG,CAAC,WAAW,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,WAAW,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC5I,GAAG,CAAC,WAAW,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,GAAG,CAAC,WAAW,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,WAAW,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;IAC3J,IAAI,CAAC,MAAM,CAAC,OAAO;QAAE,MAAM,IAAI,gBAAgB,CAAC,kBAAkB,EAAE,mBAAmB,CAAC,CAAC;IACzF,OAAO,MAAM,CAAC,IAAI,CAAC;AACrB,CAAC;AACD,MAAM,UAAU,+BAA+B,CAAC,KAAc;IAC5D,MAAM,MAAM,GAAG,iBAAiB,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;IAClD,IAAI,CAAC,MAAM,CAAC,OAAO;QAAE,MAAM,IAAI,gBAAgB,CAAC,kBAAkB,EAAE,mBAAmB,CAAC,CAAC;IACzF,MAAM,EAAE,GAAG,EAAE,GAAG,KAAK,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC;IACtC,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IAC9C,IAAI,UAAU,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,GAAG;QAAE,MAAM,IAAI,gBAAgB,CAAC,kBAAkB,EAAE,mBAAmB,CAAC,CAAC;IAC/G,yBAAyB,CAAC,UAAU,CAAC,CAAC;IACtC,OAAO,EAAE,UAAU,EAAE,cAAc,EAAE,KAAK,CAAC,cAAc;QACvD,GAAG,CAAC,KAAK,CAAC,GAAG,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACpH,GAAG,CAAC,KAAK,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;AACpI,CAAC;AACD,MAAM,UAAU,yBAAyB,CAAC,KAAc,EAAE,UAAmB;IAC3E,IAAI,kBAAkB,CAAC,KAAK,CAAC;QAAE,OAAO,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,WAAW,EAAE,KAAK,CAAC,WAAW,EAAE,CAAC;IAC3F,OAAO,EAAE,IAAI,EAAE,iBAAiB,EAAE,WAAW,EAAE,UAAU,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAAC,CAAC,gBAAgB,EAAE,CAAC;AACvG,CAAC"}
+{"version":3,"file":"browser-control-protocol.js","sourceRoot":"","sources":["../src/browser-control-protocol.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,OAAO,CAAC;AAC5B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,iBAAiB,EAAoB,MAAM,iBAAiB,CAAC;AACtE,OAAO,EAAE,gBAAgB,EAAE,kBAAkB,EAA6B,MAAM,yBAAyB,CAAC;AAE1G,MAAM,CAAC,MAAM,uBAAuB,GAAG,CAAC,CAAC;AACzC,MAAM,CAAC,MAAM,sBAAsB,GAAG,MAAM,CAAC,MAAM,CAAC;IAClD,UAAU,EAAE,EAAE,GAAG,IAAI,GAAG,IAAI,EAAE,QAAQ,EAAE,CAAC,GAAG,IAAI,GAAG,IAAI;IACvD,SAAS,EAAE,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,EAAE,GAAG,IAAI;IACzD,gBAAgB,EAAE,iBAAiB,CAAC,UAAU,EAAE,WAAW,EAAE,iBAAiB,CAAC,KAAK;IACpF,WAAW,EAAE,iBAAiB,CAAC,UAAU,EAAE,gBAAgB,EAAE,iBAAiB,CAAC,UAAU;IACzF,SAAS,EAAE,EAAE,EAAE,aAAa,EAAE,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM;IAClE,gBAAgB,EAAE,MAAM,EAAE,mBAAmB,EAAE,MAAM;CACtD,CAAC,CAAC;AAEH,MAAM,KAAK,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC;AACpE,MAAM,cAAc,GAAG,CAAC,CAAC,YAAY,CAAC,EAAE,UAAU,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,eAAe,EAAE,KAAK,EAAE,CAAC,CAAC;AACvG,MAAM,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,sBAAsB,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,sBAAsB,CAAC,SAAS,CAAC,CAAC;AAC5I,MAAM,UAAU,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,sBAAsB,CAAC,gBAAgB,CAAC;KAC9E,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,IAAI,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,sBAAsB,CAAC,WAAW,CAAC,CAAC;AAC9G,MAAM,UAAU,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;AACtE,MAAM,KAAK,GAAG,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC;AAC/C,MAAM,oBAAoB,GAAG;IAC3B,CAAC,CAAC,YAAY,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,GAAG,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC;IAC9G,CAAC,CAAC,YAAY,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,cAAc,CAAC,EAAE,GAAG,KAAK,EAAE,CAAC;IAC7D,CAAC,CAAC,YAAY,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,GAAG,KAAK,EAAE,CAAC;IACrD,CAAC,CAAC,YAAY,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,GAAG,KAAK,EAAE,EAAE,EAAE,UAAU,EAAE,EAAE,EAAE,UAAU,EAAE,CAAC;IACvF,CAAC,CAAC,YAAY,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC;IACjD,CAAC,CAAC,YAAY,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,sBAAsB,CAAC,aAAa,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,sBAAsB,CAAC,SAAS,CAAC,EAAE,CAAC;IACxK,CAAC,CAAC,YAAY,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,sBAAsB,CAAC,UAAU,CAAC,EAAE,CAAC;IAC/H,CAAC,CAAC,YAAY,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,sBAAsB,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC;IACzH,CAAC,CAAC,YAAY,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,EAAE,CAAC;CACzC,CAAC;AACX,MAAM,CAAC,MAAM,8BAA8B,GAAG,CAAC,CAAC,kBAAkB,CAAC,MAAM,EAAE,oBAAoB,CAAC,CAAC;AACjG,MAAM,CAAC,MAAM,0BAA0B,GAAG,CAAC,CAAC,kBAAkB,CAAC,MAAM,EAAE;IACrE,GAAG,oBAAoB;IACvB,CAAC,CAAC,YAAY,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC;CAC/D,CAAC,CAAC,SAAS,CAAC,CAAC,MAAM,EAAa,EAAE;IACjC,IAAI,MAAM,CAAC,IAAI,KAAK,OAAO;QAAE,OAAO,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;IACvJ,IAAI,MAAM,CAAC,IAAI,KAAK,MAAM;QAAE,OAAO,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,GAAG,CAAC,MAAM,CAAC,EAAE,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;IAChH,OAAO,MAAM,CAAC;AAChB,CAAC,CAAC,CAAC;AACH,MAAM,iBAAiB,GAAG,CAAC,CAAC,YAAY,CAAC;IACvC,EAAE,EAAE,KAAK;IACT,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,eAAe,CAAC;IAClC,IAAI,EAAE,UAAU;IAChB,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,sBAAsB,CAAC,gBAAgB,CAAC;CACjF,CAAC,CAAC;AACH,MAAM,qBAAqB,GAAG,CAAC,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,sBAAsB,CAAC,WAAW,CAAC;KACpG,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,MAAM,CAAC,CAAC;AACzE,MAAM,iBAAiB,GAAG,CAAC,CAAC,YAAY,CAAC;IACvC,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,sBAAsB,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,wBAAwB,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,KAAK,CAAC,CAAC;IACtJ,cAAc,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,OAAO,EAAE,UAAU,CAAC,QAAQ,EAAE;IACzH,WAAW,EAAE,qBAAqB,CAAC,QAAQ,EAAE;CAC9C,CAAC,CAAC;AAEH,MAAM,MAAM,GAAG;IACb,OAAO,EAAE,CAAC,CAAC,OAAO,CAAC,uBAAuB,CAAC,EAAE,QAAQ,EAAE,cAAc;IACrE,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,gBAAgB,CAAC,EAAE,SAAS,EAAE,KAAK;CAC5E,CAAC;AACF,MAAM,aAAa,GAAG,CAAC,CAAC,kBAAkB,CAAC,WAAW,EAAE;IACtD,CAAC,CAAC,YAAY,CAAC,EAAE,GAAG,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;IACxF,CAAC,CAAC,YAAY,CAAC,EAAE,GAAG,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC;IAC1F,CAAC,CAAC,YAAY,CAAC,EAAE,GAAG,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,0BAA0B,EAAE,CAAC;CAChJ,CAAC,CAAC;AACH,MAAM,SAAS,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,iBAAiB,EAAE,oBAAoB,EAAE,eAAe,EAAE,WAAW,EAAE,iBAAiB,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,iBAAiB,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,iBAAiB,EAAE,kBAAkB,CAAC,CAAC,CAAC;AACjQ,MAAM,SAAS,GAAG,EAAE,GAAG,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,EAAE,CAAC;AAC1I,MAAM,WAAW,GAAG,CAAC,CAAC,kBAAkB,CAAC,IAAI,EAAE;IAC7C,CAAC,CAAC,YAAY,CAAC,EAAE,GAAG,SAAS,EAAE,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,WAAW,EAAE,iBAAiB,CAAC,QAAQ,EAAE,EAAE,CAAC;IAChG,CAAC,CAAC,YAAY,CAAC,EAAE,GAAG,SAAS,EAAE,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,YAAY,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,gBAAgB,EAAE,mBAAmB,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;CACjK,CAAC,CAAC;AAGH,MAAM,UAAU,8BAA8B,CAAC,KAAc;IAC3D,MAAM,MAAM,GAAG,cAAc,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;IAC/C,IAAI,CAAC,MAAM,CAAC,OAAO;QAAE,MAAM,IAAI,gBAAgB,CAAC,iBAAiB,EAAE,gBAAgB,CAAC,CAAC;IACrF,OAAO,MAAM,CAAC,IAAI,CAAC;AACrB,CAAC;AACD,MAAM,UAAU,0BAA0B,CAAC,CAAyB,EAAE,CAAyB;IAC7F,OAAO,CAAC,CAAC,UAAU,KAAK,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC,SAAS,KAAK,CAAC,CAAC,SAAS,IAAI,CAAC,CAAC,eAAe,KAAK,CAAC,CAAC,eAAe,CAAC;AACjH,CAAC;AACD,MAAM,UAAU,0BAA0B,CAAC,KAAc;IACvD,MAAM,MAAM,GAAG,aAAa,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;IAC9C,IAAI,CAAC,MAAM,CAAC,OAAO;QAAE,MAAM,IAAI,gBAAgB,CAAC,iBAAiB,EAAE,gBAAgB,CAAC,CAAC;IACrF,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC;IAC5B,IAAI,OAAO,CAAC,SAAS,KAAK,WAAW,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC,OAAO,CAAC,SAAS,KAAK,SAAS,IAAI,OAAO,CAAC,QAAQ,KAAK,UAAU,OAAO,CAAC,GAAG,EAAE,CAAC;QAAE,MAAM,IAAI,gBAAgB,CAAC,iBAAiB,EAAE,gBAAgB,CAAC,CAAC;IACzM,OAAO,OAAO,CAAC;AACjB,CAAC;AACD,MAAM,UAAU,wBAAwB,CAAC,KAAc;IACrD,MAAM,MAAM,GAAG,WAAW,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;IAC5C,IAAI,CAAC,MAAM,CAAC,OAAO;QAAE,MAAM,IAAI,gBAAgB,CAAC,kBAAkB,EAAE,mBAAmB,CAAC,CAAC;IACzF,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC;IAC1B,IAAI,CAAC,KAAK,CAAC,SAAS,KAAK,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,KAAK,SAAS,CAAC;WACjE,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS,KAAK,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC;QAAE,MAAM,IAAI,gBAAgB,CAAC,kBAAkB,EAAE,mBAAmB,CAAC,CAAC;IAC9J,OAAO,KAAK,CAAC;AACf,CAAC;AACD,MAAM,UAAU,4BAA4B,CAAC,KAAc;IACzD,MAAM,MAAM,GAAG,0BAA0B,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;IAC3D,IAAI,CAAC,MAAM,CAAC,OAAO;QAAE,MAAM,IAAI,gBAAgB,CAAC,iBAAiB,EAAE,gBAAgB,CAAC,CAAC;IACrF,OAAO,MAAM,CAAC,IAAI,CAAC;AACrB,CAAC;AACD,MAAM,UAAU,mBAAmB,CAAC,KAAc;IAChD,MAAM,MAAM,GAAG,qBAAqB,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;IACtD,IAAI,CAAC,MAAM,CAAC,OAAO;QAAE,MAAM,IAAI,gBAAgB,CAAC,kBAAkB,EAAE,mBAAmB,CAAC,CAAC;IACzF,OAAO,MAAM,CAAC,IAAI,CAAC;AACrB,CAAC;AAED,4GAA4G;AAC5G,MAAM,UAAU,yBAAyB,CAAC,KAAa;IACrD,MAAM,IAAI,GAAG,GAAU,EAAE,GAAG,MAAM,IAAI,gBAAgB,CAAC,kBAAkB,EAAE,mBAAmB,CAAC,CAAC,CAAC,CAAC,CAAC;IACnG,IAAI,KAAK,CAAC,MAAM,GAAG,EAAE,IAAI,KAAK,CAAC,MAAM,GAAG,sBAAsB,CAAC,QAAQ;WAClE,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC;WAC5E,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC,KAAK,EAAE,IAAI,KAAK,CAAC,QAAQ,CAAC,OAAO,EAAE,EAAE,EAAE,EAAE,CAAC,KAAK,MAAM;QAAE,IAAI,EAAE,CAAC;IACxF,MAAM,KAAK,GAAG,KAAK,CAAC,YAAY,CAAC,EAAE,CAAC,EAAE,MAAM,GAAG,KAAK,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC;IACtE,IAAI,KAAK,KAAK,CAAC,IAAI,MAAM,KAAK,CAAC,IAAI,KAAK,GAAG,sBAAsB,CAAC,SAAS,IAAI,MAAM,GAAG,sBAAsB,CAAC,SAAS;WACnH,KAAK,GAAG,MAAM,GAAG,sBAAsB,CAAC,MAAM,IAAI,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAE,CAAC;WAC1G,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC;QAAE,IAAI,EAAE,CAAC;IACnE,wFAAwF;IACxF,IAAI,MAAM,GAAG,CAAC,EAAE,OAAO,GAAG,KAAK,EAAE,MAAM,GAAG,KAAK,CAAC;IAChD,OAAO,MAAM,GAAG,EAAE,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;QACnC,MAAM,MAAM,GAAG,KAAK,CAAC,YAAY,CAAC,MAAM,CAAC,EAAE,IAAI,GAAG,KAAK,CAAC,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,CAAC,EAAE,MAAM,GAAG,CAAC,CAAC,CAAC;QAClG,IAAI,MAAM,GAAG,KAAK,CAAC,MAAM,GAAG,MAAM,GAAG,EAAE,IAAI,CAAC,IAAI,KAAK,MAAM,IAAI,MAAM,KAAK,CAAC,CAAC;YAAE,IAAI,EAAE,CAAC;QACrF,IAAI,IAAI,KAAK,MAAM;YAAE,OAAO,GAAG,IAAI,CAAC;QACpC,MAAM,IAAI,EAAE,GAAG,MAAM,CAAC;QACtB,IAAI,IAAI,KAAK,MAAM,EAAE,CAAC;YAAC,IAAI,MAAM,KAAK,CAAC,IAAI,MAAM,KAAK,KAAK,CAAC,MAAM;gBAAE,IAAI,EAAE,CAAC;YAAC,MAAM,GAAG,IAAI,CAAC;YAAC,MAAM;QAAC,CAAC;IACrG,CAAC;IACD,IAAI,CAAC,OAAO,IAAI,CAAC,MAAM;QAAE,IAAI,EAAE,CAAC;IAChC,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;QACzD,IAAI,OAAO,CAAC,KAAK,KAAK,KAAK,IAAI,OAAO,CAAC,MAAM,KAAK,MAAM;YAAE,IAAI,EAAE,CAAC;IACnE,CAAC;IAAC,MAAM,CAAC;QAAC,IAAI,EAAE,CAAC;IAAC,CAAC;AACrB,CAAC;AACD,MAAM,UAAU,+BAA+B,CAAC,WAA2B;IACzE,oFAAoF;IACpF,IAAI,CAAC,WAAW,IAAI,WAAW,CAAC,QAAQ,KAAK,SAAS,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,UAAU,CAAC;QAAE,MAAM,IAAI,gBAAgB,CAAC,kBAAkB,EAAE,mBAAmB,CAAC,CAAC;IACxK,yBAAyB,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;IAClD,MAAM,MAAM,GAAG,iBAAiB,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,WAAW,CAAC,UAAU,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,cAAc,EAAE,WAAW,CAAC,cAAc;QACrI,GAAG,CAAC,WAAW,CAAC,GAAG,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,WAAW,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,GAAG,CAAC,WAAW,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,WAAW,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC5I,GAAG,CAAC,WAAW,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,GAAG,CAAC,WAAW,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,WAAW,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACrJ,GAAG,CAAC,WAAW,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,WAAW,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;IAChG,IAAI,CAAC,MAAM,CAAC,OAAO;QAAE,MAAM,IAAI,gBAAgB,CAAC,kBAAkB,EAAE,mBAAmB,CAAC,CAAC;IACzF,OAAO,MAAM,CAAC,IAAI,CAAC;AACrB,CAAC;AACD,MAAM,UAAU,+BAA+B,CAAC,KAAc;IAC5D,MAAM,MAAM,GAAG,iBAAiB,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;IAClD,IAAI,CAAC,MAAM,CAAC,OAAO;QAAE,MAAM,IAAI,gBAAgB,CAAC,kBAAkB,EAAE,mBAAmB,CAAC,CAAC;IACzF,MAAM,EAAE,GAAG,EAAE,GAAG,KAAK,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC;IACtC,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IAC9C,IAAI,UAAU,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,GAAG;QAAE,MAAM,IAAI,gBAAgB,CAAC,kBAAkB,EAAE,mBAAmB,CAAC,CAAC;IAC/G,yBAAyB,CAAC,UAAU,CAAC,CAAC;IACtC,OAAO,EAAE,UAAU,EAAE,cAAc,EAAE,KAAK,CAAC,cAAc;QACvD,GAAG,CAAC,KAAK,CAAC,GAAG,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACpH,GAAG,CAAC,KAAK,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7H,GAAG,CAAC,KAAK,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;AACrF,CAAC;AACD,MAAM,UAAU,yBAAyB,CAAC,KAAc,EAAE,UAAmB;IAC3E,IAAI,kBAAkB,CAAC,KAAK,CAAC;QAAE,OAAO,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,WAAW,EAAE,KAAK,CAAC,WAAW,EAAE,CAAC;IAC3F,OAAO,EAAE,IAAI,EAAE,iBAAiB,EAAE,WAAW,EAAE,UAAU,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAAC,CAAC,gBAAgB,EAAE,CAAC;AACvG,CAAC"}

package/dist/computer-use.d.ts

@@ -1,3 +1,5 @@
+import { type HeardSpeech } from "./cua-speech.js";
+export { CUA_SPEECH_LIMITS, type HeardSpeech } from "./cua-speech.js";
 import { type ActorCapabilities, type ActorCompletionReason, type ActorPersonaRef, type ActorStatus, type ActorTokenUsage, type ActorTrace, type ActorTraceItem, type ParticipantDeclaredOutcome, type ParticipantClosingReport } from "./actor-contract.js";
 import type { ActorExecutionProfile, ProviderRequestReceipt } from "./actor-contract.js";
 import type { RedactionHooks } from "./redaction.js";
@@ -38,6 +40,9 @@ export type CuaAction = {
 } | {
     kind: "wait";
     ms?: number;
+} | {
+    kind: "speak";
+    text: string;
 } | {
     kind: "screenshot";
 };
@@ -93,6 +98,8 @@ export interface CuaObservation {
     scrollY?: number;
     title?: string;
     text?: string;
+    /** Finalized speech captured from the participant's speaker sink. */
+    heardSpeech?: HeardSpeech[];
 }
 /**
  * A safety check the model raised. The triple is preserved verbatim from the
@@ -212,6 +219,8 @@ export interface CuaExecutor {
      * legacy observation/idle stall recovery. Wrappers must preserve this value.
      */
     readonly stallRecovery?: "fail_closed";
+    /** True only when this executor can play a `speak` action into the participant microphone. */
+    readonly speechEnabled?: boolean;
     /** Capture the current desktop frame and its state signature. */
     observe(): Promise<CuaObservation>;
     /**

package/dist/computer-use.js

@@ -1,3 +1,5 @@
+import { CUA_SPEECH_LIMITS } from "./cua-speech.js";
+export { CUA_SPEECH_LIMITS } from "./cua-speech.js";
 import { ACTOR_TRACE_SCHEMA } from "./actor-contract.js";
 import { CuaProviderError, isCuaProviderError } from "./cua-provider-error.js";
 import { classifyCuaAction, summarizeAffordanceUse } from "./affordance.js";
@@ -80,6 +82,8 @@ export function actionFingerprint(actions) {
                 return `scroll@${bucket(action.x)},${bucket(action.y)}:${Math.sign(action.dx)},${Math.sign(action.dy)}`;
             case "type":
                 return `type:${action.text.length}`;
+            case "speak":
+                return `speak:${action.text.length}`;
             case "keypress":
                 return `keypress:${action.keys.join("+")}`;
             case "drag":
@@ -213,6 +217,8 @@ export function describeCuaAction(action) {
             return `scroll (${action.dx}, ${action.dy}) at (${action.x}, ${action.y})`;
         case "type":
             return `type [${action.text.length} chars]`;
+        case "speak":
+            return `speak [${action.text.length} chars]`;
         case "keypress":
             return `keypress ${action.keys.join("+")}`;
         case "drag":
@@ -742,6 +748,24 @@ export async function runComputerUseLoop(options) {
                 return await raceBounded(`observe (${label}, retry)`, executor.observe(), remaining(), observationTimeoutMs, signal);
             }
         };
+        const seenSpeechIds = new Set();
+        let pendingHeardSpeech = [];
+        let heardSpeechChanged = false;
+        const collectHeardSpeech = (value) => {
+            for (const utterance of value.heardSpeech ?? []) {
+                if (seenSpeechIds.has(utterance.id))
+                    continue;
+                seenSpeechIds.add(utterance.id);
+                pendingHeardSpeech.push(utterance);
+                heardSpeechChanged = true;
+                record({ id: nextId("notice"), kind: "notice", lifecycle: "completed", status: "ok",
+                    title: "speech heard", text: redactNarration(utterance.text) });
+            }
+            if (pendingHeardSpeech.length > CUA_SPEECH_LIMITS.utterances) {
+                throw new CuaExecutorError("invalid_response", "outcome_uncertain");
+            }
+            return pendingHeardSpeech.length === 0 ? value : { ...value, heardSpeech: pendingHeardSpeech.slice() };
+        };
         // The declared observation window (#510). The harness holds, looks, and takes nothing back to
         // the model: no action, no turn, no tokens. It runs once, cut to whatever session budget is
         // left, and says in the trace that the time was deliberate.
@@ -780,7 +804,7 @@ export async function runComputerUseLoop(options) {
                     throw new CuaAbortError();
                 await sleep(Math.min(dwell.everyMs, budget - (now() - dwellStartedAtMs)));
                 currentPhase = `dwell frame ${frames + 1}`;
-                const frameObservation = await observeBounded(`dwell frame ${frames + 1}`);
+                const frameObservation = collectHeardSpeech(await observeBounded(`dwell frame ${frames + 1}`));
                 frames += 1;
                 if (frameObservation.screenshot !== undefined)
                     onScreenshot?.(frameObservation.screenshot);
@@ -804,7 +828,7 @@ export async function runComputerUseLoop(options) {
             contextHint = `The study held this page under observation for ${Math.round(dwellHeldMs / 1000)} seconds (a declared dwell window; you took no actions in that time). Continue the mission from the current state of the page.`;
             return "continue";
         };
-        let observation = await observeBounded("initial");
+        let observation = collectHeardSpeech(await observeBounded("initial"));
         // Runtime-only: hand the seat's live location.href back to the orchestrator (never persisted).
         onObservedUrl?.(observation.url);
         if (observation.screenshot !== undefined)
@@ -819,7 +843,7 @@ export async function runComputerUseLoop(options) {
         observeTasks(observation, 0);
         const initialDwell = await dwellIfDue(observation, 0);
         if (initialDwell !== undefined) {
-            observation = await observeBounded("after dwell");
+            observation = collectHeardSpeech(await observeBounded("after dwell"));
             onObservedUrl?.(observation.url);
             if (observation.screenshot !== undefined)
                 onScreenshot?.(observation.screenshot);
@@ -952,6 +976,8 @@ export async function runComputerUseLoop(options) {
                 }
             }
             bump("turns");
+            pendingHeardSpeech = [];
+            heardSpeechChanged = false;
             previousResponseId = turn.responseId ?? previousResponseId;
             lastResponseId = turn.responseId ?? lastResponseId;
             recordUsage(turn);
@@ -1112,7 +1138,7 @@ export async function runComputerUseLoop(options) {
                 if (taskTracker !== undefined) {
                     try {
                         currentPhase = "observing closing task state";
-                        const closing = await observeBounded("closing");
+                        const closing = collectHeardSpeech(await observeBounded("closing"));
                         observeTasks(closing, turnNumber);
                     }
                     catch (error) {
@@ -1286,6 +1312,7 @@ export async function runComputerUseLoop(options) {
                     kind: "ui_action",
                     lifecycle: "completed",
                     title: actionTitle,
+                    ...(action.kind === "speak" ? { text: redactNarration(action.text) } : {}),
                     // Structured pin coordinates (#441), exactly the click classes the Observer
                     // pins render — recorded fact instead of a title re-parse downstream.
                     ...(action.kind === "click" || action.kind === "double_click"
@@ -1296,7 +1323,7 @@ export async function runComputerUseLoop(options) {
             if (signal?.aborted)
                 throw new CuaAbortError();
             currentPhase = `observing UI state after turn ${turnNumber}`;
-            observation = await observeBounded(`after turn ${turnNumber}`);
+            observation = collectHeardSpeech(await observeBounded(`after turn ${turnNumber}`));
             // Runtime-only: hand the seat's live location.href back to the orchestrator (never persisted).
             onObservedUrl?.(observation.url);
             if (observation.screenshot !== undefined)
@@ -1311,7 +1338,7 @@ export async function runComputerUseLoop(options) {
             observeTasks(observation, turnNumber);
             const dwellOutcome = await dwellIfDue(observation, turnNumber);
             if (dwellOutcome !== undefined) {
-                observation = await observeBounded("after dwell");
+                observation = collectHeardSpeech(await observeBounded("after dwell"));
                 onObservedUrl?.(observation.url);
                 if (observation.screenshot !== undefined)
                     onScreenshot?.(observation.screenshot);
@@ -1359,13 +1386,14 @@ export async function runComputerUseLoop(options) {
             // behavioral signal already (the agent took nothing but screenshots and waits), so it keeps
             // reading the frame on its own — a repeated screenshot is exactly what an idle streak IS, and
             // feeding repetition into it would grant an extra forgiveness step for being idle.
-            const progressed = frameChanged || !repeatingRecentAction;
+            const observedProgress = frameChanged || heardSpeechChanged;
+            const progressed = observedProgress || !repeatingRecentAction;
             // A screenshot/wait turn while the UI visibly changes may be patience through loading or a
             // transition, so grant a bounded recovery window. Do not grant infinite immunity: animated
             // pixels or state-executor turn counters can otherwise keep a screenshot/wait loop alive
             // until the wall-clock timeout.
             if (idleThisTurn) {
-                if (frameChanged && idleProgressForgivenessUsed < IDLE_PROGRESS_FORGIVENESS_STEPS) {
+                if (observedProgress && idleProgressForgivenessUsed < IDLE_PROGRESS_FORGIVENESS_STEPS) {
                     idleProgressForgivenessUsed += 1;
                     consecutiveIdle = 0;
                 }