humanish 0.97.0 → 0.98.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +86 -79
- package/CONTRIBUTING.md +7 -2
- package/README.md +7 -1
- package/dist/actor-contract.d.ts +35 -1
- package/dist/actor-contract.js +38 -0
- package/dist/actor-contract.js.map +1 -1
- package/dist/adapter-extension.js +1 -0
- package/dist/adapter-extension.js.map +1 -1
- package/dist/browser-control-client.d.ts +14 -0
- package/dist/browser-control-client.js +134 -0
- package/dist/browser-control-client.js.map +1 -0
- package/dist/browser-control-dispatcher.d.ts +14 -0
- package/dist/browser-control-dispatcher.js +109 -0
- package/dist/browser-control-dispatcher.js.map +1 -0
- package/dist/browser-control-protocol.d.ts +371 -0
- package/dist/browser-control-protocol.js +155 -0
- package/dist/browser-control-protocol.js.map +1 -0
- package/dist/browser-control-transport.d.ts +24 -0
- package/dist/browser-control-transport.js +156 -0
- package/dist/browser-control-transport.js.map +1 -0
- package/dist/comms-lease-store.d.ts +1 -0
- package/dist/comms-lease-store.js +9 -3
- package/dist/comms-lease-store.js.map +1 -1
- package/dist/computer-use-actor.d.ts +2 -2
- package/dist/computer-use-actor.js +6 -1
- package/dist/computer-use-actor.js.map +1 -1
- package/dist/computer-use.d.ts +23 -1
- package/dist/computer-use.js +253 -70
- package/dist/computer-use.js.map +1 -1
- package/dist/cua-actor-lab.d.ts +6 -1
- package/dist/cua-actor-lab.js +55 -15
- package/dist/cua-actor-lab.js.map +1 -1
- package/dist/cua-executor-error.d.ts +31 -0
- package/dist/cua-executor-error.js +48 -0
- package/dist/cua-executor-error.js.map +1 -0
- package/dist/cua-provider-error.d.ts +12 -0
- package/dist/cua-provider-error.js +28 -0
- package/dist/cua-provider-error.js.map +1 -0
- package/dist/desktop-session.js +3 -1
- package/dist/desktop-session.js.map +1 -1
- package/dist/doctor-lab.d.ts +2 -0
- package/dist/doctor-lab.js +25 -7
- package/dist/doctor-lab.js.map +1 -1
- package/dist/e2b-desktop-executor.d.ts +1 -24
- package/dist/e2b-desktop-executor.js +2 -127
- package/dist/e2b-desktop-executor.js.map +1 -1
- package/dist/e2b-terminal-lab.js +1 -0
- package/dist/e2b-terminal-lab.js.map +1 -1
- package/dist/frame-signature.d.ts +24 -0
- package/dist/frame-signature.js +128 -0
- package/dist/frame-signature.js.map +1 -0
- package/dist/guest-bootstrap.d.ts +43 -0
- package/dist/guest-bootstrap.js +240 -0
- package/dist/guest-bootstrap.js.map +1 -0
- package/dist/guest-browser-tools.d.ts +8 -0
- package/dist/guest-browser-tools.js +66 -0
- package/dist/guest-browser-tools.js.map +1 -0
- package/dist/guest-chromium-text.d.ts +27 -0
- package/dist/guest-chromium-text.js +281 -0
- package/dist/guest-chromium-text.js.map +1 -0
- package/dist/guest-desktop-executor.d.ts +22 -0
- package/dist/guest-desktop-executor.js +177 -0
- package/dist/guest-desktop-executor.js.map +1 -0
- package/dist/guest-desktop-native.d.ts +14 -0
- package/dist/guest-desktop-native.js +131 -0
- package/dist/guest-desktop-native.js.map +1 -0
- package/dist/guest-runtime-desktop.d.ts +35 -0
- package/dist/guest-runtime-desktop.js +231 -0
- package/dist/guest-runtime-desktop.js.map +1 -0
- package/dist/guest-runtime-main.d.ts +1 -0
- package/dist/guest-runtime-main.js +31 -0
- package/dist/guest-runtime-main.js.map +1 -0
- package/dist/guest-runtime-revision.d.ts +1 -0
- package/dist/guest-runtime-revision.js +3 -0
- package/dist/guest-runtime-revision.js.map +1 -0
- package/dist/guest-runtime.d.ts +25 -0
- package/dist/guest-runtime.js +96 -0
- package/dist/guest-runtime.js.map +1 -0
- package/dist/lab-config.js +10 -3
- package/dist/lab-config.js.map +1 -1
- package/dist/lab-engine.js +6 -0
- package/dist/lab-engine.js.map +1 -1
- package/dist/lab-summary.d.ts +2 -0
- package/dist/lab-summary.js +5 -0
- package/dist/lab-summary.js.map +1 -1
- package/dist/local-agent-cli.js +1 -1
- package/dist/local-agent-cli.js.map +1 -1
- package/dist/local-firecracker-desktop.d.ts +13 -0
- package/dist/local-firecracker-desktop.js +150 -0
- package/dist/local-firecracker-desktop.js.map +1 -0
- package/dist/local-firecracker-study.d.ts +9 -0
- package/dist/local-firecracker-study.js +93 -0
- package/dist/local-firecracker-study.js.map +1 -0
- package/dist/local-runtime-config.d.ts +6 -0
- package/dist/local-runtime-config.js +56 -0
- package/dist/local-runtime-config.js.map +1 -0
- package/dist/local-runtime-release.d.ts +3 -0
- package/dist/local-runtime-release.js +8 -0
- package/dist/local-runtime-release.js.map +1 -0
- package/dist/local-runtime.d.ts +25 -0
- package/dist/local-runtime.js +113 -0
- package/dist/local-runtime.js.map +1 -0
- package/dist/observer-app.html +4 -4
- package/dist/pricing.d.ts +22 -1
- package/dist/pricing.js +22 -0
- package/dist/pricing.js.map +1 -1
- package/dist/program.js +26 -0
- package/dist/program.js.map +1 -1
- package/dist/restricted-codex-participant-policy.d.ts +39 -0
- package/dist/restricted-codex-participant-policy.js +69 -0
- package/dist/restricted-codex-participant-policy.js.map +1 -0
- package/dist/restricted-codex-participant-run.d.ts +20 -0
- package/dist/restricted-codex-participant-run.js +78 -0
- package/dist/restricted-codex-participant-run.js.map +1 -0
- package/dist/restricted-codex-participant.d.ts +14 -0
- package/dist/restricted-codex-participant.js +178 -0
- package/dist/restricted-codex-participant.js.map +1 -0
- package/dist/restricted-codex-session.d.ts +2 -2
- package/dist/restricted-codex-session.js +11 -17
- package/dist/restricted-codex-session.js.map +1 -1
- package/dist/run-detail.js +4 -2
- package/dist/run-detail.js.map +1 -1
- package/dist/run.d.ts +12 -5
- package/dist/run.js +17 -1
- package/dist/run.js.map +1 -1
- package/dist/shared-world-lab.js +2 -2
- package/dist/shared-world-lab.js.map +1 -1
- package/dist/study-analysis-job.d.ts +2 -2
- package/dist/study-analysis-validation.d.ts +14 -14
- package/dist/study-costs.js +6 -0
- package/dist/study-costs.js.map +1 -1
- package/dist/tui-app.js +37 -37
- package/docs/architecture/browser-control.md +117 -0
- package/docs/architecture/guest-desktop.md +87 -0
- package/docs/architecture/local-browser-runtime.md +100 -0
- package/docs/architecture/restricted-codex-analysis.md +4 -2
- package/docs/architecture/runtime-broker-core.md +30 -0
- package/docs/contracts/schemas.md +1 -1
- package/docs/goals/current.md +24 -8
- package/docs/product/open-source-install-experience.md +7 -0
- package/docs/ramp/README.md +24 -13
- package/docs/release/0.98.0-local-browser-studies.md +31 -0
- package/package.json +4 -2
- package/skills/humanish/SKILL.md +23 -3
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"browser-control-client.js","sourceRoot":"","sources":["../src/browser-control-client.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAC3D,OAAO,EACL,sBAAsB,EAAE,uBAAuB,EAAE,+BAA+B,EAChF,wBAAwB,EAAE,0BAA0B,EAAE,4BAA4B,EAAE,8BAA8B,EAEnH,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAAE,uBAAuB,EAAE,MAAM,gCAAgC,CAAC;AAKzE,MAAM,UAAU,0BAA0B,CAAC,OAAoC;IAC7E,MAAM,QAAQ,GAAG,8BAA8B,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAClE,MAAM,SAAS,GAAG,OAAO,CAAC,gBAAgB,IAAI,sBAAsB,CAAC,gBAAgB,CAAC;IACtF,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,SAAS,GAAG,CAAC,IAAI,SAAS,GAAG,sBAAsB,CAAC,mBAAmB;QAAE,MAAM,IAAI,gBAAgB,CAAC,iBAAiB,EAAE,gBAAgB,CAAC,CAAC;IAC5K,IAAI,GAAG,GAAG,CAAC,EAAE,IAAI,GAAG,KAAK,EAAE,KAAK,GAAG,KAAK,EAAE,MAAM,GAAG,KAAK,CAAC;IACzD,IAAI,OACyH,CAAC;IAC9H,MAAM,MAAM,GAAG,GAAS,EAAE;QACxB,IAAI,CAAC,OAAO,EAAE,KAAK,IAAI,CAAC,OAAO,CAAC,aAAa;YAAE,OAAO;QACtD,MAAM,SAAS,GAAG,OAAO,CAAC;QAAC,OAAO,GAAG,SAAS,CAAC;QAAC,SAAS,CAAC,OAAO,EAAE,CAAC;QACpE,IAAI,SAAS,CAAC,KAAM,CAAC,EAAE;YAAE,SAAS,CAAC,OAAO,CAAC,SAAS,CAAC,KAAM,CAAC,CAAC;aACxD,CAAC;YACJ,SAAS,CAAC,KAAK,CAAC,SAAS,CAAC,KAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAC7C,SAAS,CAAC,MAAM,CAAC,IAAI,gBAAgB,CAAC,SAAS,CAAC,KAAM,CAAC,KAAK,CAAC,IAAI,EAAE,SAAS,CAAC,KAAM,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC;QAC1G,CAAC;IACH,CAAC,CAAC;IACF,MAAM,SAAS,GAAG,IAAI,uBAAuB,CAAC,OAAO,CAAC,SAAS,EAAE,KAAK,CAAC,EAAE;QACvE,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;YAAC,SAAS,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;YAAC,OAAO;QAAC,CAAC;QAC/E,IAAI,KAA0B,CAAC;QAC/B,IAAI,CAAC;YAAC,KAAK,GAAG,wBAAwB,CAAC,KAAK,CAAC,CAAC;QAAC,CAAC;QAAC,MAAM,CAAC;YAAC,SAAS,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;YAAC,OAAO;QAAC,CAAC;QACvG,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;QAChC,IAAI,CAAC,0BAA0B,CAAC,QAAQ,EAAE,KAAK,CAAC,QAAQ,CAAC,IAAI,OAAO,CAAC,GAAG,KAAK,KAAK,CAAC,GAAG,IAAI,OAAO,CAAC,SAAS,KAAK,KAAK,CAAC,SAAS;eAC1H,OAAO,CAAC,SAAS,KAAK,KAAK,CAAC,SAAS,IAAI,CAAC,OAAO,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,KAAK,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,KAAK,SAAS,CAAC,EAAE,CAAC;YACrJ,SAAS,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC;YAAC,OAAO;QAC/C,CAAC;QACD,OAAO,CAAC,KAAK,GAAG,KAAK,CAAC;QAAC,MAAM,EAAE,CAAC;IAClC,CAAC,EAAE,IAAI,CAAC,EAAE;QACR,MAAM,GAAG,IAAI,CAAC;QACd,MAAM,SAAS,GAAG,OAAO,CAAC;QAAC,OAAO,GAAG,SAAS,CAAC;QAC/C,IAAI,SAAS,EAAE,CAAC;YAAC,SAAS,CAAC,OAAO,EAAE,CAAC;YAAC,SAAS,CAAC,MAAM,CAAC,IAAI,gBAAgB,CAAC,IAAI,EAAE,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC;QAAC,CAAC;IACnJ,CAAC,CAAC,CAAC;IACH,MAAM,QAAQ,GAAG,CAAC,SAA0C,EAAE,MAAkB,EAAE,MAAoB,EAAgC,EAAE;QACtI,IAAI,MAAM;YAAE,OAAO,OAAO,CAAC,MAAM,CAAC,IAAI,gBAAgB,CAAC,iBAAiB,EAAE,gBAAgB,CAAC,CAAC,CAAC;QAC7F,IAAI,MAAM,EAAE,OAAO;YAAE,OAAO,OAAO,CAAC,MAAM,CAAC,IAAI,gBAAgB,CAAC,WAAW,EAAE,gBAAgB,CAAC,CAAC,CAAC;QAChG,IAAI,GAAG,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;YAAC,SAAS,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC;YAAC,OAAO,OAAO,CAAC,MAAM,CAAC,IAAI,gBAAgB,CAAC,mBAAmB,EAAE,gBAAgB,CAAC,CAAC,CAAC;QAAC,CAAC;QACjK,GAAG,IAAI,CAAC,CAAC;QACT,MAAM,IAAI,GAAG,EAAE,OAAO,EAAE,uBAAyD,EAAE,IAAI,EAAE,SAAkB,EAAE,QAAQ,EAAE,GAAG,EAAE,SAAS,EAAE,WAAW,GAAG,EAAE,EAAE,CAAC;QAC1J,MAAM,OAAO,GAA0B,SAAS,KAAK,SAAS;YAC5D,CAAC,CAAC,EAAE,GAAG,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,UAAU,GAAG,EAAE,EAAE,MAAM,EAAE,MAAO,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,EAAE,SAAS,EAAE,CAAC;QAChG,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,MAAM,KAAK,GAAG,GAAS,EAAE,CAAC,SAAS,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;YACvD,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,KAAK,CAAC,mBAAmB,CAAC,EAAE,SAAS,CAAC,CAAC;YAChF,MAAM,IAAI,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,aAAa,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM;gBAC3E,OAAO,EAAE,GAAG,EAAE,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,mBAAmB,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YACzF,OAAO,GAAG,IAAI,CAAC;YAAC,MAAM,EAAE,gBAAgB,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;YACzE,KAAK,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE;gBACrE,IAAI,OAAO,KAAK,IAAI;oBAAE,OAAO;gBAC7B,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;gBAAC,MAAM,EAAE,CAAC;YACtC,CAAC,EAAE,GAAG,EAAE,GAAG,IAAI,OAAO,KAAK,IAAI;gBAAE,SAAS,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC3E,CAAC,CAAC,CAAC;IACL,CAAC,CAAC;IACF,MAAM,aAAa,GAAG,KAAK,EAAK,IAAsB,EAAc,EAAE;QACpE,IAAI,MAAM;YAAE,MAAM,IAAI,gBAAgB,CAAC,iBAAiB,EAAE,gBAAgB,CAAC,CAAC;QAC5E,IAAI,IAAI;YAAE,MAAM,IAAI,gBAAgB,CAAC,eAAe,EAAE,gBAAgB,CAAC,CAAC;QACxE,IAAI,GAAG,IAAI,CAAC;QACZ,IAAI,CAAC;YAAC,OAAO,MAAM,IAAI,EAAE,CAAC;QAAC,CAAC;gBAAS,CAAC;YAAC,IAAI,GAAG,KAAK,CAAC;QAAC,CAAC;IACxD,CAAC,CAAC;IACF,MAAM,WAAW,GAAG,KAAK,EAAE,MAAoB,EAAiB,EAAE;QAChE,IAAI,KAAK;YAAE,OAAO;QAClB,MAAM,QAAQ,CAAC,OAAO,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;QAAC,KAAK,GAAG,IAAI,CAAC;IAC3D,CAAC,CAAC;IACF,MAAM,QAAQ,GAA4D;QACxE,aAAa,EAAE,aAAa;QAC5B,OAAO,EAAE,GAAG,EAAE,CAAC,aAAa,CAAC,KAAK,IAA6B,EAAE;YAC/D,MAAM,WAAW,EAAE,CAAC;YACpB,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,SAAS,CAAC,CAAC;YACxC,IAAI,CAAC;gBACH,IAAI,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,WAAW;oBAAE,MAAM,IAAI,KAAK,EAAE,CAAC;gBACvD,OAAO,+BAA+B,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;YAC5D,CAAC;YAAC,MAAM,CAAC;gBAAC,SAAS,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;gBAAC,MAAM,IAAI,gBAAgB,CAAC,kBAAkB,EAAE,mBAAmB,CAAC,CAAC;YAAC,CAAC;QACvH,CAAC,CAAC;QACF,OAAO,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,CAAC,aAAa,CAAC,KAAK,IAAI,EAAE;YACpD,MAAM,WAAW,GAAG,4BAA4B,CAAC,MAAM,CAAC,CAAC;YACzD,IAAI,MAAM,EAAE,OAAO;gBAAE,MAAM,IAAI,gBAAgB,CAAC,WAAW,EAAE,gBAAgB,CAAC,CAAC;YAC/E,IAAI,CAAC;gBAAC,MAAM,WAAW,CAAC,MAAM,CAAC,CAAC;YAAC,CAAC;YAClC,OAAO,KAAK,EAAE,CAAC;gBACb,IAAI,KAAK,YAAY,gBAAgB;oBAAE,MAAM,IAAI,gBAAgB,CAAC,KAAK,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAAC;gBAChG,MAAM,KAAK,CAAC;YACd,CAAC;YACD,MAAM,QAAQ,CAAC,SAAS,EAAE,WAAW,EAAE,MAAM,CAAC,CAAC;QACjD,CAAC,CAAC;KACH,CAAC;IACF,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,aAAa,CAAC,WAAW,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,SAAS,CAAC,KAAK,EAAE,EAAE,CAAC;AAC/F,CAAC"}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import type { Duplex } from "node:stream";
|
|
2
|
+
import type { CuaExecutor } from "./computer-use.js";
|
|
3
|
+
import { type BrowserControlIdentity } from "./browser-control-protocol.js";
|
|
4
|
+
export interface BrowserControlDispatcherOptions {
|
|
5
|
+
transport: Duplex;
|
|
6
|
+
identity: BrowserControlIdentity;
|
|
7
|
+
executor: CuaExecutor;
|
|
8
|
+
isAuthorized: () => boolean;
|
|
9
|
+
authoritySignal: AbortSignal;
|
|
10
|
+
}
|
|
11
|
+
/** Caller retains physical runtime ownership. Closing this channel does not claim the browser stopped. */
|
|
12
|
+
export declare function attachBrowserControlDispatcher(options: BrowserControlDispatcherOptions): {
|
|
13
|
+
close(): void;
|
|
14
|
+
};
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
import { CuaExecutorError } from "./cua-executor-error.js";
|
|
2
|
+
import { BROWSER_CONTROL_LIMITS, BROWSER_CONTROL_VERSION, encodeBrowserControlObservation, parseBrowserControlRequest, safeBrowserControlFailure, sameBrowserControlIdentity, validateBrowserControlIdentity } from "./browser-control-protocol.js";
|
|
3
|
+
import { BrowserControlTransport } from "./browser-control-transport.js";
|
|
4
|
+
/** Caller retains physical runtime ownership. Closing this channel does not claim the browser stopped. */
|
|
5
|
+
export function attachBrowserControlDispatcher(options) {
|
|
6
|
+
const identity = validateBrowserControlIdentity(options.identity);
|
|
7
|
+
let lastSeq = 0, busy = false, ready = false, closed = false;
|
|
8
|
+
let active;
|
|
9
|
+
let timer;
|
|
10
|
+
const dispose = () => { clearTimeout(timer); timer = undefined; active?.abort(); active = undefined; };
|
|
11
|
+
const authorized = () => {
|
|
12
|
+
try {
|
|
13
|
+
return !options.authoritySignal.aborted && options.isAuthorized() === true;
|
|
14
|
+
}
|
|
15
|
+
catch {
|
|
16
|
+
return false;
|
|
17
|
+
}
|
|
18
|
+
};
|
|
19
|
+
const transport = new BrowserControlTransport(options.transport, value => {
|
|
20
|
+
if (closed)
|
|
21
|
+
return;
|
|
22
|
+
if (busy) {
|
|
23
|
+
transport.close("executor_busy");
|
|
24
|
+
return;
|
|
25
|
+
}
|
|
26
|
+
let request;
|
|
27
|
+
try {
|
|
28
|
+
request = parseBrowserControlRequest(value);
|
|
29
|
+
}
|
|
30
|
+
catch {
|
|
31
|
+
transport.close("invalid_request");
|
|
32
|
+
return;
|
|
33
|
+
}
|
|
34
|
+
if (!sameBrowserControlIdentity(identity, request.identity) || request.seq !== lastSeq + 1
|
|
35
|
+
|| (request.operation === "HELLO" ? ready || request.seq !== 1 : !ready)) {
|
|
36
|
+
transport.close("protocol_mismatch");
|
|
37
|
+
return;
|
|
38
|
+
}
|
|
39
|
+
// Reserve before invoking any async work: coalesced frames cannot race this boundary.
|
|
40
|
+
busy = true;
|
|
41
|
+
lastSeq = request.seq;
|
|
42
|
+
active = new AbortController();
|
|
43
|
+
timer = setTimeout(() => transport.close("deadline_exceeded"), BROWSER_CONTROL_LIMITS.requestTimeoutMs);
|
|
44
|
+
void dispatch(request, active.signal);
|
|
45
|
+
}, () => {
|
|
46
|
+
closed = true;
|
|
47
|
+
dispose();
|
|
48
|
+
options.authoritySignal.removeEventListener("abort", revoke);
|
|
49
|
+
});
|
|
50
|
+
function revoke() { transport.close("session_revoked"); }
|
|
51
|
+
async function dispatch(request, signal) {
|
|
52
|
+
const common = { version: BROWSER_CONTROL_VERSION, type: "reply", identity, seq: request.seq,
|
|
53
|
+
requestId: request.requestId, operation: request.operation,
|
|
54
|
+
...(request.operation === "EXECUTE" ? { actionId: request.actionId } : {}) };
|
|
55
|
+
let invoked = false, reply;
|
|
56
|
+
try {
|
|
57
|
+
if (!authorized() || signal.aborted)
|
|
58
|
+
throw new CuaExecutorError("session_revoked", "not_dispatched");
|
|
59
|
+
if (request.operation === "HELLO") {
|
|
60
|
+
ready = true;
|
|
61
|
+
reply = { ...common, ok: true };
|
|
62
|
+
}
|
|
63
|
+
else if (request.operation === "OBSERVE") {
|
|
64
|
+
invoked = true;
|
|
65
|
+
const observation = await options.executor.observe();
|
|
66
|
+
if (closed || signal.aborted || !authorized()) {
|
|
67
|
+
transport.close("session_revoked");
|
|
68
|
+
return;
|
|
69
|
+
}
|
|
70
|
+
reply = { ...common, ok: true, observation: encodeBrowserControlObservation(observation) };
|
|
71
|
+
}
|
|
72
|
+
else {
|
|
73
|
+
// No await between the owner gate above and invocation. The physical driver must
|
|
74
|
+
// check this signal again immediately before each actual input after preparation.
|
|
75
|
+
invoked = true;
|
|
76
|
+
await options.executor.execute(request.action, signal);
|
|
77
|
+
if (closed || signal.aborted || !authorized()) {
|
|
78
|
+
transport.close("session_revoked");
|
|
79
|
+
return;
|
|
80
|
+
}
|
|
81
|
+
reply = { ...common, ok: true };
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
catch (error) {
|
|
85
|
+
reply = { ...common, ok: false, error: safeBrowserControlFailure(error, invoked) };
|
|
86
|
+
}
|
|
87
|
+
if (closed)
|
|
88
|
+
return;
|
|
89
|
+
try {
|
|
90
|
+
await transport.send(reply);
|
|
91
|
+
if (!reply.ok) {
|
|
92
|
+
transport.close(reply.error.code);
|
|
93
|
+
return;
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
catch {
|
|
97
|
+
transport.close("transport_failed");
|
|
98
|
+
}
|
|
99
|
+
finally {
|
|
100
|
+
dispose();
|
|
101
|
+
busy = false;
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
options.authoritySignal.addEventListener("abort", revoke, { once: true });
|
|
105
|
+
if (!authorized())
|
|
106
|
+
transport.close("session_revoked");
|
|
107
|
+
return { close: () => transport.close() };
|
|
108
|
+
}
|
|
109
|
+
//# sourceMappingURL=browser-control-dispatcher.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"browser-control-dispatcher.js","sourceRoot":"","sources":["../src/browser-control-dispatcher.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAC3D,OAAO,EACL,sBAAsB,EAAE,uBAAuB,EAAE,+BAA+B,EAAE,0BAA0B,EAC5G,yBAAyB,EAAE,0BAA0B,EAAE,8BAA8B,EAEtF,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAAE,uBAAuB,EAAE,MAAM,gCAAgC,CAAC;AAMzE,0GAA0G;AAC1G,MAAM,UAAU,8BAA8B,CAAC,OAAwC;IACrF,MAAM,QAAQ,GAAG,8BAA8B,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAClE,IAAI,OAAO,GAAG,CAAC,EAAE,IAAI,GAAG,KAAK,EAAE,KAAK,GAAG,KAAK,EAAE,MAAM,GAAG,KAAK,CAAC;IAC7D,IAAI,MAAmC,CAAC;IACxC,IAAI,KAAgD,CAAC;IACrD,MAAM,OAAO,GAAG,GAAS,EAAE,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,GAAG,SAAS,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;IAC7G,MAAM,UAAU,GAAG,GAAY,EAAE;QAC/B,IAAI,CAAC;YAAC,OAAO,CAAC,OAAO,CAAC,eAAe,CAAC,OAAO,IAAI,OAAO,CAAC,YAAY,EAAE,KAAK,IAAI,CAAC;QAAC,CAAC;QAAC,MAAM,CAAC;YAAC,OAAO,KAAK,CAAC;QAAC,CAAC;IAC7G,CAAC,CAAC;IACF,MAAM,SAAS,GAAG,IAAI,uBAAuB,CAAC,OAAO,CAAC,SAAS,EAAE,KAAK,CAAC,EAAE;QACvE,IAAI,MAAM;YAAE,OAAO;QACnB,IAAI,IAAI,EAAE,CAAC;YAAC,SAAS,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;YAAC,OAAO;QAAC,CAAC;QACvD,IAAI,OAA8B,CAAC;QACnC,IAAI,CAAC;YAAC,OAAO,GAAG,0BAA0B,CAAC,KAAK,CAAC,CAAC;QAAC,CAAC;QAAC,MAAM,CAAC;YAAC,SAAS,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;YAAC,OAAO;QAAC,CAAC;QAC1G,IAAI,CAAC,0BAA0B,CAAC,QAAQ,EAAE,OAAO,CAAC,QAAQ,CAAC,IAAI,OAAO,CAAC,GAAG,KAAK,OAAO,GAAG,CAAC;eACrF,CAAC,OAAO,CAAC,SAAS,KAAK,OAAO,CAAC,CAAC,CAAC,KAAK,IAAI,OAAO,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC;YAAC,SAAS,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC;YAAC,OAAO;QAAC,CAAC;QAC7H,sFAAsF;QACtF,IAAI,GAAG,IAAI,CAAC;QAAC,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC;QAAC,MAAM,GAAG,IAAI,eAAe,EAAE,CAAC;QACnE,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,KAAK,CAAC,mBAAmB,CAAC,EAAE,sBAAsB,CAAC,gBAAgB,CAAC,CAAC;QACxG,KAAK,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;IACxC,CAAC,EAAE,GAAG,EAAE;QACN,MAAM,GAAG,IAAI,CAAC;QAAC,OAAO,EAAE,CAAC;QAAC,OAAO,CAAC,eAAe,CAAC,mBAAmB,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IACzF,CAAC,CAAC,CAAC;IACH,SAAS,MAAM,KAAW,SAAS,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC;IAC/D,KAAK,UAAU,QAAQ,CAAC,OAA8B,EAAE,MAAmB;QACzE,MAAM,MAAM,GAAG,EAAE,OAAO,EAAE,uBAAyD,EAAE,IAAI,EAAE,OAAgB,EAAE,QAAQ,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG;YACrI,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS;YAC1D,GAAG,CAAC,OAAO,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;QAC/E,IAAI,OAAO,GAAG,KAAK,EAAE,KAA0B,CAAC;QAChD,IAAI,CAAC;YACH,IAAI,CAAC,UAAU,EAAE,IAAI,MAAM,CAAC,OAAO;gBAAE,MAAM,IAAI,gBAAgB,CAAC,iBAAiB,EAAE,gBAAgB,CAAC,CAAC;YACrG,IAAI,OAAO,CAAC,SAAS,KAAK,OAAO,EAAE,CAAC;gBAAC,KAAK,GAAG,IAAI,CAAC;gBAAC,KAAK,GAAG,EAAE,GAAG,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC;YAAC,CAAC;iBAChF,IAAI,OAAO,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;gBACzC,OAAO,GAAG,IAAI,CAAC;gBACf,MAAM,WAAW,GAAG,MAAM,OAAO,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC;gBACrD,IAAI,MAAM,IAAI,MAAM,CAAC,OAAO,IAAI,CAAC,UAAU,EAAE,EAAE,CAAC;oBAAC,SAAS,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;oBAAC,OAAO;gBAAC,CAAC;gBAC9F,KAAK,GAAG,EAAE,GAAG,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,+BAA+B,CAAC,WAAW,CAAC,EAAE,CAAC;YAC7F,CAAC;iBAAM,CAAC;gBACN,iFAAiF;gBACjF,kFAAkF;gBAClF,OAAO,GAAG,IAAI,CAAC;gBACf,MAAM,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;gBACvD,IAAI,MAAM,IAAI,MAAM,CAAC,OAAO,IAAI,CAAC,UAAU,EAAE,EAAE,CAAC;oBAAC,SAAS,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;oBAAC,OAAO;gBAAC,CAAC;gBAC9F,KAAK,GAAG,EAAE,GAAG,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC;YAClC,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAAC,KAAK,GAAG,EAAE,GAAG,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,yBAAyB,CAAC,KAAK,EAAE,OAAO,CAAC,EAAE,CAAC;QAAC,CAAC;QACvG,IAAI,MAAM;YAAE,OAAO;QACnB,IAAI,CAAC;YACH,MAAM,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC5B,IAAI,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC;gBAAC,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBAAC,OAAO;YAAC,CAAC;QAC/D,CAAC;QAAC,MAAM,CAAC;YAAC,SAAS,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;QAAC,CAAC;gBACxC,CAAC;YAAC,OAAO,EAAE,CAAC;YAAC,IAAI,GAAG,KAAK,CAAC;QAAC,CAAC;IACtC,CAAC;IACD,OAAO,CAAC,eAAe,CAAC,gBAAgB,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;IAC1E,IAAI,CAAC,UAAU,EAAE;QAAE,SAAS,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;IACtD,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,SAAS,CAAC,KAAK,EAAE,EAAE,CAAC;AAC5C,CAAC"}
|
|
@@ -0,0 +1,371 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
import type { CuaAction, CuaObservation } from "./computer-use.js";
|
|
3
|
+
import { type CuaExecutorErrorCode } from "./cua-executor-error.js";
|
|
4
|
+
export declare const BROWSER_CONTROL_VERSION = 1;
|
|
5
|
+
export declare const BROWSER_CONTROL_LIMITS: Readonly<{
|
|
6
|
+
frameBytes: number;
|
|
7
|
+
pngBytes: number;
|
|
8
|
+
dimension: 4096;
|
|
9
|
+
pixels: 16000000;
|
|
10
|
+
textBytes: number;
|
|
11
|
+
chordKeys: 16;
|
|
12
|
+
keyCharacters: 64;
|
|
13
|
+
dragPoints: 1024;
|
|
14
|
+
waitMs: 30000;
|
|
15
|
+
requestTimeoutMs: 35000;
|
|
16
|
+
maxRequestTimeoutMs: 60000;
|
|
17
|
+
}>;
|
|
18
|
+
export interface BrowserControlIdentity {
|
|
19
|
+
generation: string;
|
|
20
|
+
challenge: string;
|
|
21
|
+
runtimeRevision: string;
|
|
22
|
+
}
|
|
23
|
+
export declare const browserControlActionSchema: z.ZodPipe<z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
24
|
+
button: z.ZodOptional<z.ZodEnum<{
|
|
25
|
+
left: "left";
|
|
26
|
+
right: "right";
|
|
27
|
+
middle: "middle";
|
|
28
|
+
}>>;
|
|
29
|
+
x: z.ZodNumber;
|
|
30
|
+
y: z.ZodNumber;
|
|
31
|
+
kind: z.ZodLiteral<"click">;
|
|
32
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
33
|
+
x: z.ZodNumber;
|
|
34
|
+
y: z.ZodNumber;
|
|
35
|
+
kind: z.ZodLiteral<"double_click">;
|
|
36
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
37
|
+
x: z.ZodNumber;
|
|
38
|
+
y: z.ZodNumber;
|
|
39
|
+
kind: z.ZodLiteral<"move">;
|
|
40
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
41
|
+
dx: z.ZodNumber;
|
|
42
|
+
dy: z.ZodNumber;
|
|
43
|
+
x: z.ZodNumber;
|
|
44
|
+
y: z.ZodNumber;
|
|
45
|
+
kind: z.ZodLiteral<"scroll">;
|
|
46
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
47
|
+
kind: z.ZodLiteral<"type">;
|
|
48
|
+
text: z.ZodString;
|
|
49
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
50
|
+
kind: z.ZodLiteral<"keypress">;
|
|
51
|
+
keys: z.ZodArray<z.ZodString>;
|
|
52
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
53
|
+
kind: z.ZodLiteral<"drag">;
|
|
54
|
+
path: z.ZodArray<z.ZodObject<{
|
|
55
|
+
x: z.ZodNumber;
|
|
56
|
+
y: z.ZodNumber;
|
|
57
|
+
}, z.core.$strict>>;
|
|
58
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
59
|
+
kind: z.ZodLiteral<"wait">;
|
|
60
|
+
ms: z.ZodOptional<z.ZodNumber>;
|
|
61
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
62
|
+
kind: z.ZodLiteral<"screenshot">;
|
|
63
|
+
}, z.core.$strict>], "kind">, z.ZodTransform<{
|
|
64
|
+
kind: "click";
|
|
65
|
+
x: number;
|
|
66
|
+
y: number;
|
|
67
|
+
button?: "left" | "right" | "middle";
|
|
68
|
+
} | {
|
|
69
|
+
kind: "double_click";
|
|
70
|
+
x: number;
|
|
71
|
+
y: number;
|
|
72
|
+
} | {
|
|
73
|
+
kind: "move";
|
|
74
|
+
x: number;
|
|
75
|
+
y: number;
|
|
76
|
+
} | {
|
|
77
|
+
kind: "scroll";
|
|
78
|
+
x: number;
|
|
79
|
+
y: number;
|
|
80
|
+
dx: number;
|
|
81
|
+
dy: number;
|
|
82
|
+
} | {
|
|
83
|
+
kind: "type";
|
|
84
|
+
text: string;
|
|
85
|
+
} | {
|
|
86
|
+
kind: "keypress";
|
|
87
|
+
keys: string[];
|
|
88
|
+
} | {
|
|
89
|
+
kind: "drag";
|
|
90
|
+
path: Array<{
|
|
91
|
+
x: number;
|
|
92
|
+
y: number;
|
|
93
|
+
}>;
|
|
94
|
+
} | {
|
|
95
|
+
kind: "wait";
|
|
96
|
+
ms?: number;
|
|
97
|
+
} | {
|
|
98
|
+
kind: "screenshot";
|
|
99
|
+
}, {
|
|
100
|
+
x: number;
|
|
101
|
+
y: number;
|
|
102
|
+
kind: "click";
|
|
103
|
+
button?: "left" | "right" | "middle" | undefined;
|
|
104
|
+
} | {
|
|
105
|
+
x: number;
|
|
106
|
+
y: number;
|
|
107
|
+
kind: "double_click";
|
|
108
|
+
} | {
|
|
109
|
+
x: number;
|
|
110
|
+
y: number;
|
|
111
|
+
kind: "move";
|
|
112
|
+
} | {
|
|
113
|
+
dx: number;
|
|
114
|
+
dy: number;
|
|
115
|
+
x: number;
|
|
116
|
+
y: number;
|
|
117
|
+
kind: "scroll";
|
|
118
|
+
} | {
|
|
119
|
+
kind: "type";
|
|
120
|
+
text: string;
|
|
121
|
+
} | {
|
|
122
|
+
kind: "keypress";
|
|
123
|
+
keys: string[];
|
|
124
|
+
} | {
|
|
125
|
+
kind: "drag";
|
|
126
|
+
path: {
|
|
127
|
+
x: number;
|
|
128
|
+
y: number;
|
|
129
|
+
}[];
|
|
130
|
+
} | {
|
|
131
|
+
kind: "wait";
|
|
132
|
+
ms?: number | undefined;
|
|
133
|
+
} | {
|
|
134
|
+
kind: "screenshot";
|
|
135
|
+
}>>;
|
|
136
|
+
declare const observationSchema: z.ZodObject<{
|
|
137
|
+
png: z.ZodString;
|
|
138
|
+
stateSignature: z.ZodString;
|
|
139
|
+
url: z.ZodOptional<z.ZodString>;
|
|
140
|
+
title: z.ZodOptional<z.ZodString>;
|
|
141
|
+
text: z.ZodOptional<z.ZodString>;
|
|
142
|
+
scrollY: z.ZodOptional<z.ZodNumber>;
|
|
143
|
+
}, z.core.$strict>;
|
|
144
|
+
type WireObservation = z.infer<typeof observationSchema>;
|
|
145
|
+
declare const requestSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
146
|
+
type: z.ZodLiteral<"request">;
|
|
147
|
+
operation: z.ZodLiteral<"HELLO">;
|
|
148
|
+
version: z.ZodLiteral<1>;
|
|
149
|
+
identity: z.ZodObject<{
|
|
150
|
+
generation: z.ZodString;
|
|
151
|
+
challenge: z.ZodString;
|
|
152
|
+
runtimeRevision: z.ZodString;
|
|
153
|
+
}, z.core.$strict>;
|
|
154
|
+
seq: z.ZodNumber;
|
|
155
|
+
requestId: z.ZodString;
|
|
156
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
157
|
+
type: z.ZodLiteral<"request">;
|
|
158
|
+
operation: z.ZodLiteral<"OBSERVE">;
|
|
159
|
+
version: z.ZodLiteral<1>;
|
|
160
|
+
identity: z.ZodObject<{
|
|
161
|
+
generation: z.ZodString;
|
|
162
|
+
challenge: z.ZodString;
|
|
163
|
+
runtimeRevision: z.ZodString;
|
|
164
|
+
}, z.core.$strict>;
|
|
165
|
+
seq: z.ZodNumber;
|
|
166
|
+
requestId: z.ZodString;
|
|
167
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
168
|
+
type: z.ZodLiteral<"request">;
|
|
169
|
+
operation: z.ZodLiteral<"EXECUTE">;
|
|
170
|
+
actionId: z.ZodString;
|
|
171
|
+
action: z.ZodPipe<z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
172
|
+
button: z.ZodOptional<z.ZodEnum<{
|
|
173
|
+
left: "left";
|
|
174
|
+
right: "right";
|
|
175
|
+
middle: "middle";
|
|
176
|
+
}>>;
|
|
177
|
+
x: z.ZodNumber;
|
|
178
|
+
y: z.ZodNumber;
|
|
179
|
+
kind: z.ZodLiteral<"click">;
|
|
180
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
181
|
+
x: z.ZodNumber;
|
|
182
|
+
y: z.ZodNumber;
|
|
183
|
+
kind: z.ZodLiteral<"double_click">;
|
|
184
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
185
|
+
x: z.ZodNumber;
|
|
186
|
+
y: z.ZodNumber;
|
|
187
|
+
kind: z.ZodLiteral<"move">;
|
|
188
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
189
|
+
dx: z.ZodNumber;
|
|
190
|
+
dy: z.ZodNumber;
|
|
191
|
+
x: z.ZodNumber;
|
|
192
|
+
y: z.ZodNumber;
|
|
193
|
+
kind: z.ZodLiteral<"scroll">;
|
|
194
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
195
|
+
kind: z.ZodLiteral<"type">;
|
|
196
|
+
text: z.ZodString;
|
|
197
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
198
|
+
kind: z.ZodLiteral<"keypress">;
|
|
199
|
+
keys: z.ZodArray<z.ZodString>;
|
|
200
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
201
|
+
kind: z.ZodLiteral<"drag">;
|
|
202
|
+
path: z.ZodArray<z.ZodObject<{
|
|
203
|
+
x: z.ZodNumber;
|
|
204
|
+
y: z.ZodNumber;
|
|
205
|
+
}, z.core.$strict>>;
|
|
206
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
207
|
+
kind: z.ZodLiteral<"wait">;
|
|
208
|
+
ms: z.ZodOptional<z.ZodNumber>;
|
|
209
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
210
|
+
kind: z.ZodLiteral<"screenshot">;
|
|
211
|
+
}, z.core.$strict>], "kind">, z.ZodTransform<{
|
|
212
|
+
kind: "click";
|
|
213
|
+
x: number;
|
|
214
|
+
y: number;
|
|
215
|
+
button?: "left" | "right" | "middle";
|
|
216
|
+
} | {
|
|
217
|
+
kind: "double_click";
|
|
218
|
+
x: number;
|
|
219
|
+
y: number;
|
|
220
|
+
} | {
|
|
221
|
+
kind: "move";
|
|
222
|
+
x: number;
|
|
223
|
+
y: number;
|
|
224
|
+
} | {
|
|
225
|
+
kind: "scroll";
|
|
226
|
+
x: number;
|
|
227
|
+
y: number;
|
|
228
|
+
dx: number;
|
|
229
|
+
dy: number;
|
|
230
|
+
} | {
|
|
231
|
+
kind: "type";
|
|
232
|
+
text: string;
|
|
233
|
+
} | {
|
|
234
|
+
kind: "keypress";
|
|
235
|
+
keys: string[];
|
|
236
|
+
} | {
|
|
237
|
+
kind: "drag";
|
|
238
|
+
path: Array<{
|
|
239
|
+
x: number;
|
|
240
|
+
y: number;
|
|
241
|
+
}>;
|
|
242
|
+
} | {
|
|
243
|
+
kind: "wait";
|
|
244
|
+
ms?: number;
|
|
245
|
+
} | {
|
|
246
|
+
kind: "screenshot";
|
|
247
|
+
}, {
|
|
248
|
+
x: number;
|
|
249
|
+
y: number;
|
|
250
|
+
kind: "click";
|
|
251
|
+
button?: "left" | "right" | "middle" | undefined;
|
|
252
|
+
} | {
|
|
253
|
+
x: number;
|
|
254
|
+
y: number;
|
|
255
|
+
kind: "double_click";
|
|
256
|
+
} | {
|
|
257
|
+
x: number;
|
|
258
|
+
y: number;
|
|
259
|
+
kind: "move";
|
|
260
|
+
} | {
|
|
261
|
+
dx: number;
|
|
262
|
+
dy: number;
|
|
263
|
+
x: number;
|
|
264
|
+
y: number;
|
|
265
|
+
kind: "scroll";
|
|
266
|
+
} | {
|
|
267
|
+
kind: "type";
|
|
268
|
+
text: string;
|
|
269
|
+
} | {
|
|
270
|
+
kind: "keypress";
|
|
271
|
+
keys: string[];
|
|
272
|
+
} | {
|
|
273
|
+
kind: "drag";
|
|
274
|
+
path: {
|
|
275
|
+
x: number;
|
|
276
|
+
y: number;
|
|
277
|
+
}[];
|
|
278
|
+
} | {
|
|
279
|
+
kind: "wait";
|
|
280
|
+
ms?: number | undefined;
|
|
281
|
+
} | {
|
|
282
|
+
kind: "screenshot";
|
|
283
|
+
}>>;
|
|
284
|
+
version: z.ZodLiteral<1>;
|
|
285
|
+
identity: z.ZodObject<{
|
|
286
|
+
generation: z.ZodString;
|
|
287
|
+
challenge: z.ZodString;
|
|
288
|
+
runtimeRevision: z.ZodString;
|
|
289
|
+
}, z.core.$strict>;
|
|
290
|
+
seq: z.ZodNumber;
|
|
291
|
+
requestId: z.ZodString;
|
|
292
|
+
}, z.core.$strict>], "operation">;
|
|
293
|
+
declare const replySchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
294
|
+
ok: z.ZodLiteral<true>;
|
|
295
|
+
observation: z.ZodOptional<z.ZodObject<{
|
|
296
|
+
png: z.ZodString;
|
|
297
|
+
stateSignature: z.ZodString;
|
|
298
|
+
url: z.ZodOptional<z.ZodString>;
|
|
299
|
+
title: z.ZodOptional<z.ZodString>;
|
|
300
|
+
text: z.ZodOptional<z.ZodString>;
|
|
301
|
+
scrollY: z.ZodOptional<z.ZodNumber>;
|
|
302
|
+
}, z.core.$strict>>;
|
|
303
|
+
type: z.ZodLiteral<"reply">;
|
|
304
|
+
operation: z.ZodEnum<{
|
|
305
|
+
HELLO: "HELLO";
|
|
306
|
+
OBSERVE: "OBSERVE";
|
|
307
|
+
EXECUTE: "EXECUTE";
|
|
308
|
+
}>;
|
|
309
|
+
actionId: z.ZodOptional<z.ZodString>;
|
|
310
|
+
version: z.ZodLiteral<1>;
|
|
311
|
+
identity: z.ZodObject<{
|
|
312
|
+
generation: z.ZodString;
|
|
313
|
+
challenge: z.ZodString;
|
|
314
|
+
runtimeRevision: z.ZodString;
|
|
315
|
+
}, z.core.$strict>;
|
|
316
|
+
seq: z.ZodNumber;
|
|
317
|
+
requestId: z.ZodString;
|
|
318
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
319
|
+
ok: z.ZodLiteral<false>;
|
|
320
|
+
error: z.ZodObject<{
|
|
321
|
+
code: z.ZodEnum<{
|
|
322
|
+
invalid_response: "invalid_response";
|
|
323
|
+
cancelled: "cancelled";
|
|
324
|
+
executor_closed: "executor_closed";
|
|
325
|
+
executor_not_ready: "executor_not_ready";
|
|
326
|
+
executor_busy: "executor_busy";
|
|
327
|
+
invalid_request: "invalid_request";
|
|
328
|
+
protocol_mismatch: "protocol_mismatch";
|
|
329
|
+
session_revoked: "session_revoked";
|
|
330
|
+
transport_failed: "transport_failed";
|
|
331
|
+
deadline_exceeded: "deadline_exceeded";
|
|
332
|
+
action_rejected: "action_rejected";
|
|
333
|
+
execution_failed: "execution_failed";
|
|
334
|
+
}>;
|
|
335
|
+
disposition: z.ZodEnum<{
|
|
336
|
+
not_dispatched: "not_dispatched";
|
|
337
|
+
outcome_uncertain: "outcome_uncertain";
|
|
338
|
+
}>;
|
|
339
|
+
}, z.core.$strict>;
|
|
340
|
+
type: z.ZodLiteral<"reply">;
|
|
341
|
+
operation: z.ZodEnum<{
|
|
342
|
+
HELLO: "HELLO";
|
|
343
|
+
OBSERVE: "OBSERVE";
|
|
344
|
+
EXECUTE: "EXECUTE";
|
|
345
|
+
}>;
|
|
346
|
+
actionId: z.ZodOptional<z.ZodString>;
|
|
347
|
+
version: z.ZodLiteral<1>;
|
|
348
|
+
identity: z.ZodObject<{
|
|
349
|
+
generation: z.ZodString;
|
|
350
|
+
challenge: z.ZodString;
|
|
351
|
+
runtimeRevision: z.ZodString;
|
|
352
|
+
}, z.core.$strict>;
|
|
353
|
+
seq: z.ZodNumber;
|
|
354
|
+
requestId: z.ZodString;
|
|
355
|
+
}, z.core.$strict>], "ok">;
|
|
356
|
+
export type BrowserControlRequest = z.infer<typeof requestSchema>;
|
|
357
|
+
export type BrowserControlReply = z.infer<typeof replySchema>;
|
|
358
|
+
export declare function validateBrowserControlIdentity(value: unknown): BrowserControlIdentity;
|
|
359
|
+
export declare function sameBrowserControlIdentity(a: BrowserControlIdentity, b: BrowserControlIdentity): boolean;
|
|
360
|
+
export declare function parseBrowserControlRequest(value: unknown): BrowserControlRequest;
|
|
361
|
+
export declare function parseBrowserControlReply(value: unknown): BrowserControlReply;
|
|
362
|
+
export declare function validateBrowserControlAction(value: unknown): CuaAction;
|
|
363
|
+
/** Check dimensions before allocating decoder output; v1 accepts browser-style 8-bit, noninterlaced PNG. */
|
|
364
|
+
export declare function validateBrowserControlPng(bytes: Buffer): void;
|
|
365
|
+
export declare function encodeBrowserControlObservation(observation: CuaObservation): WireObservation;
|
|
366
|
+
export declare function decodeBrowserControlObservation(value: unknown): CuaObservation;
|
|
367
|
+
export declare function safeBrowserControlFailure(error: unknown, dispatched: boolean): {
|
|
368
|
+
code: CuaExecutorErrorCode;
|
|
369
|
+
disposition: "not_dispatched" | "outcome_uncertain";
|
|
370
|
+
};
|
|
371
|
+
export {};
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
import { PNG } from "pngjs";
|
|
2
|
+
import { z } from "zod";
|
|
3
|
+
import { CuaExecutorError, isCuaExecutorError } from "./cua-executor-error.js";
|
|
4
|
+
export const BROWSER_CONTROL_VERSION = 1;
|
|
5
|
+
export const BROWSER_CONTROL_LIMITS = Object.freeze({
|
|
6
|
+
frameBytes: 12 * 1024 * 1024, pngBytes: 8 * 1024 * 1024,
|
|
7
|
+
dimension: 4096, pixels: 16_000_000, textBytes: 64 * 1024,
|
|
8
|
+
chordKeys: 16, keyCharacters: 64, dragPoints: 1024, waitMs: 30_000,
|
|
9
|
+
requestTimeoutMs: 35_000, maxRequestTimeoutMs: 60_000
|
|
10
|
+
});
|
|
11
|
+
const token = z.string().min(1).max(128).regex(/^[A-Za-z0-9._-]+$/);
|
|
12
|
+
const identitySchema = z.strictObject({ generation: token, challenge: token, runtimeRevision: token });
|
|
13
|
+
const text = z.string().max(BROWSER_CONTROL_LIMITS.textBytes).refine(value => Buffer.byteLength(value) <= BROWSER_CONTROL_LIMITS.textBytes);
|
|
14
|
+
const coordinate = z.number().finite().min(-1_000_000).max(1_000_000);
|
|
15
|
+
const point = { x: coordinate, y: coordinate };
|
|
16
|
+
export const browserControlActionSchema = z.discriminatedUnion("kind", [
|
|
17
|
+
z.strictObject({ kind: z.literal("click"), ...point, button: z.enum(["left", "right", "middle"]).optional() }),
|
|
18
|
+
z.strictObject({ kind: z.literal("double_click"), ...point }),
|
|
19
|
+
z.strictObject({ kind: z.literal("move"), ...point }),
|
|
20
|
+
z.strictObject({ kind: z.literal("scroll"), ...point, dx: coordinate, dy: coordinate }),
|
|
21
|
+
z.strictObject({ kind: z.literal("type"), text }),
|
|
22
|
+
z.strictObject({ kind: z.literal("keypress"), keys: z.array(z.string().min(1).max(BROWSER_CONTROL_LIMITS.keyCharacters)).min(1).max(BROWSER_CONTROL_LIMITS.chordKeys) }),
|
|
23
|
+
z.strictObject({ kind: z.literal("drag"), path: z.array(z.strictObject(point)).min(1).max(BROWSER_CONTROL_LIMITS.dragPoints) }),
|
|
24
|
+
z.strictObject({ kind: z.literal("wait"), ms: z.number().finite().min(0).max(BROWSER_CONTROL_LIMITS.waitMs).optional() }),
|
|
25
|
+
z.strictObject({ kind: z.literal("screenshot") })
|
|
26
|
+
]).transform((action) => {
|
|
27
|
+
if (action.kind === "click")
|
|
28
|
+
return { kind: action.kind, x: action.x, y: action.y, ...(action.button !== undefined ? { button: action.button } : {}) };
|
|
29
|
+
if (action.kind === "wait")
|
|
30
|
+
return { kind: action.kind, ...(action.ms !== undefined ? { ms: action.ms } : {}) };
|
|
31
|
+
return action;
|
|
32
|
+
});
|
|
33
|
+
const observationSchema = z.strictObject({
|
|
34
|
+
png: z.string().min(1).max(4 * Math.ceil(BROWSER_CONTROL_LIMITS.pngBytes / 3)).regex(/^[A-Za-z0-9+/]*={0,2}$/).refine(value => value.length % 4 === 0),
|
|
35
|
+
stateSignature: text, url: text.optional(), title: text.optional(), text: text.optional(), scrollY: coordinate.optional()
|
|
36
|
+
});
|
|
37
|
+
const common = {
|
|
38
|
+
version: z.literal(BROWSER_CONTROL_VERSION), identity: identitySchema,
|
|
39
|
+
seq: z.number().int().min(1).max(Number.MAX_SAFE_INTEGER), requestId: token
|
|
40
|
+
};
|
|
41
|
+
const requestSchema = z.discriminatedUnion("operation", [
|
|
42
|
+
z.strictObject({ ...common, type: z.literal("request"), operation: z.literal("HELLO") }),
|
|
43
|
+
z.strictObject({ ...common, type: z.literal("request"), operation: z.literal("OBSERVE") }),
|
|
44
|
+
z.strictObject({ ...common, type: z.literal("request"), operation: z.literal("EXECUTE"), actionId: token, action: browserControlActionSchema })
|
|
45
|
+
]);
|
|
46
|
+
const errorCode = z.enum(["executor_closed", "executor_not_ready", "executor_busy", "cancelled", "invalid_request", "invalid_response", "protocol_mismatch", "session_revoked", "transport_failed", "deadline_exceeded", "action_rejected", "execution_failed"]);
|
|
47
|
+
const replyBase = { ...common, type: z.literal("reply"), operation: z.enum(["HELLO", "OBSERVE", "EXECUTE"]), actionId: token.optional() };
|
|
48
|
+
const replySchema = z.discriminatedUnion("ok", [
|
|
49
|
+
z.strictObject({ ...replyBase, ok: z.literal(true), observation: observationSchema.optional() }),
|
|
50
|
+
z.strictObject({ ...replyBase, ok: z.literal(false), error: z.strictObject({ code: errorCode, disposition: z.enum(["not_dispatched", "outcome_uncertain"]) }) })
|
|
51
|
+
]);
|
|
52
|
+
export function validateBrowserControlIdentity(value) {
|
|
53
|
+
const parsed = identitySchema.safeParse(value);
|
|
54
|
+
if (!parsed.success)
|
|
55
|
+
throw new CuaExecutorError("invalid_request", "not_dispatched");
|
|
56
|
+
return parsed.data;
|
|
57
|
+
}
|
|
58
|
+
export function sameBrowserControlIdentity(a, b) {
|
|
59
|
+
return a.generation === b.generation && a.challenge === b.challenge && a.runtimeRevision === b.runtimeRevision;
|
|
60
|
+
}
|
|
61
|
+
export function parseBrowserControlRequest(value) {
|
|
62
|
+
const parsed = requestSchema.safeParse(value);
|
|
63
|
+
if (!parsed.success)
|
|
64
|
+
throw new CuaExecutorError("invalid_request", "not_dispatched");
|
|
65
|
+
const request = parsed.data;
|
|
66
|
+
if (request.requestId !== `request-${request.seq}` || (request.operation === "EXECUTE" && request.actionId !== `action-${request.seq}`))
|
|
67
|
+
throw new CuaExecutorError("invalid_request", "not_dispatched");
|
|
68
|
+
return request;
|
|
69
|
+
}
|
|
70
|
+
export function parseBrowserControlReply(value) {
|
|
71
|
+
const parsed = replySchema.safeParse(value);
|
|
72
|
+
if (!parsed.success)
|
|
73
|
+
throw new CuaExecutorError("invalid_response", "outcome_uncertain");
|
|
74
|
+
const reply = parsed.data;
|
|
75
|
+
if ((reply.operation === "EXECUTE") !== (reply.actionId !== undefined)
|
|
76
|
+
|| (reply.ok && (reply.operation === "OBSERVE") !== (reply.observation !== undefined)))
|
|
77
|
+
throw new CuaExecutorError("invalid_response", "outcome_uncertain");
|
|
78
|
+
return reply;
|
|
79
|
+
}
|
|
80
|
+
export function validateBrowserControlAction(value) {
|
|
81
|
+
const parsed = browserControlActionSchema.safeParse(value);
|
|
82
|
+
if (!parsed.success)
|
|
83
|
+
throw new CuaExecutorError("invalid_request", "not_dispatched");
|
|
84
|
+
return parsed.data;
|
|
85
|
+
}
|
|
86
|
+
/** Check dimensions before allocating decoder output; v1 accepts browser-style 8-bit, noninterlaced PNG. */
|
|
87
|
+
export function validateBrowserControlPng(bytes) {
|
|
88
|
+
const fail = () => { throw new CuaExecutorError("invalid_response", "outcome_uncertain"); };
|
|
89
|
+
if (bytes.length < 45 || bytes.length > BROWSER_CONTROL_LIMITS.pngBytes
|
|
90
|
+
|| !bytes.subarray(0, 8).equals(Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]))
|
|
91
|
+
|| bytes.readUInt32BE(8) !== 13 || bytes.toString("ascii", 12, 16) !== "IHDR")
|
|
92
|
+
fail();
|
|
93
|
+
const width = bytes.readUInt32BE(16), height = bytes.readUInt32BE(20);
|
|
94
|
+
if (width === 0 || height === 0 || width > BROWSER_CONTROL_LIMITS.dimension || height > BROWSER_CONTROL_LIMITS.dimension
|
|
95
|
+
|| width * height > BROWSER_CONTROL_LIMITS.pixels || bytes[24] !== 8 || ![0, 2, 3, 4, 6].includes(bytes[25])
|
|
96
|
+
|| bytes[26] !== 0 || bytes[27] !== 0 || bytes[28] !== 0)
|
|
97
|
+
fail();
|
|
98
|
+
// Reject trailing bytes, malformed chunk lengths, duplicate IHDR and absent image data.
|
|
99
|
+
let offset = 8, sawData = false, sawEnd = false;
|
|
100
|
+
while (offset + 12 <= bytes.length) {
|
|
101
|
+
const length = bytes.readUInt32BE(offset), kind = bytes.toString("ascii", offset + 4, offset + 8);
|
|
102
|
+
if (length > bytes.length - offset - 12 || (kind === "IHDR" && offset !== 8))
|
|
103
|
+
fail();
|
|
104
|
+
if (kind === "IDAT")
|
|
105
|
+
sawData = true;
|
|
106
|
+
offset += 12 + length;
|
|
107
|
+
if (kind === "IEND") {
|
|
108
|
+
if (length !== 0 || offset !== bytes.length)
|
|
109
|
+
fail();
|
|
110
|
+
sawEnd = true;
|
|
111
|
+
break;
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
if (!sawData || !sawEnd)
|
|
115
|
+
fail();
|
|
116
|
+
try {
|
|
117
|
+
const decoded = PNG.sync.read(bytes, { checkCRC: true });
|
|
118
|
+
if (decoded.width !== width || decoded.height !== height)
|
|
119
|
+
fail();
|
|
120
|
+
}
|
|
121
|
+
catch {
|
|
122
|
+
fail();
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
export function encodeBrowserControlObservation(observation) {
|
|
126
|
+
// appState has no closed browser schema in v1. Refuse rather than silently drop it.
|
|
127
|
+
if (!observation || observation.appState !== undefined || !Buffer.isBuffer(observation.screenshot))
|
|
128
|
+
throw new CuaExecutorError("invalid_response", "outcome_uncertain");
|
|
129
|
+
validateBrowserControlPng(observation.screenshot);
|
|
130
|
+
const parsed = observationSchema.safeParse({ png: observation.screenshot.toString("base64"), stateSignature: observation.stateSignature,
|
|
131
|
+
...(observation.url !== undefined ? { url: observation.url } : {}), ...(observation.title !== undefined ? { title: observation.title } : {}),
|
|
132
|
+
...(observation.text !== undefined ? { text: observation.text } : {}), ...(observation.scrollY !== undefined ? { scrollY: observation.scrollY } : {}) });
|
|
133
|
+
if (!parsed.success)
|
|
134
|
+
throw new CuaExecutorError("invalid_response", "outcome_uncertain");
|
|
135
|
+
return parsed.data;
|
|
136
|
+
}
|
|
137
|
+
export function decodeBrowserControlObservation(value) {
|
|
138
|
+
const parsed = observationSchema.safeParse(value);
|
|
139
|
+
if (!parsed.success)
|
|
140
|
+
throw new CuaExecutorError("invalid_response", "outcome_uncertain");
|
|
141
|
+
const { png, ...state } = parsed.data;
|
|
142
|
+
const screenshot = Buffer.from(png, "base64");
|
|
143
|
+
if (screenshot.toString("base64") !== png)
|
|
144
|
+
throw new CuaExecutorError("invalid_response", "outcome_uncertain");
|
|
145
|
+
validateBrowserControlPng(screenshot);
|
|
146
|
+
return { screenshot, stateSignature: state.stateSignature,
|
|
147
|
+
...(state.url !== undefined ? { url: state.url } : {}), ...(state.title !== undefined ? { title: state.title } : {}),
|
|
148
|
+
...(state.text !== undefined ? { text: state.text } : {}), ...(state.scrollY !== undefined ? { scrollY: state.scrollY } : {}) };
|
|
149
|
+
}
|
|
150
|
+
export function safeBrowserControlFailure(error, dispatched) {
|
|
151
|
+
if (isCuaExecutorError(error))
|
|
152
|
+
return { code: error.code, disposition: error.disposition };
|
|
153
|
+
return { code: "action_rejected", disposition: dispatched ? "outcome_uncertain" : "not_dispatched" };
|
|
154
|
+
}
|
|
155
|
+
//# sourceMappingURL=browser-control-protocol.js.map
|