@trycua/cua 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +121 -0
- package/bin/cua.js +37 -0
- package/browser/cua_sdk-ffi.d.ts +6 -0
- package/browser/cua_sdk-ffi.js +3 -0
- package/browser/cua_sdk-ffi.ts +14 -0
- package/browser/cua_sdk.d.ts +2237 -0
- package/browser/cua_sdk.js +2607 -0
- package/browser/cua_sdk.ts +3644 -0
- package/browser/cyclops_sdk_schema-ffi.d.ts +6 -0
- package/browser/cyclops_sdk_schema-ffi.js +3 -0
- package/browser/cyclops_sdk_schema-ffi.ts +14 -0
- package/browser/cyclops_sdk_schema.d.ts +996 -0
- package/browser/cyclops_sdk_schema.js +2194 -0
- package/browser/cyclops_sdk_schema.ts +2934 -0
- package/browser/fleet_sdk-ffi.d.ts +28 -0
- package/browser/fleet_sdk-ffi.js +3 -0
- package/browser/fleet_sdk-ffi.ts +35 -0
- package/browser/fleet_sdk.d.ts +3056 -0
- package/browser/fleet_sdk.js +5088 -0
- package/browser/fleet_sdk.ts +6830 -0
- package/browser/index.d.ts +2 -0
- package/browser/index.js +18 -0
- package/browser/index.web.ts +34 -0
- package/browser/tsconfig.json +20 -0
- package/browser/wasm-bindgen/index.d.ts +2346 -0
- package/browser/wasm-bindgen/index.js +6642 -0
- package/browser/wasm-bindgen/index_bg.wasm +0 -0
- package/browser/wasm-bindgen/index_bg.wasm.d.ts +1154 -0
- package/dist/index.d.ts +118 -0
- package/dist/index.js +207 -0
- package/dist/mcp.d.ts +33 -0
- package/dist/mcp.js +41 -0
- package/dist/native/cua_sdk-ffi.d.ts +1038 -0
- package/dist/native/cua_sdk-ffi.js +5095 -0
- package/dist/native/cua_sdk.d.ts +15906 -0
- package/dist/native/cua_sdk.js +27061 -0
- package/dist/native/index.d.ts +7 -0
- package/dist/native/index.js +12 -0
- package/dist/native/node-runtime.d.ts +72 -0
- package/dist/native/node-runtime.js +35 -0
- package/dist/spaces/cursorArt.d.ts +27 -0
- package/dist/spaces/cursorArt.js +59 -0
- package/dist/spaces/errors.d.ts +81 -0
- package/dist/spaces/errors.js +102 -0
- package/dist/spaces/events.d.ts +154 -0
- package/dist/spaces/events.js +182 -0
- package/dist/spaces/groups.d.ts +130 -0
- package/dist/spaces/groups.js +275 -0
- package/dist/spaces/host.d.ts +71 -0
- package/dist/spaces/host.js +154 -0
- package/dist/spaces/index.d.ts +48 -0
- package/dist/spaces/index.js +44 -0
- package/dist/spaces/pip.d.ts +128 -0
- package/dist/spaces/pip.js +250 -0
- package/dist/spaces/presence.d.ts +187 -0
- package/dist/spaces/presence.js +449 -0
- package/dist/spaces/presenceTypes.check.d.ts +12 -0
- package/dist/spaces/presenceTypes.check.js +1 -0
- package/dist/spaces/presenceTypes.d.ts +47 -0
- package/dist/spaces/presenceTypes.js +7 -0
- package/dist/spaces/routines.d.ts +158 -0
- package/dist/spaces/routines.js +339 -0
- package/dist/spaces/thread.d.ts +147 -0
- package/dist/spaces/thread.js +285 -0
- package/dist/spaces/transport/http.d.ts +51 -0
- package/dist/spaces/transport/http.js +113 -0
- package/dist/spaces/transport/index.d.ts +19 -0
- package/dist/spaces/transport/index.js +19 -0
- package/dist/spaces/transport/session.d.ts +120 -0
- package/dist/spaces/transport/session.js +188 -0
- package/dist/spaces/transport/tauri.d.ts +50 -0
- package/dist/spaces/transport/tauri.js +90 -0
- package/dist/spaces/transport/types.d.ts +123 -0
- package/dist/spaces/transport/types.js +59 -0
- package/dist/teleport/controller.d.ts +34 -0
- package/dist/teleport/controller.js +107 -0
- package/dist/teleport/drop.d.ts +62 -0
- package/dist/teleport/drop.js +138 -0
- package/dist/teleport/dropZone.d.ts +59 -0
- package/dist/teleport/dropZone.js +166 -0
- package/dist/teleport/element.d.ts +28 -0
- package/dist/teleport/element.js +340 -0
- package/dist/teleport/index.d.ts +32 -0
- package/dist/teleport/index.js +32 -0
- package/dist/teleport/install.d.ts +31 -0
- package/dist/teleport/install.js +65 -0
- package/dist/teleport/model.d.ts +245 -0
- package/dist/teleport/model.js +334 -0
- package/dist/teleport/windowDrag.d.ts +86 -0
- package/dist/teleport/windowDrag.js +71 -0
- package/package.json +93 -0
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `@trycua/cua/spaces/host` — the host presentation adapter. **Explicitly unstable.**
|
|
3
|
+
*
|
|
4
|
+
* This is the one thing the SDK itself cannot do: put a Space on
|
|
5
|
+
* the *operator's own Mac desktop* — pinned as picture-in-picture, opened in a
|
|
6
|
+
* viewer window, or a single Space window streamed back bidirectionally over
|
|
7
|
+
* rcdp.
|
|
8
|
+
*
|
|
9
|
+
* It talks to the loopback control server inside the Cua Spaces desktop app,
|
|
10
|
+
* discovered through `$CUA_HOME/spaces-control.json` (default `~/.cua`; `{port, token}`). That means
|
|
11
|
+
* all of the following, and you should assume all of it will change:
|
|
12
|
+
*
|
|
13
|
+
* - it only works on the machine where the app is running, as the user who
|
|
14
|
+
* owns that home directory;
|
|
15
|
+
* - the port is ephemeral and the token is regenerated on every app launch;
|
|
16
|
+
* - a Local Space's rcdp token rotates on every rcdpd restart, so a token read
|
|
17
|
+
* once and cached will start failing — `streamWindow` takes a fresh one;
|
|
18
|
+
* - the control server resolves `local:<vm>` ids only. `pin` and `openViewer`
|
|
19
|
+
* with any other id are rejected by the app with HTTP 400, and this adapter
|
|
20
|
+
* says so rather than reporting success.
|
|
21
|
+
*
|
|
22
|
+
* Every method throws `host_unavailable` when the app is not running. None of
|
|
23
|
+
* them degrades to a silent no-op — a `forget_rcdp_token` with zero callers is
|
|
24
|
+
* exactly the kind of thing this codebase has shipped before.
|
|
25
|
+
*/
|
|
26
|
+
import { SpacesError, excerpt } from './errors.js';
|
|
27
|
+
/** `$CUA_HOME` when set and non-empty, else `~/.cua` (the Rust core's rule). */
|
|
28
|
+
export function cuaHome(home, env = globalThis.process?.env ?? {}) {
|
|
29
|
+
const set = env.CUA_HOME;
|
|
30
|
+
return set ? set : `${home.replace(/[\\/]+$/, '')}/.cua`;
|
|
31
|
+
}
|
|
32
|
+
/** Read `{port, token}` from the app's control file. */
|
|
33
|
+
export async function readControlEndpoint(controlFile) {
|
|
34
|
+
const { readFile } = await import('node:fs/promises');
|
|
35
|
+
const { homedir } = await import('node:os');
|
|
36
|
+
const { join } = await import('node:path');
|
|
37
|
+
const path = controlFile ?? join(cuaHome(homedir()), 'spaces-control.json');
|
|
38
|
+
let raw;
|
|
39
|
+
try {
|
|
40
|
+
raw = await readFile(path, 'utf8');
|
|
41
|
+
}
|
|
42
|
+
catch (cause) {
|
|
43
|
+
throw new SpacesError('host_unavailable', `the Cua Spaces app does not appear to be running: cannot read ${path}. ` +
|
|
44
|
+
`Host presentation (PiP, viewer, window stream) needs the desktop app on this machine.`, { cause });
|
|
45
|
+
}
|
|
46
|
+
let parsed;
|
|
47
|
+
try {
|
|
48
|
+
parsed = JSON.parse(raw);
|
|
49
|
+
}
|
|
50
|
+
catch (cause) {
|
|
51
|
+
throw SpacesError.protocol(`${path} is not valid JSON`, { cause, detail: excerpt(raw, 120) });
|
|
52
|
+
}
|
|
53
|
+
const record = parsed;
|
|
54
|
+
if (typeof record.port !== 'number' || typeof record.token !== 'string' || !record.token) {
|
|
55
|
+
throw SpacesError.protocol(`${path} does not contain a usable {port, token}`);
|
|
56
|
+
}
|
|
57
|
+
return { port: record.port, token: record.token };
|
|
58
|
+
}
|
|
59
|
+
export class HostPresentation {
|
|
60
|
+
options;
|
|
61
|
+
constructor(options = {}) {
|
|
62
|
+
this.options = options;
|
|
63
|
+
}
|
|
64
|
+
/** Pin a Local Space as picture-in-picture on this Mac's desktop. */
|
|
65
|
+
async pin(spaceId) {
|
|
66
|
+
this.assertLocal(spaceId, 'pin');
|
|
67
|
+
await this.post('/pip/pin', { space_id: spaceId });
|
|
68
|
+
}
|
|
69
|
+
async unpin(spaceId) {
|
|
70
|
+
await this.post('/pip/unpin', { space_id: spaceId });
|
|
71
|
+
}
|
|
72
|
+
/** Open the full viewer window for a Local Space. */
|
|
73
|
+
async openViewer(spaceId) {
|
|
74
|
+
this.assertLocal(spaceId, 'openViewer');
|
|
75
|
+
await this.post('/viewer/open', { space_id: spaceId });
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* Stream one window of a Local Space back to a bidirectional window here.
|
|
79
|
+
*
|
|
80
|
+
* `rcdpToken` must be read fresh: a Local Space's rcdp token rotates on every
|
|
81
|
+
* rcdpd restart, so a cached one silently stops working.
|
|
82
|
+
*
|
|
83
|
+
* `replica: true` opens an *additional* stream for the same target rather
|
|
84
|
+
* than focusing the existing one. Note the real limit: only one page per app
|
|
85
|
+
* can hardware-decode H.264, so a second concurrent stream falls back to an
|
|
86
|
+
* uncompressed codec and is visibly heavier.
|
|
87
|
+
*/
|
|
88
|
+
async streamWindow(input) {
|
|
89
|
+
this.assertLocal(input.spaceId, 'streamWindow');
|
|
90
|
+
if (!input.windowId)
|
|
91
|
+
throw SpacesError.usage('streamWindow requires a windowId');
|
|
92
|
+
if (!input.rcdpToken) {
|
|
93
|
+
throw SpacesError.usage('streamWindow requires a fresh rcdpToken; the token rotates on every rcdpd restart');
|
|
94
|
+
}
|
|
95
|
+
await this.post('/window/stream', {
|
|
96
|
+
space_id: input.spaceId,
|
|
97
|
+
window_id: input.windowId,
|
|
98
|
+
rcdp_token: input.rcdpToken,
|
|
99
|
+
app_name: input.appName ?? '',
|
|
100
|
+
title: input.title ?? '',
|
|
101
|
+
replica: input.replica ?? false,
|
|
102
|
+
});
|
|
103
|
+
}
|
|
104
|
+
/** Is the desktop app reachable right now? */
|
|
105
|
+
async available() {
|
|
106
|
+
try {
|
|
107
|
+
await readControlEndpoint(this.options.controlFile);
|
|
108
|
+
return true;
|
|
109
|
+
}
|
|
110
|
+
catch {
|
|
111
|
+
return false;
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
assertLocal(spaceId, method) {
|
|
115
|
+
if (!spaceId.startsWith('local:') && !spaceId.startsWith('space://local/')) {
|
|
116
|
+
throw SpacesError.usage(`${method} accepts a Local Space id (local:<vm>, or legacy space://local/<vm>) only — ` +
|
|
117
|
+
`the app's control server rejects other ids with HTTP 400. Stream any Space with ` +
|
|
118
|
+
`space.openStream() / space.attachStream() instead.`);
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
async post(path, body) {
|
|
122
|
+
const endpoint = await readControlEndpoint(this.options.controlFile);
|
|
123
|
+
const doFetch = this.options.fetch ?? globalThis.fetch;
|
|
124
|
+
const url = `http://127.0.0.1:${endpoint.port}${path}`;
|
|
125
|
+
let response;
|
|
126
|
+
try {
|
|
127
|
+
response = await doFetch(url, {
|
|
128
|
+
method: 'POST',
|
|
129
|
+
headers: {
|
|
130
|
+
authorization: `Bearer ${endpoint.token}`,
|
|
131
|
+
'content-type': 'application/json',
|
|
132
|
+
},
|
|
133
|
+
body: JSON.stringify(body),
|
|
134
|
+
});
|
|
135
|
+
}
|
|
136
|
+
catch (cause) {
|
|
137
|
+
throw new SpacesError('host_unavailable', `the Cua Spaces control server at 127.0.0.1:${endpoint.port} did not answer; the app may ` +
|
|
138
|
+
`have restarted, which also invalidates the token in the control file`, { cause });
|
|
139
|
+
}
|
|
140
|
+
const text = await response.text();
|
|
141
|
+
if (!response.ok) {
|
|
142
|
+
throw new SpacesError('http', `POST ${path} -> HTTP ${response.status}`, {
|
|
143
|
+
status: response.status,
|
|
144
|
+
target: path,
|
|
145
|
+
detail: excerpt(text, 200),
|
|
146
|
+
});
|
|
147
|
+
}
|
|
148
|
+
// The control server answers `{"ok":true}`. Anything else means the route
|
|
149
|
+
// did not do what we asked, and reporting success would be a lie.
|
|
150
|
+
if (!text.includes('"ok"') && !text.includes('"active"')) {
|
|
151
|
+
throw SpacesError.protocol(`POST ${path} returned HTTP 200 without an ok acknowledgement, so the effect is unproven`, { detail: excerpt(text, 200) });
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `@trycua/cua/spaces`: Cua Spaces for Node, on the native cua SDK.
|
|
3
|
+
*
|
|
4
|
+
* ```ts
|
|
5
|
+
* import { embedded } from "@trycua/cua"
|
|
6
|
+
* import { approve, startThread, SpaceSendFileOptions } from "@trycua/cua/spaces"
|
|
7
|
+
*
|
|
8
|
+
* const spaces = embedded().spaces() // or connect().spaces() (cua daemon)
|
|
9
|
+
* const info = await spaces.add("http://10.0.0.5:3211", token, "dev")
|
|
10
|
+
* const space = await spaces.space(info.id)
|
|
11
|
+
* console.log((await space.bash("uname -a", undefined)).stdout)
|
|
12
|
+
* await space.sendFile("./report.pdf", SpaceSendFileOptions.create({}))
|
|
13
|
+
* await space.teleport("firefox", undefined, approve((manifest) => ({
|
|
14
|
+
* include: undefined, acknowledgeSensitive: true, // after asking a human
|
|
15
|
+
* })))
|
|
16
|
+
* const bot = await startThread(spaces, {
|
|
17
|
+
* agent: "claude-code", prompt: "summarise the open PRs",
|
|
18
|
+
* placement: { kind: "shared", space, acknowledgeNoIsolation: true },
|
|
19
|
+
* })
|
|
20
|
+
* for await (const e of bot.events({ maxPolls: 60 })) if (e.kind === "text") console.log(e.text)
|
|
21
|
+
* ```
|
|
22
|
+
*
|
|
23
|
+
* Every call is the Rust `cua-spaces` implementation (embedded, or in
|
|
24
|
+
* `cua daemon` through `SpaceService`); `spaces.create({ on })` makes a
|
|
25
|
+
* Space locally or in the cloud. Webviews and browsers, which cannot
|
|
26
|
+
* load the native binding, use `@trycua/cua/spaces/transport` (MCP over the
|
|
27
|
+
* daemon's `/mcp` or a Tauri `invoke`) instead.
|
|
28
|
+
*/
|
|
29
|
+
import { type TeleportApprover, type TeleportDecision, type TeleportManifest } from "../native/index.js";
|
|
30
|
+
export { AgentActionReport, AgentRunStatus, AgentStartReport, PresenceCursor, PresenceIdentity, Space, SpaceCreateOptions, SpaceCreateResult, SpacePresence, SpaceSendFileOptions, SpaceStreamOptions, SpaceStreamSession, Spaces, TeleportDecision, spacesToolMethods, } from "../native/index.js";
|
|
31
|
+
export type { AudioPacket, AudioSink, FrameSink, MediaEvent, MediaSessionLike, MediaStats, PresenceEvent, PresenceMember, PresenceParticipant, SpaceBashResult, SpaceHotspotStatus, SpaceInfo, SpaceLike, SpacePresenceLike, SpaceSendFileReport, SpaceSentFile, SpaceStreamSessionLike, SpaceStreamStats, SpaceStreamTicket, SpaceToolInfo, SpaceToolResult, SpaceTransferReport, SpaceWindow, SpaceWriteReport, SpacesLike, SpacesToolMethod, TeleportApprover, TeleportItem, TeleportManifest, TeleportReceipt, VideoFrame, } from "../native/index.js";
|
|
32
|
+
export { SpacesError, isSpacesError, excerpt, toSpacesError, wrapErrors } from "./errors.js";
|
|
33
|
+
export type { SpacesErrorCode, SpacesErrorOptions } from "./errors.js";
|
|
34
|
+
export { TranscriptAdapter, detectApproval } from "./events.js";
|
|
35
|
+
export type { AgentStatus, ApprovalRequestEvent, ErrorEvent, FileEvent, ImageEvent, LinkEvent, StateEvent, TextEvent, ThreadEvent, ThreadEventKind, ThreadStatus, } from "./events.js";
|
|
36
|
+
export { AGENT_IDS, SETTLED, SpaceTurnLock, Thread, adoptThread, lockFor, startThread, validatePlacement, } from "./thread.js";
|
|
37
|
+
export type { AgentId, EventOptions, StartThreadOptions, ThreadPlacement, Turn } from "./thread.js";
|
|
38
|
+
export { CURSOR_ART, CURSOR_SHAPES, CursorSmoother, DATAGRAM_DELAY_MS, PRESENCE_FALLBACK_COLOR, PRESENCE_PALETTE, PresenceRoster, PresenceView, STREAM_DELAY_MS, cursorArt, cursorArtSvg, idleAlpha, isCursorShape, agentIdentity, presenceColor, presenceTextColor, waitForPresence, } from "./presence.js";
|
|
39
|
+
export type { CursorArtShape, CursorShapeName, PresenceDrawable, PresenceEntry } from "./presence.js";
|
|
40
|
+
export * from "./routines.js";
|
|
41
|
+
export * from "./groups.js";
|
|
42
|
+
/**
|
|
43
|
+
* A {@link TeleportApprover} from a function. The function sees exactly what
|
|
44
|
+
* would leave this machine and returns the human's decision, or `undefined`
|
|
45
|
+
* to cancel (the teleport then fails with `TeleportRefused`). It runs on a
|
|
46
|
+
* worker thread of the SDK and must be synchronous.
|
|
47
|
+
*/
|
|
48
|
+
export declare function approve(decide: (manifest: TeleportManifest) => TeleportDecision | undefined): TeleportApprover;
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `@trycua/cua/spaces`: Cua Spaces for Node, on the native cua SDK.
|
|
3
|
+
*
|
|
4
|
+
* ```ts
|
|
5
|
+
* import { embedded } from "@trycua/cua"
|
|
6
|
+
* import { approve, startThread, SpaceSendFileOptions } from "@trycua/cua/spaces"
|
|
7
|
+
*
|
|
8
|
+
* const spaces = embedded().spaces() // or connect().spaces() (cua daemon)
|
|
9
|
+
* const info = await spaces.add("http://10.0.0.5:3211", token, "dev")
|
|
10
|
+
* const space = await spaces.space(info.id)
|
|
11
|
+
* console.log((await space.bash("uname -a", undefined)).stdout)
|
|
12
|
+
* await space.sendFile("./report.pdf", SpaceSendFileOptions.create({}))
|
|
13
|
+
* await space.teleport("firefox", undefined, approve((manifest) => ({
|
|
14
|
+
* include: undefined, acknowledgeSensitive: true, // after asking a human
|
|
15
|
+
* })))
|
|
16
|
+
* const bot = await startThread(spaces, {
|
|
17
|
+
* agent: "claude-code", prompt: "summarise the open PRs",
|
|
18
|
+
* placement: { kind: "shared", space, acknowledgeNoIsolation: true },
|
|
19
|
+
* })
|
|
20
|
+
* for await (const e of bot.events({ maxPolls: 60 })) if (e.kind === "text") console.log(e.text)
|
|
21
|
+
* ```
|
|
22
|
+
*
|
|
23
|
+
* Every call is the Rust `cua-spaces` implementation (embedded, or in
|
|
24
|
+
* `cua daemon` through `SpaceService`); `spaces.create({ on })` makes a
|
|
25
|
+
* Space locally or in the cloud. Webviews and browsers, which cannot
|
|
26
|
+
* load the native binding, use `@trycua/cua/spaces/transport` (MCP over the
|
|
27
|
+
* daemon's `/mcp` or a Tauri `invoke`) instead.
|
|
28
|
+
*/
|
|
29
|
+
export { AgentActionReport, AgentRunStatus, AgentStartReport, PresenceCursor, PresenceIdentity, Space, SpaceCreateOptions, SpaceCreateResult, SpacePresence, SpaceSendFileOptions, SpaceStreamOptions, SpaceStreamSession, Spaces, TeleportDecision, spacesToolMethods, } from "../native/index.js";
|
|
30
|
+
export { SpacesError, isSpacesError, excerpt, toSpacesError, wrapErrors } from "./errors.js";
|
|
31
|
+
export { TranscriptAdapter, detectApproval } from "./events.js";
|
|
32
|
+
export { AGENT_IDS, SETTLED, SpaceTurnLock, Thread, adoptThread, lockFor, startThread, validatePlacement, } from "./thread.js";
|
|
33
|
+
export { CURSOR_ART, CURSOR_SHAPES, CursorSmoother, DATAGRAM_DELAY_MS, PRESENCE_FALLBACK_COLOR, PRESENCE_PALETTE, PresenceRoster, PresenceView, STREAM_DELAY_MS, cursorArt, cursorArtSvg, idleAlpha, isCursorShape, agentIdentity, presenceColor, presenceTextColor, waitForPresence, } from "./presence.js";
|
|
34
|
+
export * from "./routines.js";
|
|
35
|
+
export * from "./groups.js";
|
|
36
|
+
/**
|
|
37
|
+
* A {@link TeleportApprover} from a function. The function sees exactly what
|
|
38
|
+
* would leave this machine and returns the human's decision, or `undefined`
|
|
39
|
+
* to cancel (the teleport then fails with `TeleportRefused`). It runs on a
|
|
40
|
+
* worker thread of the SDK and must be synchronous.
|
|
41
|
+
*/
|
|
42
|
+
export function approve(decide) {
|
|
43
|
+
return { approve: decide };
|
|
44
|
+
}
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `@trycua/cua/spaces/pip`: picture-in-picture of a Space, for browsers and
|
|
3
|
+
* webviews. No native library, no Node imports.
|
|
4
|
+
*
|
|
5
|
+
* A PiP shows one {@link PipSource}: the whole desktop, or a single window
|
|
6
|
+
* from the Space's window list (`space.windows()`, streamed with
|
|
7
|
+
* `SpaceStreamOptions.windowId`). The stream itself is the caller's: whatever
|
|
8
|
+
* paints a canvas (WebCodecs over the media socket) is what the PiP shows.
|
|
9
|
+
*
|
|
10
|
+
* - In a browser, {@link PictureInPicture} opens the Document
|
|
11
|
+
* Picture-in-Picture API when it exists (an always-on-top window holding a
|
|
12
|
+
* live `<video>` of the canvas) and falls back to video picture-in-picture.
|
|
13
|
+
* Browsers allow one PiP window at a time, so opening another source
|
|
14
|
+
* replaces the current one.
|
|
15
|
+
* - In a desktop shell (Tauri and the like), open a small always-on-top
|
|
16
|
+
* window per source: {@link pipWindowLabel} names it, {@link encodePipRoute}
|
|
17
|
+
* tells its page what to stream and {@link parsePipRoute} reads that back.
|
|
18
|
+
*
|
|
19
|
+
* ```ts
|
|
20
|
+
* import { PictureInPicture, desktopSource, windowSource } from "@trycua/cua/spaces/pip"
|
|
21
|
+
*
|
|
22
|
+
* const pip = new PictureInPicture()
|
|
23
|
+
* button.onclick = () => pip.toggle(desktopSource, canvas, { title: "dev" })
|
|
24
|
+
* row.onclick = () => pip.open(windowSource(w), windowCanvas, { title: pipLabel(windowSource(w)) })
|
|
25
|
+
* ```
|
|
26
|
+
*/
|
|
27
|
+
/** What a PiP shows: the whole desktop or one window of the Space. */
|
|
28
|
+
export type PipSource = {
|
|
29
|
+
kind: "desktop";
|
|
30
|
+
} | {
|
|
31
|
+
kind: "window";
|
|
32
|
+
windowId: string;
|
|
33
|
+
app: string;
|
|
34
|
+
title: string;
|
|
35
|
+
};
|
|
36
|
+
export declare const desktopSource: PipSource;
|
|
37
|
+
/** A window source from a window-list row (`SpaceWindow` or the core's JSON). */
|
|
38
|
+
export declare function windowSource(w: {
|
|
39
|
+
windowId?: string;
|
|
40
|
+
window_id?: string;
|
|
41
|
+
appName?: string;
|
|
42
|
+
app_name?: string;
|
|
43
|
+
app?: string;
|
|
44
|
+
title?: string;
|
|
45
|
+
}): PipSource;
|
|
46
|
+
/** A stable key: `desktop`, or `window:<id>`. */
|
|
47
|
+
export declare function pipKey(source: PipSource): string;
|
|
48
|
+
/** One line for a title bar or a list row. */
|
|
49
|
+
export declare function pipLabel(source: PipSource, spaceName?: string): string;
|
|
50
|
+
/**
|
|
51
|
+
* A window label every shell accepts (`[A-Za-z0-9_-]`, bounded): `pip-desktop`,
|
|
52
|
+
* or `pip-w-` plus the window id with anything else replaced. Distinct ids
|
|
53
|
+
* that collide after replacement get a short hash suffix.
|
|
54
|
+
*/
|
|
55
|
+
export declare function pipWindowLabel(source: PipSource): string;
|
|
56
|
+
/** The PiP page's route: `pip=desktop`, or `pip=window&id=…&app=…&title=…`. */
|
|
57
|
+
export declare function encodePipRoute(source: PipSource): string;
|
|
58
|
+
/** Reads {@link encodePipRoute} from a hash or query (`#…`/`?…` allowed); null when absent. */
|
|
59
|
+
export declare function parsePipRoute(route: string): PipSource | null;
|
|
60
|
+
/** Fits a `width`×`height` picture into a PiP whose long edge is `longEdge`. */
|
|
61
|
+
export declare function pipSize(width: number, height: number, longEdge?: number): {
|
|
62
|
+
width: number;
|
|
63
|
+
height: number;
|
|
64
|
+
};
|
|
65
|
+
/** `document`: Document Picture-in-Picture; `video`: video PiP; `none`: neither. */
|
|
66
|
+
export type PipMode = "document" | "video" | "none";
|
|
67
|
+
interface DocumentPipApi {
|
|
68
|
+
requestWindow(options?: {
|
|
69
|
+
width?: number;
|
|
70
|
+
height?: number;
|
|
71
|
+
}): Promise<Window>;
|
|
72
|
+
window?: Window | null;
|
|
73
|
+
}
|
|
74
|
+
/** The window-like object {@link PictureInPicture} needs (a real `window` in a page). */
|
|
75
|
+
export interface PipHostWindow {
|
|
76
|
+
document: Document;
|
|
77
|
+
documentPictureInPicture?: DocumentPipApi;
|
|
78
|
+
}
|
|
79
|
+
/** Which PiP this page can open. */
|
|
80
|
+
export declare function pipSupport(win?: PipHostWindow | undefined): PipMode;
|
|
81
|
+
export interface PipOpenOptions {
|
|
82
|
+
/** The PiP's title (Document PiP only; video PiP has none). */
|
|
83
|
+
title?: string;
|
|
84
|
+
/** Initial size; defaults to the canvas fitted by {@link pipSize}. */
|
|
85
|
+
width?: number;
|
|
86
|
+
height?: number;
|
|
87
|
+
}
|
|
88
|
+
/** An open PiP. */
|
|
89
|
+
export interface PipHandle {
|
|
90
|
+
readonly mode: Exclude<PipMode, "none">;
|
|
91
|
+
readonly source: PipSource;
|
|
92
|
+
/** The Document PiP window, when that is the mode. */
|
|
93
|
+
readonly window: Window | null;
|
|
94
|
+
/** Resolves once the PiP is closed, by {@link close} or by the user. */
|
|
95
|
+
readonly closed: Promise<void>;
|
|
96
|
+
close(): void;
|
|
97
|
+
}
|
|
98
|
+
export interface PictureInPictureOptions {
|
|
99
|
+
/** The page's window (default: `globalThis`). */
|
|
100
|
+
window?: PipHostWindow;
|
|
101
|
+
/** Frames per second of the canvas capture (default 30). */
|
|
102
|
+
fps?: number;
|
|
103
|
+
}
|
|
104
|
+
/**
|
|
105
|
+
* One PiP at a time, of the desktop or of a window. The canvas keeps being
|
|
106
|
+
* painted by its stream; the PiP shows a live capture of it, so nothing moves
|
|
107
|
+
* out of the page and closing the PiP never stops the stream.
|
|
108
|
+
*/
|
|
109
|
+
export declare class PictureInPicture {
|
|
110
|
+
#private;
|
|
111
|
+
constructor(options?: PictureInPictureOptions);
|
|
112
|
+
get mode(): PipMode;
|
|
113
|
+
/** What is in the PiP now, or null. */
|
|
114
|
+
get current(): PipSource | null;
|
|
115
|
+
/** Whether `source` is what the PiP shows. */
|
|
116
|
+
isOpen(source: PipSource): boolean;
|
|
117
|
+
/** Called with the new source (or null) on every open and close. */
|
|
118
|
+
subscribe(fn: (source: PipSource | null) => void): () => void;
|
|
119
|
+
/**
|
|
120
|
+
* Opens `source` from `canvas`, replacing any open PiP. Call it from the
|
|
121
|
+
* click handler: browsers only open a PiP on a user gesture.
|
|
122
|
+
*/
|
|
123
|
+
open(source: PipSource, canvas: HTMLCanvasElement, options?: PipOpenOptions): Promise<PipHandle>;
|
|
124
|
+
/** Opens `source`, or closes it when it is what the PiP shows. */
|
|
125
|
+
toggle(source: PipSource, canvas: HTMLCanvasElement, options?: PipOpenOptions): Promise<PipHandle | null>;
|
|
126
|
+
close(): void;
|
|
127
|
+
}
|
|
128
|
+
export {};
|
|
@@ -0,0 +1,250 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `@trycua/cua/spaces/pip`: picture-in-picture of a Space, for browsers and
|
|
3
|
+
* webviews. No native library, no Node imports.
|
|
4
|
+
*
|
|
5
|
+
* A PiP shows one {@link PipSource}: the whole desktop, or a single window
|
|
6
|
+
* from the Space's window list (`space.windows()`, streamed with
|
|
7
|
+
* `SpaceStreamOptions.windowId`). The stream itself is the caller's: whatever
|
|
8
|
+
* paints a canvas (WebCodecs over the media socket) is what the PiP shows.
|
|
9
|
+
*
|
|
10
|
+
* - In a browser, {@link PictureInPicture} opens the Document
|
|
11
|
+
* Picture-in-Picture API when it exists (an always-on-top window holding a
|
|
12
|
+
* live `<video>` of the canvas) and falls back to video picture-in-picture.
|
|
13
|
+
* Browsers allow one PiP window at a time, so opening another source
|
|
14
|
+
* replaces the current one.
|
|
15
|
+
* - In a desktop shell (Tauri and the like), open a small always-on-top
|
|
16
|
+
* window per source: {@link pipWindowLabel} names it, {@link encodePipRoute}
|
|
17
|
+
* tells its page what to stream and {@link parsePipRoute} reads that back.
|
|
18
|
+
*
|
|
19
|
+
* ```ts
|
|
20
|
+
* import { PictureInPicture, desktopSource, windowSource } from "@trycua/cua/spaces/pip"
|
|
21
|
+
*
|
|
22
|
+
* const pip = new PictureInPicture()
|
|
23
|
+
* button.onclick = () => pip.toggle(desktopSource, canvas, { title: "dev" })
|
|
24
|
+
* row.onclick = () => pip.open(windowSource(w), windowCanvas, { title: pipLabel(windowSource(w)) })
|
|
25
|
+
* ```
|
|
26
|
+
*/
|
|
27
|
+
export const desktopSource = Object.freeze({ kind: "desktop" });
|
|
28
|
+
/** A window source from a window-list row (`SpaceWindow` or the core's JSON). */
|
|
29
|
+
export function windowSource(w) {
|
|
30
|
+
return {
|
|
31
|
+
kind: "window",
|
|
32
|
+
windowId: String(w.windowId ?? w.window_id ?? ""),
|
|
33
|
+
app: String(w.appName ?? w.app_name ?? w.app ?? ""),
|
|
34
|
+
title: String(w.title ?? ""),
|
|
35
|
+
};
|
|
36
|
+
}
|
|
37
|
+
/** A stable key: `desktop`, or `window:<id>`. */
|
|
38
|
+
export function pipKey(source) {
|
|
39
|
+
return source.kind === "desktop" ? "desktop" : `window:${source.windowId}`;
|
|
40
|
+
}
|
|
41
|
+
/** One line for a title bar or a list row. */
|
|
42
|
+
export function pipLabel(source, spaceName = "") {
|
|
43
|
+
if (source.kind === "desktop")
|
|
44
|
+
return spaceName || "Desktop";
|
|
45
|
+
if (!source.title)
|
|
46
|
+
return source.app || "Window";
|
|
47
|
+
if (!source.app || source.title.includes(source.app))
|
|
48
|
+
return source.title;
|
|
49
|
+
return `${source.app} · ${source.title}`;
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* A window label every shell accepts (`[A-Za-z0-9_-]`, bounded): `pip-desktop`,
|
|
53
|
+
* or `pip-w-` plus the window id with anything else replaced. Distinct ids
|
|
54
|
+
* that collide after replacement get a short hash suffix.
|
|
55
|
+
*/
|
|
56
|
+
export function pipWindowLabel(source) {
|
|
57
|
+
if (source.kind === "desktop")
|
|
58
|
+
return "pip-desktop";
|
|
59
|
+
const clean = source.windowId.replace(/[^A-Za-z0-9_-]/g, "_").slice(0, 48);
|
|
60
|
+
const suffix = clean === source.windowId ? "" : `-${hash(source.windowId)}`;
|
|
61
|
+
return `pip-w-${clean}${suffix}`;
|
|
62
|
+
}
|
|
63
|
+
function hash(s) {
|
|
64
|
+
let h = 0x811c9dc5;
|
|
65
|
+
for (let i = 0; i < s.length; i++) {
|
|
66
|
+
h ^= s.charCodeAt(i);
|
|
67
|
+
h = Math.imul(h, 0x01000193);
|
|
68
|
+
}
|
|
69
|
+
return (h >>> 0).toString(36);
|
|
70
|
+
}
|
|
71
|
+
/** The PiP page's route: `pip=desktop`, or `pip=window&id=…&app=…&title=…`. */
|
|
72
|
+
export function encodePipRoute(source) {
|
|
73
|
+
const p = new URLSearchParams();
|
|
74
|
+
if (source.kind === "desktop")
|
|
75
|
+
p.set("pip", "desktop");
|
|
76
|
+
else {
|
|
77
|
+
p.set("pip", "window");
|
|
78
|
+
p.set("id", source.windowId);
|
|
79
|
+
if (source.app)
|
|
80
|
+
p.set("app", source.app);
|
|
81
|
+
if (source.title)
|
|
82
|
+
p.set("title", source.title);
|
|
83
|
+
}
|
|
84
|
+
return p.toString();
|
|
85
|
+
}
|
|
86
|
+
/** Reads {@link encodePipRoute} from a hash or query (`#…`/`?…` allowed); null when absent. */
|
|
87
|
+
export function parsePipRoute(route) {
|
|
88
|
+
const p = new URLSearchParams(route.replace(/^[#?]/, ""));
|
|
89
|
+
switch (p.get("pip")) {
|
|
90
|
+
case "desktop":
|
|
91
|
+
return desktopSource;
|
|
92
|
+
case "window": {
|
|
93
|
+
const id = p.get("id");
|
|
94
|
+
return id ? { kind: "window", windowId: id, app: p.get("app") ?? "", title: p.get("title") ?? "" } : null;
|
|
95
|
+
}
|
|
96
|
+
default:
|
|
97
|
+
return null;
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
/** Fits a `width`×`height` picture into a PiP whose long edge is `longEdge`. */
|
|
101
|
+
export function pipSize(width, height, longEdge = 480) {
|
|
102
|
+
const w = width > 0 ? width : 16;
|
|
103
|
+
const h = height > 0 ? height : 9;
|
|
104
|
+
const scale = longEdge / Math.max(w, h);
|
|
105
|
+
return { width: Math.max(1, Math.round(w * scale)), height: Math.max(1, Math.round(h * scale)) };
|
|
106
|
+
}
|
|
107
|
+
/** Which PiP this page can open. */
|
|
108
|
+
export function pipSupport(win = globalThis) {
|
|
109
|
+
if (!win?.document)
|
|
110
|
+
return "none";
|
|
111
|
+
if (win.documentPictureInPicture && typeof win.documentPictureInPicture.requestWindow === "function")
|
|
112
|
+
return "document";
|
|
113
|
+
return win.document.pictureInPictureEnabled ? "video" : "none";
|
|
114
|
+
}
|
|
115
|
+
/**
|
|
116
|
+
* One PiP at a time, of the desktop or of a window. The canvas keeps being
|
|
117
|
+
* painted by its stream; the PiP shows a live capture of it, so nothing moves
|
|
118
|
+
* out of the page and closing the PiP never stops the stream.
|
|
119
|
+
*/
|
|
120
|
+
export class PictureInPicture {
|
|
121
|
+
#win;
|
|
122
|
+
#fps;
|
|
123
|
+
#current = null;
|
|
124
|
+
#listeners = new Set();
|
|
125
|
+
constructor(options = {}) {
|
|
126
|
+
this.#win = options.window ?? globalThis;
|
|
127
|
+
this.#fps = options.fps ?? 30;
|
|
128
|
+
}
|
|
129
|
+
get mode() {
|
|
130
|
+
return pipSupport(this.#win);
|
|
131
|
+
}
|
|
132
|
+
/** What is in the PiP now, or null. */
|
|
133
|
+
get current() {
|
|
134
|
+
return this.#current?.source ?? null;
|
|
135
|
+
}
|
|
136
|
+
/** Whether `source` is what the PiP shows. */
|
|
137
|
+
isOpen(source) {
|
|
138
|
+
return this.#current !== null && pipKey(this.#current.source) === pipKey(source);
|
|
139
|
+
}
|
|
140
|
+
/** Called with the new source (or null) on every open and close. */
|
|
141
|
+
subscribe(fn) {
|
|
142
|
+
this.#listeners.add(fn);
|
|
143
|
+
return () => this.#listeners.delete(fn);
|
|
144
|
+
}
|
|
145
|
+
/**
|
|
146
|
+
* Opens `source` from `canvas`, replacing any open PiP. Call it from the
|
|
147
|
+
* click handler: browsers only open a PiP on a user gesture.
|
|
148
|
+
*/
|
|
149
|
+
async open(source, canvas, options = {}) {
|
|
150
|
+
const win = this.#win;
|
|
151
|
+
const mode = pipSupport(win);
|
|
152
|
+
if (!win || mode === "none")
|
|
153
|
+
throw new Error("this browser has no picture-in-picture");
|
|
154
|
+
const size = options.width && options.height ? { width: options.width, height: options.height } : pipSize(canvas.width, canvas.height);
|
|
155
|
+
// The PiP being replaced closes quietly: listeners see the new source,
|
|
156
|
+
// not a null in between.
|
|
157
|
+
const previous = this.#current;
|
|
158
|
+
this.#current = null;
|
|
159
|
+
let pipWindow = null;
|
|
160
|
+
try {
|
|
161
|
+
// Document PiP first, before any other await: it needs the gesture.
|
|
162
|
+
if (mode === "document")
|
|
163
|
+
pipWindow = await win.documentPictureInPicture.requestWindow(size);
|
|
164
|
+
}
|
|
165
|
+
catch (e) {
|
|
166
|
+
this.#current = previous;
|
|
167
|
+
throw e;
|
|
168
|
+
}
|
|
169
|
+
previous?.close();
|
|
170
|
+
const stream = canvas.captureStream(this.#fps);
|
|
171
|
+
const doc = pipWindow?.document ?? win.document;
|
|
172
|
+
const video = doc.createElement("video");
|
|
173
|
+
video.muted = true;
|
|
174
|
+
video.autoplay = true;
|
|
175
|
+
video.playsInline = true;
|
|
176
|
+
video.srcObject = stream;
|
|
177
|
+
let resolveClosed = () => { };
|
|
178
|
+
const closed = new Promise((r) => (resolveClosed = r));
|
|
179
|
+
let done = false;
|
|
180
|
+
const finish = () => {
|
|
181
|
+
if (done)
|
|
182
|
+
return;
|
|
183
|
+
done = true;
|
|
184
|
+
for (const t of stream.getTracks())
|
|
185
|
+
t.stop();
|
|
186
|
+
video.srcObject = null;
|
|
187
|
+
if (this.#current === handle) {
|
|
188
|
+
this.#current = null;
|
|
189
|
+
this.#emit(null);
|
|
190
|
+
}
|
|
191
|
+
resolveClosed();
|
|
192
|
+
};
|
|
193
|
+
const handle = {
|
|
194
|
+
mode: mode,
|
|
195
|
+
source,
|
|
196
|
+
window: pipWindow,
|
|
197
|
+
closed,
|
|
198
|
+
close: () => {
|
|
199
|
+
if (done)
|
|
200
|
+
return;
|
|
201
|
+
if (pipWindow)
|
|
202
|
+
pipWindow.close();
|
|
203
|
+
else if (win.document.pictureInPictureElement === video)
|
|
204
|
+
void win.document.exitPictureInPicture?.().catch(() => { });
|
|
205
|
+
finish();
|
|
206
|
+
},
|
|
207
|
+
};
|
|
208
|
+
if (pipWindow) {
|
|
209
|
+
pipWindow.document.title = options.title ?? pipLabel(source);
|
|
210
|
+
const style = pipWindow.document.createElement("style");
|
|
211
|
+
style.textContent =
|
|
212
|
+
"html,body{margin:0;height:100%;background:#000;overflow:hidden}video{display:block;width:100%;height:100%;object-fit:contain}";
|
|
213
|
+
pipWindow.document.head.append(style);
|
|
214
|
+
pipWindow.document.body.append(video);
|
|
215
|
+
pipWindow.addEventListener("pagehide", finish, { once: true });
|
|
216
|
+
await video.play().catch(() => { });
|
|
217
|
+
}
|
|
218
|
+
else {
|
|
219
|
+
video.addEventListener("leavepictureinpicture", finish, { once: true });
|
|
220
|
+
try {
|
|
221
|
+
await video.play();
|
|
222
|
+
await video.requestPictureInPicture();
|
|
223
|
+
}
|
|
224
|
+
catch (e) {
|
|
225
|
+
finish();
|
|
226
|
+
if (previous)
|
|
227
|
+
this.#emit(null);
|
|
228
|
+
throw e;
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
this.#current = handle;
|
|
232
|
+
this.#emit(source);
|
|
233
|
+
return handle;
|
|
234
|
+
}
|
|
235
|
+
/** Opens `source`, or closes it when it is what the PiP shows. */
|
|
236
|
+
async toggle(source, canvas, options = {}) {
|
|
237
|
+
if (this.isOpen(source)) {
|
|
238
|
+
this.close();
|
|
239
|
+
return null;
|
|
240
|
+
}
|
|
241
|
+
return this.open(source, canvas, options);
|
|
242
|
+
}
|
|
243
|
+
close() {
|
|
244
|
+
this.#current?.close();
|
|
245
|
+
}
|
|
246
|
+
#emit(source) {
|
|
247
|
+
for (const fn of this.#listeners)
|
|
248
|
+
fn(source);
|
|
249
|
+
}
|
|
250
|
+
}
|