@openchamber/sdk 1.24.2 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/API.md +7 -3
- package/DOCUMENTATION.md +2 -0
- package/GUEST_SERVICES.md +84 -1
- package/README.md +1 -1
- package/dist/index.d.ts +6 -2
- package/dist/index.js +3 -1
- package/dist/manifest.d.ts +36 -0
- package/dist/manifest.js +20 -0
- package/dist/parse.d.ts +22 -0
- package/dist/parse.js +27 -3
- package/dist/schemas.d.ts +3 -0
- package/dist/schemas.js +2 -0
- package/dist/service-provider-schemas.d.ts +25 -0
- package/dist/service-provider-schemas.js +12 -0
- package/dist/service-providers.d.ts +218 -0
- package/dist/service-providers.js +73 -0
- package/dist/service-surface-schemas.d.ts +173 -0
- package/dist/service-surface-schemas.js +59 -0
- package/dist/service-surface.d.ts +116 -0
- package/dist/service-surface.js +154 -0
- package/dist/workspace.d.ts +1 -0
- package/package.json +1 -1
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Contracts a service fulfils when it stands in for a host role
|
|
3
|
+
* (`contributes.service.provides`).
|
|
4
|
+
*
|
|
5
|
+
* Browser: the host's `openchamber_web` tool sends its `browser.*` actions to
|
|
6
|
+
* the service instead of the in-app browser view. The service answers with the
|
|
7
|
+
* same shapes the in-app view answers with, so the agent sees one tool whoever
|
|
8
|
+
* drives the page. The host starts the service on the first action, needs no
|
|
9
|
+
* panel, and stops it after `BROWSER_PROVIDER_IDLE_MS` without actions.
|
|
10
|
+
*/
|
|
11
|
+
/** Route on the service loopback the host posts every action to. */
|
|
12
|
+
export declare const BROWSER_PROVIDER_PATH = "/browser-control";
|
|
13
|
+
export declare const BROWSER_CONTROL_ACTIONS: readonly ["browser.open", "browser.snapshot", "browser.click", "browser.type", "browser.scroll", "browser.back", "browser.forward", "browser.inspect", "browser.capture", "browser.resize"];
|
|
14
|
+
export type BrowserControlAction = (typeof BROWSER_CONTROL_ACTIONS)[number];
|
|
15
|
+
export declare const BROWSER_VIEWPORT_MODES: readonly ["mobile", "tablet", "desktop", "fill"];
|
|
16
|
+
export type BrowserViewportMode = (typeof BROWSER_VIEWPORT_MODES)[number];
|
|
17
|
+
export declare const BROWSER_SCROLL_DIRECTIONS: readonly ["up", "down", "top", "bottom"];
|
|
18
|
+
export type BrowserScrollDirection = (typeof BROWSER_SCROLL_DIRECTIONS)[number];
|
|
19
|
+
/** `browser.open` may wait for a slow page; every other action is quick. */
|
|
20
|
+
export declare const BROWSER_PROVIDER_OPEN_TIMEOUT_MS = 45000;
|
|
21
|
+
export declare const BROWSER_PROVIDER_ACTION_TIMEOUT_MS = 20000;
|
|
22
|
+
/** A screenshot is the largest answer; snapshots are capped by the provider itself. */
|
|
23
|
+
export declare const BROWSER_PROVIDER_RESPONSE_MAX = 12000000;
|
|
24
|
+
/** No action for this long stops the service; the next action starts it again. */
|
|
25
|
+
export declare const BROWSER_PROVIDER_IDLE_MS: number;
|
|
26
|
+
export type BrowserOpenParameters = {
|
|
27
|
+
url: string;
|
|
28
|
+
viewport?: BrowserViewportMode;
|
|
29
|
+
};
|
|
30
|
+
export type BrowserSnapshotParameters = {
|
|
31
|
+
selector?: string;
|
|
32
|
+
};
|
|
33
|
+
export type BrowserClickParameters = {
|
|
34
|
+
selector?: string;
|
|
35
|
+
text?: string;
|
|
36
|
+
};
|
|
37
|
+
export type BrowserTypeParameters = {
|
|
38
|
+
selector: string;
|
|
39
|
+
value: string;
|
|
40
|
+
submit: boolean;
|
|
41
|
+
};
|
|
42
|
+
export type BrowserScrollParameters = {
|
|
43
|
+
selector?: string;
|
|
44
|
+
direction?: BrowserScrollDirection;
|
|
45
|
+
};
|
|
46
|
+
export type BrowserInspectParameters = {
|
|
47
|
+
selector: string;
|
|
48
|
+
};
|
|
49
|
+
export type BrowserCaptureParameters = {
|
|
50
|
+
label?: string;
|
|
51
|
+
};
|
|
52
|
+
export type BrowserResizeParameters = {
|
|
53
|
+
viewport: BrowserViewportMode;
|
|
54
|
+
};
|
|
55
|
+
/**
|
|
56
|
+
* Where an action came from: the project the agent works in and the chat it
|
|
57
|
+
* runs in. Filled by the host from the tool call, never typed by the model.
|
|
58
|
+
* A provider that keeps one browser per project or chat keys on these; one
|
|
59
|
+
* that keeps a single browser ignores them. Either is `null` when the host
|
|
60
|
+
* had none (a call from the CLI, for example): treat that as "unknown", not
|
|
61
|
+
* as a scope of its own.
|
|
62
|
+
*/
|
|
63
|
+
export type BrowserProviderContext = {
|
|
64
|
+
directory: string | null;
|
|
65
|
+
sessionId: string | null;
|
|
66
|
+
};
|
|
67
|
+
type ProviderRequestEnvelope = {
|
|
68
|
+
requestId: string;
|
|
69
|
+
context: BrowserProviderContext;
|
|
70
|
+
};
|
|
71
|
+
/** One action, as the host posts it. Parameters were validated by the host. */
|
|
72
|
+
export type BrowserProviderRequest = ProviderRequestEnvelope & ({
|
|
73
|
+
action: 'browser.open';
|
|
74
|
+
parameters: BrowserOpenParameters;
|
|
75
|
+
} | {
|
|
76
|
+
action: 'browser.snapshot';
|
|
77
|
+
parameters: BrowserSnapshotParameters;
|
|
78
|
+
} | {
|
|
79
|
+
action: 'browser.click';
|
|
80
|
+
parameters: BrowserClickParameters;
|
|
81
|
+
} | {
|
|
82
|
+
action: 'browser.type';
|
|
83
|
+
parameters: BrowserTypeParameters;
|
|
84
|
+
} | {
|
|
85
|
+
action: 'browser.scroll';
|
|
86
|
+
parameters: BrowserScrollParameters;
|
|
87
|
+
} | {
|
|
88
|
+
action: 'browser.back';
|
|
89
|
+
parameters: Record<never, never>;
|
|
90
|
+
} | {
|
|
91
|
+
action: 'browser.forward';
|
|
92
|
+
parameters: Record<never, never>;
|
|
93
|
+
} | {
|
|
94
|
+
action: 'browser.inspect';
|
|
95
|
+
parameters: BrowserInspectParameters;
|
|
96
|
+
} | {
|
|
97
|
+
action: 'browser.capture';
|
|
98
|
+
parameters: BrowserCaptureParameters;
|
|
99
|
+
} | {
|
|
100
|
+
action: 'browser.resize';
|
|
101
|
+
parameters: BrowserResizeParameters;
|
|
102
|
+
});
|
|
103
|
+
export type BrowserViewportSummary = {
|
|
104
|
+
mode: BrowserViewportMode | 'custom';
|
|
105
|
+
width: number | null;
|
|
106
|
+
height: number | null;
|
|
107
|
+
};
|
|
108
|
+
export type BrowserElementBounds = {
|
|
109
|
+
x: number;
|
|
110
|
+
y: number;
|
|
111
|
+
width: number;
|
|
112
|
+
height: number;
|
|
113
|
+
};
|
|
114
|
+
/** One interactive element of a snapshot; absent fields mean empty or default. */
|
|
115
|
+
export type BrowserSnapshotElement = {
|
|
116
|
+
selector: string;
|
|
117
|
+
tag: string;
|
|
118
|
+
bounds: BrowserElementBounds;
|
|
119
|
+
inViewport?: true;
|
|
120
|
+
type?: string;
|
|
121
|
+
role?: string;
|
|
122
|
+
label?: string;
|
|
123
|
+
disabled?: true;
|
|
124
|
+
missingAccessibleName?: true;
|
|
125
|
+
};
|
|
126
|
+
export type BrowserOpenData = {
|
|
127
|
+
url: string;
|
|
128
|
+
title: string;
|
|
129
|
+
opened: true;
|
|
130
|
+
/** `false` means the page is still loading, not that opening failed. */
|
|
131
|
+
settled: boolean;
|
|
132
|
+
viewport: BrowserViewportSummary;
|
|
133
|
+
};
|
|
134
|
+
export type BrowserSnapshotData = {
|
|
135
|
+
url: string;
|
|
136
|
+
title: string;
|
|
137
|
+
scope: string;
|
|
138
|
+
scrollY: number;
|
|
139
|
+
maxScrollY: number;
|
|
140
|
+
text: string;
|
|
141
|
+
elements: BrowserSnapshotElement[];
|
|
142
|
+
textTruncated?: true;
|
|
143
|
+
textTotalChars?: number;
|
|
144
|
+
elementsTruncated?: true;
|
|
145
|
+
interactiveElementsOnPage?: number;
|
|
146
|
+
viewport: BrowserViewportSummary;
|
|
147
|
+
/** Warnings and errors the page logged since it was opened; absent when none. */
|
|
148
|
+
consoleProblems?: BrowserConsoleProblem[];
|
|
149
|
+
};
|
|
150
|
+
export type BrowserConsoleProblem = {
|
|
151
|
+
level: 'warning' | 'error';
|
|
152
|
+
message: string;
|
|
153
|
+
source: string;
|
|
154
|
+
};
|
|
155
|
+
export type BrowserClickData = {
|
|
156
|
+
clicked: string;
|
|
157
|
+
label: string;
|
|
158
|
+
url: string;
|
|
159
|
+
};
|
|
160
|
+
export type BrowserTypeData = {
|
|
161
|
+
selector: string;
|
|
162
|
+
url: string;
|
|
163
|
+
};
|
|
164
|
+
export type BrowserScrollData = {
|
|
165
|
+
scrollY: number;
|
|
166
|
+
maxScrollY: number;
|
|
167
|
+
atTop: boolean;
|
|
168
|
+
atBottom: boolean;
|
|
169
|
+
scrolledTo?: string;
|
|
170
|
+
direction?: BrowserScrollDirection;
|
|
171
|
+
};
|
|
172
|
+
export type BrowserNavigationData = {
|
|
173
|
+
url: string;
|
|
174
|
+
title: string;
|
|
175
|
+
};
|
|
176
|
+
export type BrowserInspectData = {
|
|
177
|
+
selector: string;
|
|
178
|
+
tag: string;
|
|
179
|
+
label: string;
|
|
180
|
+
bounds: BrowserElementBounds;
|
|
181
|
+
inViewport: boolean;
|
|
182
|
+
styles: Record<string, string>;
|
|
183
|
+
};
|
|
184
|
+
/** The host writes the image to the project and hands the agent its path. */
|
|
185
|
+
export type BrowserCaptureData = {
|
|
186
|
+
base64: string;
|
|
187
|
+
mime: string;
|
|
188
|
+
width: number;
|
|
189
|
+
height: number;
|
|
190
|
+
url: string;
|
|
191
|
+
title: string;
|
|
192
|
+
viewport: BrowserViewportSummary;
|
|
193
|
+
};
|
|
194
|
+
export type BrowserResizeData = {
|
|
195
|
+
viewport: BrowserViewportSummary;
|
|
196
|
+
};
|
|
197
|
+
export type BrowserProviderData = BrowserOpenData | BrowserSnapshotData | BrowserClickData | BrowserTypeData | BrowserScrollData | BrowserNavigationData | BrowserInspectData | BrowserCaptureData | BrowserResizeData;
|
|
198
|
+
/**
|
|
199
|
+
* What the service answers. `error` is read by the agent, so it should say
|
|
200
|
+
* what to do differently ("No element matches #save"), not what broke inside.
|
|
201
|
+
*/
|
|
202
|
+
export type BrowserProviderResult = {
|
|
203
|
+
ok: true;
|
|
204
|
+
data: BrowserProviderData;
|
|
205
|
+
} | {
|
|
206
|
+
ok: false;
|
|
207
|
+
error: string;
|
|
208
|
+
};
|
|
209
|
+
export declare const isBrowserControlAction: (value: string) => value is BrowserControlAction;
|
|
210
|
+
/**
|
|
211
|
+
* A provider's read of the body the host posted to `BROWSER_PROVIDER_PATH`.
|
|
212
|
+
* The host is trusted and validated the parameters against the action before
|
|
213
|
+
* sending, so this checks the envelope and the action, not every field, and
|
|
214
|
+
* the service bundle needs no schema library for it. `null` is a body that is
|
|
215
|
+
* not a browser action; answer it with HTTP 400.
|
|
216
|
+
*/
|
|
217
|
+
export declare const readBrowserProviderRequest: (body: string) => BrowserProviderRequest | null;
|
|
218
|
+
export {};
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Contracts a service fulfils when it stands in for a host role
|
|
3
|
+
* (`contributes.service.provides`).
|
|
4
|
+
*
|
|
5
|
+
* Browser: the host's `openchamber_web` tool sends its `browser.*` actions to
|
|
6
|
+
* the service instead of the in-app browser view. The service answers with the
|
|
7
|
+
* same shapes the in-app view answers with, so the agent sees one tool whoever
|
|
8
|
+
* drives the page. The host starts the service on the first action, needs no
|
|
9
|
+
* panel, and stops it after `BROWSER_PROVIDER_IDLE_MS` without actions.
|
|
10
|
+
*/
|
|
11
|
+
/** Route on the service loopback the host posts every action to. */
|
|
12
|
+
export const BROWSER_PROVIDER_PATH = '/browser-control';
|
|
13
|
+
export const BROWSER_CONTROL_ACTIONS = [
|
|
14
|
+
'browser.open',
|
|
15
|
+
'browser.snapshot',
|
|
16
|
+
'browser.click',
|
|
17
|
+
'browser.type',
|
|
18
|
+
'browser.scroll',
|
|
19
|
+
'browser.back',
|
|
20
|
+
'browser.forward',
|
|
21
|
+
'browser.inspect',
|
|
22
|
+
'browser.capture',
|
|
23
|
+
'browser.resize',
|
|
24
|
+
];
|
|
25
|
+
export const BROWSER_VIEWPORT_MODES = ['mobile', 'tablet', 'desktop', 'fill'];
|
|
26
|
+
export const BROWSER_SCROLL_DIRECTIONS = ['up', 'down', 'top', 'bottom'];
|
|
27
|
+
/** `browser.open` may wait for a slow page; every other action is quick. */
|
|
28
|
+
export const BROWSER_PROVIDER_OPEN_TIMEOUT_MS = 45_000;
|
|
29
|
+
export const BROWSER_PROVIDER_ACTION_TIMEOUT_MS = 20_000;
|
|
30
|
+
/** A screenshot is the largest answer; snapshots are capped by the provider itself. */
|
|
31
|
+
export const BROWSER_PROVIDER_RESPONSE_MAX = 12_000_000;
|
|
32
|
+
/** No action for this long stops the service; the next action starts it again. */
|
|
33
|
+
export const BROWSER_PROVIDER_IDLE_MS = 10 * 60_000;
|
|
34
|
+
const CONTROL_ACTIONS = new Set(BROWSER_CONTROL_ACTIONS);
|
|
35
|
+
export const isBrowserControlAction = (value) => CONTROL_ACTIONS.has(value);
|
|
36
|
+
/** Hosts before the context field posted none; that reads as an unknown scope. */
|
|
37
|
+
const readContext = (wire) => {
|
|
38
|
+
const directory = wire?.directory;
|
|
39
|
+
const sessionId = wire?.sessionId;
|
|
40
|
+
return {
|
|
41
|
+
directory: String(directory) === directory && directory.length > 0 ? directory : null,
|
|
42
|
+
sessionId: String(sessionId) === sessionId && sessionId.length > 0 ? sessionId : null,
|
|
43
|
+
};
|
|
44
|
+
};
|
|
45
|
+
/**
|
|
46
|
+
* A provider's read of the body the host posted to `BROWSER_PROVIDER_PATH`.
|
|
47
|
+
* The host is trusted and validated the parameters against the action before
|
|
48
|
+
* sending, so this checks the envelope and the action, not every field, and
|
|
49
|
+
* the service bundle needs no schema library for it. `null` is a body that is
|
|
50
|
+
* not a browser action; answer it with HTTP 400.
|
|
51
|
+
*/
|
|
52
|
+
export const readBrowserProviderRequest = (body) => {
|
|
53
|
+
let wire;
|
|
54
|
+
try {
|
|
55
|
+
const parsed = JSON.parse(body);
|
|
56
|
+
if (Object(parsed) !== parsed || parsed === null)
|
|
57
|
+
return null;
|
|
58
|
+
wire = parsed;
|
|
59
|
+
}
|
|
60
|
+
catch {
|
|
61
|
+
return null;
|
|
62
|
+
}
|
|
63
|
+
const { requestId, action, parameters, context } = wire;
|
|
64
|
+
if (String(requestId) !== requestId || requestId.length === 0)
|
|
65
|
+
return null;
|
|
66
|
+
if (String(action) !== action || !isBrowserControlAction(action))
|
|
67
|
+
return null;
|
|
68
|
+
if (Object(parameters) !== parameters)
|
|
69
|
+
return null;
|
|
70
|
+
// SAFETY: requestId, action, and the parameters object were checked above,
|
|
71
|
+
// and the host validated the parameters for this action before posting.
|
|
72
|
+
return { requestId, action, parameters, context: readContext(context) };
|
|
73
|
+
};
|
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
export declare const surfaceInputEventSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
3
|
+
type: z.ZodLiteral<"pointer">;
|
|
4
|
+
action: z.ZodEnum<{
|
|
5
|
+
up: "up";
|
|
6
|
+
down: "down";
|
|
7
|
+
move: "move";
|
|
8
|
+
}>;
|
|
9
|
+
x: z.ZodNumber;
|
|
10
|
+
y: z.ZodNumber;
|
|
11
|
+
button: z.ZodNumber;
|
|
12
|
+
buttons: z.ZodNumber;
|
|
13
|
+
modifiers: z.ZodObject<{
|
|
14
|
+
alt: z.ZodBoolean;
|
|
15
|
+
ctrl: z.ZodBoolean;
|
|
16
|
+
meta: z.ZodBoolean;
|
|
17
|
+
shift: z.ZodBoolean;
|
|
18
|
+
}, z.core.$strip>;
|
|
19
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
20
|
+
type: z.ZodLiteral<"wheel">;
|
|
21
|
+
x: z.ZodNumber;
|
|
22
|
+
y: z.ZodNumber;
|
|
23
|
+
deltaX: z.ZodNumber;
|
|
24
|
+
deltaY: z.ZodNumber;
|
|
25
|
+
modifiers: z.ZodObject<{
|
|
26
|
+
alt: z.ZodBoolean;
|
|
27
|
+
ctrl: z.ZodBoolean;
|
|
28
|
+
meta: z.ZodBoolean;
|
|
29
|
+
shift: z.ZodBoolean;
|
|
30
|
+
}, z.core.$strip>;
|
|
31
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
32
|
+
type: z.ZodLiteral<"key">;
|
|
33
|
+
action: z.ZodEnum<{
|
|
34
|
+
up: "up";
|
|
35
|
+
down: "down";
|
|
36
|
+
}>;
|
|
37
|
+
key: z.ZodString;
|
|
38
|
+
code: z.ZodString;
|
|
39
|
+
modifiers: z.ZodObject<{
|
|
40
|
+
alt: z.ZodBoolean;
|
|
41
|
+
ctrl: z.ZodBoolean;
|
|
42
|
+
meta: z.ZodBoolean;
|
|
43
|
+
shift: z.ZodBoolean;
|
|
44
|
+
}, z.core.$strip>;
|
|
45
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
46
|
+
type: z.ZodLiteral<"text">;
|
|
47
|
+
text: z.ZodString;
|
|
48
|
+
}, z.core.$strip>], "type">;
|
|
49
|
+
/** What a viewer (the host panel) sends the host over the surface socket. */
|
|
50
|
+
export declare const surfaceViewerMessageSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
51
|
+
type: z.ZodLiteral<"ack">;
|
|
52
|
+
seq: z.ZodNumber;
|
|
53
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
54
|
+
type: z.ZodLiteral<"input">;
|
|
55
|
+
events: z.ZodArray<z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
56
|
+
type: z.ZodLiteral<"pointer">;
|
|
57
|
+
action: z.ZodEnum<{
|
|
58
|
+
up: "up";
|
|
59
|
+
down: "down";
|
|
60
|
+
move: "move";
|
|
61
|
+
}>;
|
|
62
|
+
x: z.ZodNumber;
|
|
63
|
+
y: z.ZodNumber;
|
|
64
|
+
button: z.ZodNumber;
|
|
65
|
+
buttons: z.ZodNumber;
|
|
66
|
+
modifiers: z.ZodObject<{
|
|
67
|
+
alt: z.ZodBoolean;
|
|
68
|
+
ctrl: z.ZodBoolean;
|
|
69
|
+
meta: z.ZodBoolean;
|
|
70
|
+
shift: z.ZodBoolean;
|
|
71
|
+
}, z.core.$strip>;
|
|
72
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
73
|
+
type: z.ZodLiteral<"wheel">;
|
|
74
|
+
x: z.ZodNumber;
|
|
75
|
+
y: z.ZodNumber;
|
|
76
|
+
deltaX: z.ZodNumber;
|
|
77
|
+
deltaY: z.ZodNumber;
|
|
78
|
+
modifiers: z.ZodObject<{
|
|
79
|
+
alt: z.ZodBoolean;
|
|
80
|
+
ctrl: z.ZodBoolean;
|
|
81
|
+
meta: z.ZodBoolean;
|
|
82
|
+
shift: z.ZodBoolean;
|
|
83
|
+
}, z.core.$strip>;
|
|
84
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
85
|
+
type: z.ZodLiteral<"key">;
|
|
86
|
+
action: z.ZodEnum<{
|
|
87
|
+
up: "up";
|
|
88
|
+
down: "down";
|
|
89
|
+
}>;
|
|
90
|
+
key: z.ZodString;
|
|
91
|
+
code: z.ZodString;
|
|
92
|
+
modifiers: z.ZodObject<{
|
|
93
|
+
alt: z.ZodBoolean;
|
|
94
|
+
ctrl: z.ZodBoolean;
|
|
95
|
+
meta: z.ZodBoolean;
|
|
96
|
+
shift: z.ZodBoolean;
|
|
97
|
+
}, z.core.$strip>;
|
|
98
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
99
|
+
type: z.ZodLiteral<"text">;
|
|
100
|
+
text: z.ZodString;
|
|
101
|
+
}, z.core.$strip>], "type">>;
|
|
102
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
103
|
+
type: z.ZodLiteral<"release">;
|
|
104
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
105
|
+
type: z.ZodLiteral<"resize">;
|
|
106
|
+
width: z.ZodNumber;
|
|
107
|
+
height: z.ZodNumber;
|
|
108
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
109
|
+
type: z.ZodLiteral<"clipboard-read">;
|
|
110
|
+
id: z.ZodString;
|
|
111
|
+
}, z.core.$strip>], "type">;
|
|
112
|
+
export type SurfaceViewerMessage = z.infer<typeof surfaceViewerMessageSchema>;
|
|
113
|
+
export declare const surfaceControllerSchema: z.ZodEnum<{
|
|
114
|
+
user: "user";
|
|
115
|
+
none: "none";
|
|
116
|
+
agent: "agent";
|
|
117
|
+
}>;
|
|
118
|
+
/**
|
|
119
|
+
* What the host sends a viewer. A `frame` text message is followed by one
|
|
120
|
+
* binary message carrying exactly `bytes` bytes of the image.
|
|
121
|
+
*/
|
|
122
|
+
export declare const surfaceHostMessageSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
123
|
+
type: z.ZodLiteral<"hello">;
|
|
124
|
+
viewerId: z.ZodString;
|
|
125
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
126
|
+
type: z.ZodLiteral<"frame">;
|
|
127
|
+
seq: z.ZodNumber;
|
|
128
|
+
width: z.ZodNumber;
|
|
129
|
+
height: z.ZodNumber;
|
|
130
|
+
mime: z.ZodEnum<{
|
|
131
|
+
"image/jpeg": "image/jpeg";
|
|
132
|
+
"image/png": "image/png";
|
|
133
|
+
}>;
|
|
134
|
+
bytes: z.ZodNumber;
|
|
135
|
+
title: z.ZodOptional<z.ZodString>;
|
|
136
|
+
agentActive: z.ZodBoolean;
|
|
137
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
138
|
+
type: z.ZodLiteral<"control">;
|
|
139
|
+
controller: z.ZodEnum<{
|
|
140
|
+
user: "user";
|
|
141
|
+
none: "none";
|
|
142
|
+
agent: "agent";
|
|
143
|
+
}>;
|
|
144
|
+
mine: z.ZodBoolean;
|
|
145
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
146
|
+
type: z.ZodLiteral<"resized">;
|
|
147
|
+
width: z.ZodNumber;
|
|
148
|
+
height: z.ZodNumber;
|
|
149
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
150
|
+
type: z.ZodLiteral<"clipboard">;
|
|
151
|
+
id: z.ZodString;
|
|
152
|
+
text: z.ZodString;
|
|
153
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
154
|
+
type: z.ZodLiteral<"error">;
|
|
155
|
+
code: z.ZodString;
|
|
156
|
+
message: z.ZodString;
|
|
157
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
158
|
+
type: z.ZodLiteral<"ended">;
|
|
159
|
+
reason: z.ZodEnum<{
|
|
160
|
+
"service-stopped": "service-stopped";
|
|
161
|
+
"extension-unavailable": "extension-unavailable";
|
|
162
|
+
"host-shutdown": "host-shutdown";
|
|
163
|
+
}>;
|
|
164
|
+
}, z.core.$strip>], "type">;
|
|
165
|
+
export type SurfaceHostMessage = z.infer<typeof surfaceHostMessageSchema>;
|
|
166
|
+
/** Service answers the host parses. */
|
|
167
|
+
export declare const surfaceResizeAnswerSchema: z.ZodObject<{
|
|
168
|
+
width: z.ZodNumber;
|
|
169
|
+
height: z.ZodNumber;
|
|
170
|
+
}, z.core.$strip>;
|
|
171
|
+
export declare const surfaceClipboardAnswerSchema: z.ZodObject<{
|
|
172
|
+
text: z.ZodString;
|
|
173
|
+
}, z.core.$strip>;
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
import { SURFACE_CONTROLLERS, SURFACE_DIMENSION_MAX, SURFACE_FRAME_MIMES, SURFACE_INPUT_BATCH_MAX, SURFACE_TEXT_MAX, SURFACE_TITLE_MAX, } from "./service-surface.js";
|
|
3
|
+
const modifiersSchema = z.object({ alt: z.boolean(), ctrl: z.boolean(), meta: z.boolean(), shift: z.boolean() });
|
|
4
|
+
const finite = z.number().finite();
|
|
5
|
+
export const surfaceInputEventSchema = z.discriminatedUnion('type', [
|
|
6
|
+
z.object({
|
|
7
|
+
type: z.literal('pointer'),
|
|
8
|
+
action: z.enum(['down', 'up', 'move']),
|
|
9
|
+
x: finite,
|
|
10
|
+
y: finite,
|
|
11
|
+
button: z.number().int().min(-1).max(4),
|
|
12
|
+
buttons: z.number().int().min(0).max(31),
|
|
13
|
+
modifiers: modifiersSchema,
|
|
14
|
+
}),
|
|
15
|
+
z.object({ type: z.literal('wheel'), x: finite, y: finite, deltaX: finite, deltaY: finite, modifiers: modifiersSchema }),
|
|
16
|
+
z.object({
|
|
17
|
+
type: z.literal('key'),
|
|
18
|
+
action: z.enum(['down', 'up']),
|
|
19
|
+
key: z.string().min(1).max(64),
|
|
20
|
+
code: z.string().max(64),
|
|
21
|
+
modifiers: modifiersSchema,
|
|
22
|
+
}),
|
|
23
|
+
z.object({ type: z.literal('text'), text: z.string().min(1).max(SURFACE_TEXT_MAX) }),
|
|
24
|
+
]);
|
|
25
|
+
const dimension = z.number().int().min(1).max(SURFACE_DIMENSION_MAX);
|
|
26
|
+
/** What a viewer (the host panel) sends the host over the surface socket. */
|
|
27
|
+
export const surfaceViewerMessageSchema = z.discriminatedUnion('type', [
|
|
28
|
+
z.object({ type: z.literal('ack'), seq: z.number().int().min(0) }),
|
|
29
|
+
z.object({ type: z.literal('input'), events: z.array(surfaceInputEventSchema).min(1).max(SURFACE_INPUT_BATCH_MAX) }),
|
|
30
|
+
z.object({ type: z.literal('release') }),
|
|
31
|
+
z.object({ type: z.literal('resize'), width: dimension, height: dimension }),
|
|
32
|
+
z.object({ type: z.literal('clipboard-read'), id: z.string().min(1).max(64) }),
|
|
33
|
+
]);
|
|
34
|
+
export const surfaceControllerSchema = z.enum(SURFACE_CONTROLLERS);
|
|
35
|
+
/**
|
|
36
|
+
* What the host sends a viewer. A `frame` text message is followed by one
|
|
37
|
+
* binary message carrying exactly `bytes` bytes of the image.
|
|
38
|
+
*/
|
|
39
|
+
export const surfaceHostMessageSchema = z.discriminatedUnion('type', [
|
|
40
|
+
z.object({ type: z.literal('hello'), viewerId: z.string().min(1) }),
|
|
41
|
+
z.object({
|
|
42
|
+
type: z.literal('frame'),
|
|
43
|
+
seq: z.number().int().min(0),
|
|
44
|
+
width: dimension,
|
|
45
|
+
height: dimension,
|
|
46
|
+
mime: z.enum(SURFACE_FRAME_MIMES),
|
|
47
|
+
bytes: z.number().int().min(1),
|
|
48
|
+
title: z.string().max(SURFACE_TITLE_MAX).optional(),
|
|
49
|
+
agentActive: z.boolean(),
|
|
50
|
+
}),
|
|
51
|
+
z.object({ type: z.literal('control'), controller: surfaceControllerSchema, mine: z.boolean() }),
|
|
52
|
+
z.object({ type: z.literal('resized'), width: dimension, height: dimension }),
|
|
53
|
+
z.object({ type: z.literal('clipboard'), id: z.string().min(1).max(64), text: z.string().max(SURFACE_TEXT_MAX) }),
|
|
54
|
+
z.object({ type: z.literal('error'), code: z.string().min(1).max(64), message: z.string().min(1).max(500) }),
|
|
55
|
+
z.object({ type: z.literal('ended'), reason: z.enum(['service-stopped', 'extension-unavailable', 'host-shutdown']) }),
|
|
56
|
+
]);
|
|
57
|
+
/** Service answers the host parses. */
|
|
58
|
+
export const surfaceResizeAnswerSchema = z.object({ width: dimension, height: dimension });
|
|
59
|
+
export const surfaceClipboardAnswerSchema = z.object({ text: z.string().max(SURFACE_TEXT_MAX) });
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared surface: a service shows the host a live picture of something an
|
|
3
|
+
* agent is working in (a browser, a simulator, a desktop app) and takes the
|
|
4
|
+
* user's pointer and keyboard back, so a person can watch, step in for a
|
|
5
|
+
* manual step, and hand control back. The host draws it in the extension's
|
|
6
|
+
* rail panel; the service only produces frames and consumes input.
|
|
7
|
+
*
|
|
8
|
+
* Declared with `contributes.service.surface: true`. The host keeps the
|
|
9
|
+
* service running while a viewer is attached and talks plain HTTP to it:
|
|
10
|
+
*
|
|
11
|
+
* - `GET /surface/frame?after=<seq>&wait=<ms>`: the next frame newer than
|
|
12
|
+
* `seq`, held up to `wait` ms. 200 with the image bytes (`Content-Type`
|
|
13
|
+
* `image/jpeg` or `image/png`) and the `x-surface-*` headers below; 204 when
|
|
14
|
+
* nothing newer arrived in time. Sequence numbers start at 1, so `after=0`
|
|
15
|
+
* is "the current picture, now". The host asks for one frame at a time per
|
|
16
|
+
* viewer, so a slow viewer never piles up frames: that is the backpressure.
|
|
17
|
+
* - `POST /surface/input`: `{ events: SurfaceInputEvent[] }`, coordinates in
|
|
18
|
+
* frame pixels. Sent only while the user holds control.
|
|
19
|
+
* - `POST /surface/control`: `{ controller }` whenever control changes, so the
|
|
20
|
+
* service can pause its own automation while the user is in.
|
|
21
|
+
* - `POST /surface/resize`: `{ width, height }` the panel can show; the
|
|
22
|
+
* service answers the size it settled on, or 400 to keep its own.
|
|
23
|
+
* - `GET /surface/clipboard`: `{ text }`, what the user copied inside the
|
|
24
|
+
* surface; the host puts it on their clipboard.
|
|
25
|
+
*/
|
|
26
|
+
export declare const SURFACE_FRAME_PATH = "/surface/frame";
|
|
27
|
+
export declare const SURFACE_INPUT_PATH = "/surface/input";
|
|
28
|
+
export declare const SURFACE_CONTROL_PATH = "/surface/control";
|
|
29
|
+
export declare const SURFACE_RESIZE_PATH = "/surface/resize";
|
|
30
|
+
export declare const SURFACE_CLIPBOARD_PATH = "/surface/clipboard";
|
|
31
|
+
export declare const SURFACE_SEQ_HEADER = "x-surface-seq";
|
|
32
|
+
export declare const SURFACE_WIDTH_HEADER = "x-surface-width";
|
|
33
|
+
export declare const SURFACE_HEIGHT_HEADER = "x-surface-height";
|
|
34
|
+
export declare const SURFACE_TITLE_HEADER = "x-surface-title";
|
|
35
|
+
/** `1` while the service's own automation (an agent) is driving the surface. */
|
|
36
|
+
export declare const SURFACE_AGENT_ACTIVE_HEADER = "x-surface-agent-active";
|
|
37
|
+
export declare const SURFACE_FRAME_MIMES: readonly ["image/jpeg", "image/png"];
|
|
38
|
+
export type SurfaceFrameMime = (typeof SURFACE_FRAME_MIMES)[number];
|
|
39
|
+
/** How long the host lets one frame request wait before asking again. */
|
|
40
|
+
export declare const SURFACE_FRAME_WAIT_MS = 25000;
|
|
41
|
+
export declare const SURFACE_FRAME_MAX_BYTES = 8000000;
|
|
42
|
+
export declare const SURFACE_INPUT_BATCH_MAX = 256;
|
|
43
|
+
export declare const SURFACE_TEXT_MAX = 64000;
|
|
44
|
+
export declare const SURFACE_TITLE_MAX = 200;
|
|
45
|
+
export declare const SURFACE_DIMENSION_MAX = 16384;
|
|
46
|
+
/** After an agent action the agent counts as controlling for this long. */
|
|
47
|
+
export declare const SURFACE_AGENT_HOLD_MS = 30000;
|
|
48
|
+
export type SurfaceModifiers = {
|
|
49
|
+
alt: boolean;
|
|
50
|
+
ctrl: boolean;
|
|
51
|
+
meta: boolean;
|
|
52
|
+
shift: boolean;
|
|
53
|
+
};
|
|
54
|
+
export type SurfacePointerEvent = {
|
|
55
|
+
type: 'pointer';
|
|
56
|
+
action: 'down' | 'up' | 'move';
|
|
57
|
+
x: number;
|
|
58
|
+
y: number;
|
|
59
|
+
/** 0 left, 1 middle, 2 right; `-1` for a move without a button change. */
|
|
60
|
+
button: number;
|
|
61
|
+
/** Bitmask of buttons held, as `MouseEvent.buttons`. */
|
|
62
|
+
buttons: number;
|
|
63
|
+
modifiers: SurfaceModifiers;
|
|
64
|
+
};
|
|
65
|
+
export type SurfaceWheelEvent = {
|
|
66
|
+
type: 'wheel';
|
|
67
|
+
x: number;
|
|
68
|
+
y: number;
|
|
69
|
+
deltaX: number;
|
|
70
|
+
deltaY: number;
|
|
71
|
+
modifiers: SurfaceModifiers;
|
|
72
|
+
};
|
|
73
|
+
export type SurfaceKeyEvent = {
|
|
74
|
+
type: 'key';
|
|
75
|
+
action: 'down' | 'up';
|
|
76
|
+
/** `KeyboardEvent.key`. */
|
|
77
|
+
key: string;
|
|
78
|
+
/** `KeyboardEvent.code`. */
|
|
79
|
+
code: string;
|
|
80
|
+
modifiers: SurfaceModifiers;
|
|
81
|
+
};
|
|
82
|
+
/** Text the user pasted or composed; type it as is. */
|
|
83
|
+
export type SurfaceTextEvent = {
|
|
84
|
+
type: 'text';
|
|
85
|
+
text: string;
|
|
86
|
+
};
|
|
87
|
+
export type SurfaceInputEvent = SurfacePointerEvent | SurfaceWheelEvent | SurfaceKeyEvent | SurfaceTextEvent;
|
|
88
|
+
export type SurfaceInputBatch = {
|
|
89
|
+
events: SurfaceInputEvent[];
|
|
90
|
+
};
|
|
91
|
+
export declare const SURFACE_CONTROLLERS: readonly ["none", "agent", "user"];
|
|
92
|
+
export type SurfaceController = (typeof SURFACE_CONTROLLERS)[number];
|
|
93
|
+
export type SurfaceControlNotice = {
|
|
94
|
+
controller: SurfaceController;
|
|
95
|
+
};
|
|
96
|
+
export type SurfaceResizeRequest = {
|
|
97
|
+
width: number;
|
|
98
|
+
height: number;
|
|
99
|
+
};
|
|
100
|
+
export type SurfaceResizeAnswer = {
|
|
101
|
+
width: number;
|
|
102
|
+
height: number;
|
|
103
|
+
};
|
|
104
|
+
export type SurfaceClipboardAnswer = {
|
|
105
|
+
text: string;
|
|
106
|
+
};
|
|
107
|
+
/**
|
|
108
|
+
* A service's read of the `POST /surface/input` body. Every event is checked
|
|
109
|
+
* field by field; one bad event refuses the whole batch (`null`), which the
|
|
110
|
+
* service answers with HTTP 400. No schema library needed.
|
|
111
|
+
*/
|
|
112
|
+
export declare const readSurfaceInputBatch: (body: string) => SurfaceInputBatch | null;
|
|
113
|
+
/** A service's read of the `POST /surface/control` body. */
|
|
114
|
+
export declare const readSurfaceControlNotice: (body: string) => SurfaceControlNotice | null;
|
|
115
|
+
/** A service's read of the `POST /surface/resize` body. */
|
|
116
|
+
export declare const readSurfaceResizeRequest: (body: string) => SurfaceResizeRequest | null;
|