dsh-builtin-browser 0.1.5 → 0.1.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -1
- package/lib/browser/runtime.d.ts +24 -8
- package/lib/browser/runtime.js +35 -7
- package/lib/browser/types.d.ts +122 -7
- package/lib/browser/types.js +1 -1
- package/lib/browser-electron/host-main.js +85 -0
- package/lib/browser-electron/provider.d.ts +49 -8
- package/lib/browser-electron/provider.js +406 -37
- package/lib/browser-electron/remote-host.js +35 -2
- package/lib/tool-browser/index.d.ts +13 -7
- package/lib/tool-browser/index.js +509 -34
- package/package.json +4 -2
- package/src/browser/runtime.ts +260 -0
- package/src/browser/types.ts +338 -0
- package/src/browser-electron/entry.ts +58 -0
- package/src/browser-electron/host-main.ts +229 -0
- package/src/browser-electron/provider.ts +922 -0
- package/src/browser-electron/remote-host.ts +421 -0
- package/src/index.ts +33 -0
- package/src/tool-browser/index.ts +862 -0
- package/src/types/electron-shim.d.ts +82 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "dsh-builtin-browser",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.7",
|
|
4
4
|
"description": "Shared real browser capability for DeepSeek Harness: a visible browser the human watches and can take over, driven by the agent over CDP (snapshot/execute/content/tab management); host-side seam, provider, and browser_* tools",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "lib/index.js",
|
|
@@ -30,13 +30,15 @@
|
|
|
30
30
|
},
|
|
31
31
|
"files": [
|
|
32
32
|
"lib",
|
|
33
|
+
"src",
|
|
33
34
|
"cordis.patch.yml",
|
|
34
35
|
"README.md",
|
|
35
36
|
"README.en.md",
|
|
36
37
|
"LICENSE"
|
|
37
38
|
],
|
|
38
39
|
"scripts": {
|
|
39
|
-
"build": "tsc -p tsconfig.json"
|
|
40
|
+
"build": "tsc -p tsconfig.json",
|
|
41
|
+
"prepack": "npm run build"
|
|
40
42
|
},
|
|
41
43
|
"keywords": [
|
|
42
44
|
"dsh-plugin",
|
|
@@ -0,0 +1,260 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Service Definition for the browser capability seam (`ctx.browser`): the
|
|
3
|
+
* provider registry and provider-selecting execution for browser sessions.
|
|
4
|
+
* Duplicate ids are rejected. At execution time, a configured provider must
|
|
5
|
+
* exist and be usable; without one, exactly one usable provider is required,
|
|
6
|
+
* so selection never depends on registration order.
|
|
7
|
+
* @module dsh-browser/browser
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
import { Context, Service } from '@deepseek-ai/cordis'
|
|
11
|
+
import z from '@deepseek-ai/schemastery'
|
|
12
|
+
import type {
|
|
13
|
+
BrowserClickRequest,
|
|
14
|
+
BrowserContentRequest,
|
|
15
|
+
BrowserContentResult,
|
|
16
|
+
BrowserDownloadRequest,
|
|
17
|
+
BrowserExecuteRequest,
|
|
18
|
+
BrowserExecuteResult,
|
|
19
|
+
BrowserFillRequest,
|
|
20
|
+
BrowserFillResult,
|
|
21
|
+
BrowserHistoryEntry,
|
|
22
|
+
BrowserNavigateRequest,
|
|
23
|
+
BrowserOpenRequest,
|
|
24
|
+
BrowserProvider,
|
|
25
|
+
BrowserScreenshotRequest,
|
|
26
|
+
BrowserScreenshotResult,
|
|
27
|
+
BrowserSessionId,
|
|
28
|
+
BrowserSnapshotResult,
|
|
29
|
+
BrowserTab,
|
|
30
|
+
BrowserTypeRequest,
|
|
31
|
+
BrowserChallenge,
|
|
32
|
+
ExportedCookie,
|
|
33
|
+
} from './types.ts'
|
|
34
|
+
import { BrowserError } from './types.ts'
|
|
35
|
+
|
|
36
|
+
export {
|
|
37
|
+
BrowserError,
|
|
38
|
+
} from './types.ts'
|
|
39
|
+
export type {
|
|
40
|
+
BrowserChallenge,
|
|
41
|
+
BrowserClickRequest,
|
|
42
|
+
BrowserContentFormat,
|
|
43
|
+
BrowserContentRequest,
|
|
44
|
+
BrowserContentResult,
|
|
45
|
+
BrowserDownloadRequest,
|
|
46
|
+
BrowserExecuteRequest,
|
|
47
|
+
BrowserExecuteResult,
|
|
48
|
+
BrowserFillField,
|
|
49
|
+
BrowserFillRequest,
|
|
50
|
+
BrowserFillResult,
|
|
51
|
+
BrowserHistoryEntry,
|
|
52
|
+
BrowserNavigateRequest,
|
|
53
|
+
BrowserOpenRequest,
|
|
54
|
+
BrowserProvider,
|
|
55
|
+
BrowserScreenshotRequest,
|
|
56
|
+
BrowserScreenshotResult,
|
|
57
|
+
BrowserSessionId,
|
|
58
|
+
BrowserSnapshotElement,
|
|
59
|
+
BrowserSnapshotResult,
|
|
60
|
+
BrowserTab,
|
|
61
|
+
BrowserTypeRequest,
|
|
62
|
+
ExportedCookie,
|
|
63
|
+
} from './types.ts'
|
|
64
|
+
|
|
65
|
+
declare module '@deepseek-ai/cordis' {
|
|
66
|
+
interface Context {
|
|
67
|
+
browser: BrowserRuntime
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* Config for the browser seam. `browserProvider` pins which provider wins;
|
|
73
|
+
* it is optional (a single registered usable provider auto-selects).
|
|
74
|
+
*/
|
|
75
|
+
export interface BrowserRuntimeConfig {
|
|
76
|
+
/** Explicit browser provider id. Omitted = auto-select when exactly one usable. */
|
|
77
|
+
readonly browserProvider?: string
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* The browser access service. Registered as `ctx.browser` (one instance per
|
|
82
|
+
* context).
|
|
83
|
+
*
|
|
84
|
+
* Selection semantics (resolved at execution time, never order-dependent):
|
|
85
|
+
* - A configured id that is registered and `available()` → that provider.
|
|
86
|
+
* - A configured id not registered → `BROWSER_PROVIDER_CONFIGURED_MISSING`.
|
|
87
|
+
* - A configured id registered but unavailable → `BROWSER_PROVIDER_CONFIGURED_UNAVAILABLE`.
|
|
88
|
+
* - No id configured, exactly one registered usable provider → that provider.
|
|
89
|
+
* - No id configured, multiple usable providers → `BROWSER_PROVIDER_AMBIGUOUS`.
|
|
90
|
+
* - No id configured, no usable provider → `BROWSER_PROVIDER_UNAVAILABLE`.
|
|
91
|
+
*/
|
|
92
|
+
export class BrowserRuntime extends Service {
|
|
93
|
+
/** Provider selection config. */
|
|
94
|
+
static Config: z<BrowserRuntimeConfig> = z.object({
|
|
95
|
+
browserProvider: z.string(),
|
|
96
|
+
})
|
|
97
|
+
|
|
98
|
+
private providers = new Map<string, BrowserProvider>()
|
|
99
|
+
private readonly providerId: string | undefined
|
|
100
|
+
|
|
101
|
+
constructor(ctx: Context, config: BrowserRuntimeConfig = {}) {
|
|
102
|
+
super(ctx, 'browser')
|
|
103
|
+
this.providerId = config.browserProvider
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/**
|
|
107
|
+
* Register a browser provider. Throws {@link BrowserError}
|
|
108
|
+
* `BROWSER_DUPLICATE_PROVIDER` if its id is already registered. Returns a
|
|
109
|
+
* disposer; disposed with the calling fiber.
|
|
110
|
+
* @param provider - the provider; its `id` is the registry key.
|
|
111
|
+
* @returns the disposer that unregisters the provider.
|
|
112
|
+
*/
|
|
113
|
+
registerBrowserProvider(provider: BrowserProvider): () => void {
|
|
114
|
+
if (this.providers.has(provider.id)) {
|
|
115
|
+
throw new BrowserError(`a browser provider with id "${provider.id}" is already registered`, 'BROWSER_DUPLICATE_PROVIDER')
|
|
116
|
+
}
|
|
117
|
+
// Bind the generator to this instance instead of aliasing `this` to a
|
|
118
|
+
// local (no-this-alias): the effect body mutates the instance registry.
|
|
119
|
+
const dispose = this.ctx.effect(function* (this: BrowserRuntime) {
|
|
120
|
+
this.providers.set(provider.id, provider)
|
|
121
|
+
yield () => this.providers.delete(provider.id)
|
|
122
|
+
}.bind(this), 'browser.registerProvider()')
|
|
123
|
+
// ctx.effect's disposer returns Promise<void>; our disposer API is
|
|
124
|
+
// synchronous fire-and-forget — discard the (always-resolved) promise.
|
|
125
|
+
return () => void dispose()
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
/** Resolve the selected provider or throw the matching {@link BrowserError}. */
|
|
129
|
+
private resolveProvider(): BrowserProvider {
|
|
130
|
+
const { providerId, providers } = this
|
|
131
|
+
if (providerId !== undefined) {
|
|
132
|
+
const provider = providers.get(providerId)
|
|
133
|
+
if (!provider) {
|
|
134
|
+
throw new BrowserError(`configured browser provider "${providerId}" is not registered`, 'BROWSER_PROVIDER_CONFIGURED_MISSING')
|
|
135
|
+
}
|
|
136
|
+
if (!provider.available()) {
|
|
137
|
+
throw new BrowserError(`configured browser provider "${providerId}" is registered but unavailable`, 'BROWSER_PROVIDER_CONFIGURED_UNAVAILABLE')
|
|
138
|
+
}
|
|
139
|
+
return provider
|
|
140
|
+
}
|
|
141
|
+
const usable = [...providers.values()].filter(provider => provider.available())
|
|
142
|
+
const [single] = usable
|
|
143
|
+
if (single === undefined) {
|
|
144
|
+
throw new BrowserError('no usable browser provider is registered', 'BROWSER_PROVIDER_UNAVAILABLE')
|
|
145
|
+
}
|
|
146
|
+
if (usable.length > 1) {
|
|
147
|
+
const ids = usable.map(provider => provider.id).join(', ')
|
|
148
|
+
throw new BrowserError(`multiple usable browser providers are registered (${ids}); configure one explicitly`, 'BROWSER_PROVIDER_AMBIGUOUS')
|
|
149
|
+
}
|
|
150
|
+
return single
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
/** Open a new browser session through the selected provider. */
|
|
154
|
+
async open(): Promise<BrowserSessionId> {
|
|
155
|
+
return this.resolveProvider().open()
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
/** Open a URL through the selected provider, optionally in a new tab. */
|
|
159
|
+
async openUrl(session: BrowserSessionId, request: BrowserOpenRequest, signal?: AbortSignal): Promise<void> {
|
|
160
|
+
return this.resolveProvider().openUrl(session, request, signal)
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
/** List the session's tabs through the selected provider. */
|
|
164
|
+
async listTabs(session: BrowserSessionId): Promise<readonly BrowserTab[]> {
|
|
165
|
+
return this.resolveProvider().listTabs(session)
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
/** Switch to a tab through the selected provider. */
|
|
169
|
+
async switchTab(session: BrowserSessionId, tabId: string): Promise<void> {
|
|
170
|
+
return this.resolveProvider().switchTab(session, tabId)
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
/** Close one tab through the selected provider. */
|
|
174
|
+
async closeTab(session: BrowserSessionId, tabId: string): Promise<void> {
|
|
175
|
+
return this.resolveProvider().closeTab(session, tabId)
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
/** Close every tab and reset the session through the selected provider. */
|
|
179
|
+
async reset(session: BrowserSessionId): Promise<void> {
|
|
180
|
+
return this.resolveProvider().reset(session)
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
/** Navigate the session's page through the selected provider. */
|
|
184
|
+
async navigate(session: BrowserSessionId, request: BrowserNavigateRequest, signal?: AbortSignal): Promise<void> {
|
|
185
|
+
return this.resolveProvider().navigate(session, request, signal)
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
/** Execute JS in the session's page context through the selected provider. */
|
|
189
|
+
async execute(session: BrowserSessionId, request: BrowserExecuteRequest, signal?: AbortSignal): Promise<BrowserExecuteResult> {
|
|
190
|
+
return this.resolveProvider().execute(session, request, signal)
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
/** Produce an AI-friendly snapshot of the session's page. */
|
|
194
|
+
async snapshot(session: BrowserSessionId, signal?: AbortSignal): Promise<BrowserSnapshotResult> {
|
|
195
|
+
return this.resolveProvider().snapshot(session, signal)
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
/** Fetch page content in a requested format. */
|
|
199
|
+
async content(session: BrowserSessionId, request: BrowserContentRequest, signal?: AbortSignal): Promise<BrowserContentResult> {
|
|
200
|
+
return this.resolveProvider().content(session, request, signal)
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
/** Click at viewport coordinates through the selected provider. */
|
|
204
|
+
async click(session: BrowserSessionId, request: BrowserClickRequest, signal?: AbortSignal): Promise<void> {
|
|
205
|
+
return this.resolveProvider().click(session, request, signal)
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
/** Type into the focused element through the selected provider. */
|
|
209
|
+
async type(session: BrowserSessionId, request: BrowserTypeRequest, signal?: AbortSignal): Promise<void> {
|
|
210
|
+
return this.resolveProvider().type(session, request, signal)
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
/** Fill a form's fields in one batch through the selected provider. */
|
|
214
|
+
async fillForm(session: BrowserSessionId, request: BrowserFillRequest, signal?: AbortSignal): Promise<BrowserFillResult> {
|
|
215
|
+
return this.resolveProvider().fillForm(session, request, signal)
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
/** Capture the current page through the selected provider. */
|
|
219
|
+
async screenshot(session: BrowserSessionId, request?: BrowserScreenshotRequest, signal?: AbortSignal): Promise<BrowserScreenshotResult> {
|
|
220
|
+
return this.resolveProvider().screenshot(session, request, signal)
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
/** Check for a human-verification challenge on the active tab. */
|
|
224
|
+
async detectChallenge(session: BrowserSessionId, signal?: AbortSignal): Promise<BrowserChallenge> {
|
|
225
|
+
return this.resolveProvider().detectChallenge(session, signal)
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
/** Return the session's chronological operation log through the provider. */
|
|
229
|
+
async history(session: BrowserSessionId): Promise<readonly BrowserHistoryEntry[]> {
|
|
230
|
+
return this.resolveProvider().history(session)
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
/** Replay one recorded operation by sequence number through the provider. */
|
|
234
|
+
async replay(session: BrowserSessionId, seq: number): Promise<void> {
|
|
235
|
+
return this.resolveProvider().replay(session, seq)
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
/** Download a URL to a local file through the provider. */
|
|
239
|
+
async download(session: BrowserSessionId, request: BrowserDownloadRequest, signal?: AbortSignal): Promise<{ readonly path: string }> {
|
|
240
|
+
return this.resolveProvider().download(session, request, signal)
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
/** Export the session's cookies through the provider. */
|
|
244
|
+
async flushAuth(session: BrowserSessionId): Promise<readonly ExportedCookie[]> {
|
|
245
|
+
return this.resolveProvider().flushAuth(session)
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
/** Import cookies into the session through the provider. */
|
|
249
|
+
async restoreAuth(session: BrowserSessionId, cookies: readonly ExportedCookie[]): Promise<number> {
|
|
250
|
+
return this.resolveProvider().restoreAuth(session, cookies)
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
/** Close the session through the selected provider. Idempotent. */
|
|
254
|
+
async close(session: BrowserSessionId): Promise<void> {
|
|
255
|
+
return this.resolveProvider().close(session)
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
export default BrowserRuntime
|
|
260
|
+
|
|
@@ -0,0 +1,338 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Vocabulary for the browser capability seam (`ctx.browser`). One seam owns
|
|
3
|
+
* provider registration, session lifecycle, cancellation, errors, and product
|
|
4
|
+
* configuration; providers differ only in what backs a session (an Electron
|
|
5
|
+
* `WebContentsView` in the desktop shell, a headless Chromium relay for
|
|
6
|
+
* remote deployments, and so on).
|
|
7
|
+
* @module dsh-browser/browser/types
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
import { HarnessError } from '@deepseek-ai/dsh-llm'
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* Stable identity of one browser session, resolved by the Host through a
|
|
14
|
+
* Typert `LookupMap` to the live session object (the same mechanism
|
|
15
|
+
* `Agent → agentId` uses). The id is opaque to the wire; only the provider
|
|
16
|
+
* and its host binding can interpret it.
|
|
17
|
+
*/
|
|
18
|
+
export type BrowserSessionId = string
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* What one browser-capable backend is asked to do. Navigation is the minimal
|
|
22
|
+
* operation every provider shares; the seam deliberately keeps the request
|
|
23
|
+
* minimal so a provider swap never changes how the model asks.
|
|
24
|
+
*/
|
|
25
|
+
export interface BrowserNavigateRequest {
|
|
26
|
+
/** The URL to open. Admission (HTTP(S) only, no credentials/private targets) is provider-owned. */
|
|
27
|
+
readonly url: string
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* Click at viewport coordinates. Coordinates are viewport-relative pixels in
|
|
32
|
+
* the session's own coordinate space — the same space the human interacts
|
|
33
|
+
* with, so a real view and a relay agree without scaling.
|
|
34
|
+
*/
|
|
35
|
+
export interface BrowserClickRequest {
|
|
36
|
+
/** Viewport-relative x in CSS pixels. */
|
|
37
|
+
readonly x: number
|
|
38
|
+
/** Viewport-relative y in CSS pixels. */
|
|
39
|
+
readonly y: number
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/** Type text into the focused element. */
|
|
43
|
+
export interface BrowserTypeRequest {
|
|
44
|
+
/** The text to insert. */
|
|
45
|
+
readonly text: string
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/** One field of a batch form fill. Match by selector, or by name/label/placeholder. */
|
|
49
|
+
export interface BrowserFillField {
|
|
50
|
+
/** CSS selector; when present, candidates are scoped to it. */
|
|
51
|
+
readonly selector?: string
|
|
52
|
+
/** Match by the field's `name` attribute. */
|
|
53
|
+
readonly name?: string
|
|
54
|
+
/** Match by associated `<label>` text or `aria-label`. */
|
|
55
|
+
readonly label?: string
|
|
56
|
+
/** Match by `placeholder` text. */
|
|
57
|
+
readonly placeholder?: string
|
|
58
|
+
/** Field kind; defaults to text. */
|
|
59
|
+
readonly kind?: 'text' | 'textarea' | 'checkbox' | 'radio' | 'select'
|
|
60
|
+
/** Value to set: string, number, or boolean (checkbox/radio). For a
|
|
61
|
+
* multi-select or radio group, the option value (or visible text). */
|
|
62
|
+
readonly value: string | number | boolean
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/** Batch form fill: set several fields, optionally submitting the form. */
|
|
66
|
+
export interface BrowserFillRequest {
|
|
67
|
+
/** The fields to fill, in order. */
|
|
68
|
+
readonly fields: readonly BrowserFillField[]
|
|
69
|
+
/** Submit the containing form after filling. Default false. */
|
|
70
|
+
readonly submit?: boolean
|
|
71
|
+
/** Per-field evaluation budget in ms. Default 30000. */
|
|
72
|
+
readonly timeoutMs?: number
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/** One field's outcome in a batch fill. */
|
|
76
|
+
export interface BrowserFillFieldResult {
|
|
77
|
+
readonly ok: boolean
|
|
78
|
+
/** The field description matched (selector/name/label/placeholder). */
|
|
79
|
+
readonly target: string
|
|
80
|
+
/** How the value was applied: input | textarea | select | checkbox | radio | contenteditable. */
|
|
81
|
+
readonly method?: string
|
|
82
|
+
/** Error text when the field could not be filled. */
|
|
83
|
+
readonly error?: string
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/** Outcome of a batch form fill. */
|
|
87
|
+
export interface BrowserFillResult {
|
|
88
|
+
/** Per-field outcomes, in request order. */
|
|
89
|
+
readonly fields: readonly BrowserFillFieldResult[]
|
|
90
|
+
/** Whether the containing form was submitted. */
|
|
91
|
+
readonly submitted: boolean
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/**
|
|
95
|
+
* A screenshot of the current page state. `dataUrl` is a base64 data URL the
|
|
96
|
+
* tool layer renders directly; the live view itself never crosses the wire
|
|
97
|
+
* (the human watches the real view in the desktop shape).
|
|
98
|
+
*/
|
|
99
|
+
export interface BrowserScreenshotResult {
|
|
100
|
+
/** Base64 data URL of the capture (PNG or JPEG per the request). */
|
|
101
|
+
readonly dataUrl: string
|
|
102
|
+
/** File path the capture was also saved to, when the request asked for it. */
|
|
103
|
+
readonly path?: string
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/** Screenshot capture options. PNG only (CDP JPEG hangs on Electron 43). */
|
|
107
|
+
export interface BrowserScreenshotRequest {
|
|
108
|
+
/** Capture the full scrollable page instead of the viewport. Default false. */
|
|
109
|
+
readonly fullPage?: boolean
|
|
110
|
+
/** Absolute file path to also save the PNG to (for vision-tool reads). */
|
|
111
|
+
readonly savePath?: string
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/** Download options: fetch a URL into a local file, keeping the session's cookies. */
|
|
115
|
+
export interface BrowserDownloadRequest {
|
|
116
|
+
/** The URL to download. */
|
|
117
|
+
readonly url: string
|
|
118
|
+
/** Absolute path of the file to write. */
|
|
119
|
+
readonly savePath: string
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/** A serializable cookie, exported from or imported into the browser session. */
|
|
123
|
+
export interface ExportedCookie {
|
|
124
|
+
/** Cookie URL (scheme + domain + path), as accepted by cookies.set. */
|
|
125
|
+
readonly url: string
|
|
126
|
+
readonly name: string
|
|
127
|
+
readonly value: string
|
|
128
|
+
readonly domain?: string
|
|
129
|
+
readonly path?: string
|
|
130
|
+
readonly secure?: boolean
|
|
131
|
+
readonly httpOnly?: boolean
|
|
132
|
+
readonly expirationDate?: number
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/**
|
|
136
|
+
* Execute arbitrary JavaScript in the session's page context. This is the
|
|
137
|
+
* primary interaction path — DOM-referenced, not screen-coordinate-based:
|
|
138
|
+
* focus/input/click go through the page's own JS (native setters for
|
|
139
|
+
* framework-controlled inputs), matching how a human-driven agent operates.
|
|
140
|
+
*/
|
|
141
|
+
export interface BrowserExecuteRequest {
|
|
142
|
+
/** The JavaScript expression to evaluate in the page context. */
|
|
143
|
+
readonly script: string
|
|
144
|
+
/** Optional arguments injected into the script scope as `arguments[0..n]`. */
|
|
145
|
+
readonly args?: readonly unknown[]
|
|
146
|
+
/** Evaluation budget in ms; a wedged page rejects with a timeout. Default 30000. */
|
|
147
|
+
readonly timeoutMs?: number
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
/**
|
|
151
|
+
* Result of an execute. Mirrors CDP `Runtime.evaluate` semantics: either a
|
|
152
|
+
* value (by value, not by reference) or the exception thrown.
|
|
153
|
+
*/
|
|
154
|
+
export type BrowserExecuteResult =
|
|
155
|
+
| { readonly ok: true; readonly value: unknown }
|
|
156
|
+
| { readonly ok: false; readonly exception: string }
|
|
157
|
+
|
|
158
|
+
/**
|
|
159
|
+
* One interactive element in a snapshot, with a stable reference number the
|
|
160
|
+
* tool layer (and the model) can cite to target the element.
|
|
161
|
+
*/
|
|
162
|
+
export interface BrowserSnapshotElement {
|
|
163
|
+
/** 1-based reference number, stable for the lifetime of one snapshot. */
|
|
164
|
+
readonly ref: number
|
|
165
|
+
/** Element kind: 'input' | 'button' | 'link' | 'textarea' | 'select' | 'checkbox' | 'other'. */
|
|
166
|
+
readonly kind: string
|
|
167
|
+
/** Text content or placeholder/label, truncated for the snapshot. */
|
|
168
|
+
readonly label: string
|
|
169
|
+
/** Best-effort CSS selector; may be empty when none is derivable. */
|
|
170
|
+
readonly selector: string
|
|
171
|
+
/** Viewport-relative center, for coordinate fallbacks. */
|
|
172
|
+
readonly x: number
|
|
173
|
+
readonly y: number
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
/**
|
|
177
|
+
* AI-friendly page snapshot: a compact, numbered inventory of interactive
|
|
178
|
+
* elements plus page facts. Not raw HTML — the model dialogues with this.
|
|
179
|
+
*/
|
|
180
|
+
export interface BrowserSnapshotResult {
|
|
181
|
+
/** Final page URL. */
|
|
182
|
+
readonly url: string
|
|
183
|
+
/** Page title, if any. */
|
|
184
|
+
readonly title?: string
|
|
185
|
+
/** Numbered interactive elements. */
|
|
186
|
+
readonly elements: readonly BrowserSnapshotElement[]
|
|
187
|
+
/** True when the snapshot was truncated (element cap reached). */
|
|
188
|
+
readonly truncated: boolean
|
|
189
|
+
/** Human-verification challenge blocking the page, when one is detected. */
|
|
190
|
+
readonly challenge?: BrowserChallenge
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
/**
|
|
194
|
+
* A detected human-verification (bot-detection / CAPTCHA) challenge blocking
|
|
195
|
+
* the page. Detection is best-effort and marker-based; a clean result is not a
|
|
196
|
+
* guarantee that no challenge exists.
|
|
197
|
+
*/
|
|
198
|
+
export interface BrowserChallenge {
|
|
199
|
+
/** Whether a challenge currently blocks the page. */
|
|
200
|
+
readonly blocked: boolean
|
|
201
|
+
/** Machine-readable kind: cloudflare | hcaptcha | recaptcha | turnstile | generic. */
|
|
202
|
+
readonly kind?: string
|
|
203
|
+
/** Short human-readable description, e.g. 'Cloudflare "Just a moment" interstitial'. */
|
|
204
|
+
readonly reason?: string
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
/**
|
|
208
|
+
* Content fetch formats. `html` is the raw DOM; `markdown` is a structured
|
|
209
|
+
* reading view; `txt` is plain text (smallest); `json` is structured data.
|
|
210
|
+
*/
|
|
211
|
+
export type BrowserContentFormat = 'html' | 'markdown' | 'txt' | 'json'
|
|
212
|
+
|
|
213
|
+
/** Content fetch request: format, optional selector scoping, and caps. */
|
|
214
|
+
export interface BrowserContentRequest {
|
|
215
|
+
/** Desired format. */
|
|
216
|
+
readonly format: BrowserContentFormat
|
|
217
|
+
/** Optional CSS selector limiting the fetch to one region (e.g. `#main`). */
|
|
218
|
+
readonly selector?: string
|
|
219
|
+
/** Maximum characters of the returned content. */
|
|
220
|
+
readonly maxChars?: number
|
|
221
|
+
/** Evaluation timeout in ms; the fetch aborts after this. Default 30000. */
|
|
222
|
+
readonly timeoutMs?: number
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
/** A fetched page content in the requested format. */
|
|
226
|
+
export interface BrowserContentResult {
|
|
227
|
+
/** The content in the requested format, capped at `maxChars`. */
|
|
228
|
+
readonly content: string
|
|
229
|
+
/** True when the content was truncated. */
|
|
230
|
+
readonly truncated: boolean
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
/**
|
|
234
|
+
* A tab inside a browser session. Tabs share the session's profile/cookies
|
|
235
|
+
* and are switchable without losing state.
|
|
236
|
+
*/
|
|
237
|
+
export interface BrowserTab {
|
|
238
|
+
/** Stable tab id within the session. */
|
|
239
|
+
readonly id: string
|
|
240
|
+
/** Current URL, or empty for a blank tab. */
|
|
241
|
+
readonly url: string
|
|
242
|
+
/** Page title, if any. */
|
|
243
|
+
readonly title?: string
|
|
244
|
+
/** Whether this tab is the active one. */
|
|
245
|
+
readonly active: boolean
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
/** Request to open a URL: into the active tab or a new tab. */
|
|
249
|
+
export interface BrowserOpenRequest {
|
|
250
|
+
/** The URL to open. */
|
|
251
|
+
readonly url: string
|
|
252
|
+
/** Open in a new tab (parallel). Default false (reuse the active tab). */
|
|
253
|
+
readonly newTab?: boolean
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
/**
|
|
257
|
+
* A browser-capable backend. Registered with `ctx.browser.registerBrowserProvider`.
|
|
258
|
+
* `id` is a stable string, unique across the seam.
|
|
259
|
+
*/
|
|
260
|
+
export interface BrowserProvider {
|
|
261
|
+
readonly id: string
|
|
262
|
+
/** Cheap local usability check; must not make network calls. */
|
|
263
|
+
available(): boolean
|
|
264
|
+
/**
|
|
265
|
+
* Open a new session. The provider mints the session id and prepares its
|
|
266
|
+
* backing surface (a view, a headless page, and so on). Sessions are isolated from
|
|
267
|
+
* each other; a session must not be visible to any other session's caller.
|
|
268
|
+
*/
|
|
269
|
+
open(): Promise<BrowserSessionId>
|
|
270
|
+
/** Open a URL, optionally in a new tab. Honor `signal` for cancellation. */
|
|
271
|
+
openUrl(session: BrowserSessionId, request: BrowserOpenRequest, signal?: AbortSignal): Promise<void>
|
|
272
|
+
/** List the session's tabs. */
|
|
273
|
+
listTabs(session: BrowserSessionId): Promise<readonly BrowserTab[]>
|
|
274
|
+
/** Switch to a tab by id. Unknown id -> `BROWSER_TAB_UNKNOWN`. */
|
|
275
|
+
switchTab(session: BrowserSessionId, tabId: string): Promise<void>
|
|
276
|
+
/** Close one tab. Closing the active tab activates the next. */
|
|
277
|
+
closeTab(session: BrowserSessionId, tabId: string): Promise<void>
|
|
278
|
+
/** Close every tab and reset the session's state. */
|
|
279
|
+
reset(session: BrowserSessionId): Promise<void>
|
|
280
|
+
/** Navigate the active tab. Honor `signal` for cancellation. */
|
|
281
|
+
navigate(session: BrowserSessionId, request: BrowserNavigateRequest, signal?: AbortSignal): Promise<void>
|
|
282
|
+
/** Execute JS in the active tab's page context. */
|
|
283
|
+
execute(session: BrowserSessionId, request: BrowserExecuteRequest, signal?: AbortSignal): Promise<BrowserExecuteResult>
|
|
284
|
+
/** Produce an AI-friendly snapshot of the active tab. */
|
|
285
|
+
snapshot(session: BrowserSessionId, signal?: AbortSignal): Promise<BrowserSnapshotResult>
|
|
286
|
+
/** Check whether a human-verification challenge is blocking the active tab. */
|
|
287
|
+
detectChallenge(session: BrowserSessionId, signal?: AbortSignal): Promise<BrowserChallenge>
|
|
288
|
+
/** Fetch page content in a requested format. */
|
|
289
|
+
content(session: BrowserSessionId, request: BrowserContentRequest, signal?: AbortSignal): Promise<BrowserContentResult>
|
|
290
|
+
/** Click at viewport coordinates (fallback path; execute is preferred). Honor `signal` for cancellation. */
|
|
291
|
+
click(session: BrowserSessionId, request: BrowserClickRequest, signal?: AbortSignal): Promise<void>
|
|
292
|
+
/** Type into the focused element (fallback path). Honor `signal` for cancellation. */
|
|
293
|
+
type(session: BrowserSessionId, request: BrowserTypeRequest, signal?: AbortSignal): Promise<void>
|
|
294
|
+
/** Fill a form's fields in one batch. Honor `signal` for cancellation. */
|
|
295
|
+
fillForm(session: BrowserSessionId, request: BrowserFillRequest, signal?: AbortSignal): Promise<BrowserFillResult>
|
|
296
|
+
/** Capture the current page. Honor `signal` for cancellation. */
|
|
297
|
+
screenshot(session: BrowserSessionId, request?: BrowserScreenshotRequest, signal?: AbortSignal): Promise<BrowserScreenshotResult>
|
|
298
|
+
/** Download a URL to a local file. Honor `signal` for cancellation. */
|
|
299
|
+
download(session: BrowserSessionId, request: BrowserDownloadRequest, signal?: AbortSignal): Promise<{ readonly path: string }>
|
|
300
|
+
/** Export the session's cookies (login state). */
|
|
301
|
+
flushAuth(session: BrowserSessionId): Promise<readonly ExportedCookie[]>
|
|
302
|
+
/** Import cookies into the session (restore login state). */
|
|
303
|
+
restoreAuth(session: BrowserSessionId, cookies: readonly ExportedCookie[]): Promise<number>
|
|
304
|
+
/** Return the session's chronological operation log. */
|
|
305
|
+
history(session: BrowserSessionId): Promise<readonly BrowserHistoryEntry[]>
|
|
306
|
+
/** Replay one recorded operation by sequence number. */
|
|
307
|
+
replay(session: BrowserSessionId, seq: number): Promise<void>
|
|
308
|
+
/** Close the session and destroy its backing surface. Idempotent. */
|
|
309
|
+
close(session: BrowserSessionId): Promise<void>
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
/** One recorded browser operation, in chronological order (seq 1, 2, 3…). */
|
|
313
|
+
export interface BrowserHistoryEntry {
|
|
314
|
+
/** Monotonic sequence number within the session. */
|
|
315
|
+
readonly seq: number
|
|
316
|
+
/** The operation name: navigate | execute | click | type | replay. */
|
|
317
|
+
readonly action: string
|
|
318
|
+
/** The operation's arguments. */
|
|
319
|
+
readonly params: Record<string, unknown>
|
|
320
|
+
/** Whether the operation succeeded. */
|
|
321
|
+
readonly ok: boolean
|
|
322
|
+
/** Truncated result text, when the operation returned one. */
|
|
323
|
+
readonly result?: string
|
|
324
|
+
/** Error text, when the operation failed. */
|
|
325
|
+
readonly error?: string
|
|
326
|
+
/** Epoch millis of the operation. */
|
|
327
|
+
readonly at: number
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
/**
|
|
331
|
+
* Typed browser error with a machine-routable, open-string `code` and chained
|
|
332
|
+
* `cause`. Consumers must tolerate provider-specific codes. Shared codes cover
|
|
333
|
+
* unavailable, missing, unusable, ambiguous, or duplicate providers, unknown
|
|
334
|
+
* or closed sessions, cancellation, and provider failure; a provider adds its
|
|
335
|
+
* own (e.g. `BROWSER_CDP_ATTACH_FAILED`).
|
|
336
|
+
*/
|
|
337
|
+
export class BrowserError extends HarnessError {}
|
|
338
|
+
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Electron browser provider plugin entry: registers the Electron-backed
|
|
3
|
+
* `BrowserProvider` with `ctx.browser`. The provider needs a view host (real
|
|
4
|
+
* Electron `WebContentsView` objects). When a desktop shell supplies
|
|
5
|
+
* `ctx.electronViewHost`, that host is used (embedded, human-machine shared
|
|
6
|
+
* view). Otherwise the plugin self-hosts: it spawns its own Electron child
|
|
7
|
+
* (`host-main.js`) and drives it over a local TCP JSON-RPC socket, so
|
|
8
|
+
* installing the plugin is enough for `browser_*` tools to work on any
|
|
9
|
+
* surface.
|
|
10
|
+
* @module dsh-browser/browser-electron
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
import type { Context } from '@deepseek-ai/cordis'
|
|
14
|
+
import z from '@deepseek-ai/schemastery'
|
|
15
|
+
import type { BrowserRuntime } from '../browser/runtime.ts'
|
|
16
|
+
import { ElectronBrowserProvider } from './provider.ts'
|
|
17
|
+
import type { ElectronBrowserViewHost } from './provider.ts'
|
|
18
|
+
import { defaultHostMainPath, RemoteElectronViewHost } from './remote-host.ts'
|
|
19
|
+
|
|
20
|
+
export {
|
|
21
|
+
ELECTRON_BROWSER_PROVIDER_ID,
|
|
22
|
+
ElectronBrowserProvider,
|
|
23
|
+
} from './provider.ts'
|
|
24
|
+
export type { ElectronBrowserViewHost, ElectronViewHandle } from './provider.ts'
|
|
25
|
+
export { RemoteElectronViewHost, defaultHostMainPath } from './remote-host.ts'
|
|
26
|
+
|
|
27
|
+
/** Cordis plugin name used by loader diagnostics. */
|
|
28
|
+
export const name = 'browser-electron'
|
|
29
|
+
|
|
30
|
+
/** The browser seam this provider registers into. */
|
|
31
|
+
export const inject = ['browser']
|
|
32
|
+
|
|
33
|
+
/** Plugin config: an optional externally-supplied view host. */
|
|
34
|
+
export interface Config {
|
|
35
|
+
/** View host supplied by a desktop shell; absent -> self-host. */
|
|
36
|
+
readonly viewHost?: ElectronBrowserViewHost
|
|
37
|
+
/** Allow navigation only to HTTP(S) URLs. Default true. */
|
|
38
|
+
readonly httpOnly?: boolean
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export const Config: z<Config> = z.object({
|
|
42
|
+
// Absent on surfaces without a desktop shell; the plugin self-hosts then.
|
|
43
|
+
viewHost: z.any(),
|
|
44
|
+
httpOnly: z.boolean().default(true),
|
|
45
|
+
})
|
|
46
|
+
|
|
47
|
+
/** Register the Electron browser provider with `ctx.browser`. */
|
|
48
|
+
export function apply(ctx: Context & { browser: BrowserRuntime }, config: Config): void {
|
|
49
|
+
// External host (desktop shell) wins; otherwise self-host. The self-hosted
|
|
50
|
+
// child is disposed with the fiber, mirroring the shell's lifetime.
|
|
51
|
+
const host: ElectronBrowserViewHost = config.viewHost ?? new RemoteElectronViewHost(defaultHostMainPath())
|
|
52
|
+
ctx.effect(() => () => {
|
|
53
|
+
if (config.viewHost === undefined && host instanceof RemoteElectronViewHost) {
|
|
54
|
+
host.dispose()
|
|
55
|
+
}
|
|
56
|
+
})
|
|
57
|
+
ctx.browser.registerBrowserProvider(new ElectronBrowserProvider(host, { httpOnly: config.httpOnly }))
|
|
58
|
+
}
|