@memberjunction/remote-browser-base 0.0.1 → 5.42.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,488 @@
1
+ /**
2
+ * The live-session contract for the Remote Browser channel and the self-contained, strongly-typed
3
+ * action / input / output payloads it carries.
4
+ *
5
+ * These types are defined **self-contained here** (no dependency on `@memberjunction/computer-use` or
6
+ * Playwright) so this Base package stays universal — usable client *and* server. The server package
7
+ * and the backend drivers (which depend on `@memberjunction/computer-use`) *implement*
8
+ * {@link IRemoteBrowserSession} against a real CDP-connected browser; this package only declares the
9
+ * shape both sides agree on.
10
+ *
11
+ * @see `/plans/realtime/realtime-bridges-architecture.md` §4d (the Remote Browser channel) and §4d-i
12
+ * (the one-primitive-CDP build decision).
13
+ */
14
+ import type { UserInfo } from '@memberjunction/core';
15
+ import type { RemoteBrowserControlStrategy } from './control.js';
16
+ /**
17
+ * Navigate the browser to a URL.
18
+ */
19
+ export interface RemoteBrowserNavigateAction {
20
+ Kind: 'navigate';
21
+ /** The absolute URL to load. */
22
+ Url: string;
23
+ }
24
+ /**
25
+ * Click an element. Identify the target either by CSS `Selector` or by viewport coordinates
26
+ * (`X` / `Y`); a driver prefers `Selector` when both are present.
27
+ */
28
+ export interface RemoteBrowserClickAction {
29
+ Kind: 'click';
30
+ /** Optional CSS selector identifying the element to click. */
31
+ Selector?: string;
32
+ /** Optional viewport X coordinate (used when no `Selector` is given). */
33
+ X?: number;
34
+ /** Optional viewport Y coordinate (used when no `Selector` is given). */
35
+ Y?: number;
36
+ }
37
+ /**
38
+ * Type text. When `Selector` is provided the driver focuses that element first; otherwise the text is
39
+ * sent to the currently-focused element.
40
+ */
41
+ export interface RemoteBrowserTypeAction {
42
+ Kind: 'type';
43
+ /** The text to type. */
44
+ Text: string;
45
+ /** Optional CSS selector to focus before typing. */
46
+ Selector?: string;
47
+ }
48
+ /**
49
+ * Press a single key or key-combination (e.g. `'Enter'`, `'Escape'`, `'Control+A'`).
50
+ */
51
+ export interface RemoteBrowserKeyAction {
52
+ Kind: 'key';
53
+ /** The key (or combination) to press, in Playwright/CDP key syntax. */
54
+ Key: string;
55
+ }
56
+ /**
57
+ * Scroll the viewport (by `DeltaX` / `DeltaY` pixels) or scroll a specific element into view (by
58
+ * `Selector`). At least one of the three should be supplied; a driver scrolls `Selector` into view
59
+ * when present, otherwise applies the deltas.
60
+ */
61
+ export interface RemoteBrowserScrollAction {
62
+ Kind: 'scroll';
63
+ /** Horizontal scroll distance in pixels (positive = right). */
64
+ DeltaX?: number;
65
+ /** Vertical scroll distance in pixels (positive = down). */
66
+ DeltaY?: number;
67
+ /** Optional CSS selector to scroll into view instead of applying deltas. */
68
+ Selector?: string;
69
+ }
70
+ /**
71
+ * Navigate back in history.
72
+ */
73
+ export interface RemoteBrowserBackAction {
74
+ Kind: 'back';
75
+ }
76
+ /**
77
+ * Navigate forward in history.
78
+ */
79
+ export interface RemoteBrowserForwardAction {
80
+ Kind: 'forward';
81
+ }
82
+ /**
83
+ * Wait — either for a fixed duration (`Ms`) or until an element matching `Selector` appears. At least
84
+ * one should be supplied; a driver waits for `Selector` when present, otherwise sleeps for `Ms`.
85
+ */
86
+ export interface RemoteBrowserWaitAction {
87
+ Kind: 'wait';
88
+ /** Fixed wait in milliseconds. */
89
+ Ms?: number;
90
+ /** Optional CSS selector to wait for instead of a fixed duration. */
91
+ Selector?: string;
92
+ }
93
+ /**
94
+ * The full agent action vocabulary — a discriminated union over the `Kind` field. The realtime agent
95
+ * emits these as tool calls and the channel's session executes them over CDP. Strongly typed with no
96
+ * `any`; narrow on `action.Kind` to access the kind-specific fields.
97
+ */
98
+ export type RemoteBrowserAction = RemoteBrowserNavigateAction | RemoteBrowserClickAction | RemoteBrowserTypeAction | RemoteBrowserKeyAction | RemoteBrowserScrollAction | RemoteBrowserBackAction | RemoteBrowserForwardAction | RemoteBrowserWaitAction;
99
+ /**
100
+ * The outcome of executing a {@link RemoteBrowserAction} (or {@link IRemoteBrowserSession.Navigate}).
101
+ */
102
+ export interface RemoteBrowserActionResult {
103
+ /** Whether the action completed successfully. */
104
+ Success: boolean;
105
+ /** The page URL after the action, when known. */
106
+ CurrentUrl?: string;
107
+ /** Optional human-readable detail (an error message on failure, a note on success). */
108
+ Detail?: string;
109
+ }
110
+ /** One progress update from a goal run — emitted per perceive-act step (transport-neutral). */
111
+ export interface RemoteBrowserGoalProgress {
112
+ /** 1-based step number. */
113
+ Step: number;
114
+ /** A short, model-safe human-readable note (e.g. the controller's reasoning summary). */
115
+ Message: string;
116
+ /** The page URL at this step, when known. */
117
+ Url?: string;
118
+ }
119
+ /**
120
+ * Options for {@link IRemoteBrowserSession.RunComputerUseGoal}. Transport-neutral — concrete model
121
+ * selection (vision/action controller, judge) is resolved by the goal engine the CDP layer binds, so
122
+ * this base type carries no computer-use SDK types.
123
+ */
124
+ export interface RunComputerUseGoalOptions {
125
+ /** Optional URL to navigate to before the goal loop begins. */
126
+ StartUrl?: string;
127
+ /** Maximum perceive-act steps before the loop gives up. */
128
+ MaxSteps?: number;
129
+ /** Optional controller (vision/action) model id, when overriding the engine's auto-selection. */
130
+ ControllerModelId?: string;
131
+ /** Optional judge model id, when overriding the engine's auto-selection. */
132
+ JudgeModelId?: string;
133
+ /**
134
+ * Model-blind context object. Values are referenced by `{{path}}` label in the goal/actions and
135
+ * injected at the action-execution boundary — never seen by any model. (Wired in a later phase; see
136
+ * `plans/realtime/computer-use-remote-browser-blend.md` §4.)
137
+ */
138
+ Context?: Record<string, unknown>;
139
+ /** Invoked per step so the caller (e.g. a realtime voice session) can narrate progress. */
140
+ OnProgress?: (progress: RemoteBrowserGoalProgress) => void;
141
+ /** Abort signal — when aborted (barge-in), the goal loop stops cooperatively. */
142
+ Signal?: AbortSignal;
143
+ /**
144
+ * The MJ user the goal run executes as. The CDP layer forwards it to the bound goal engine so an
145
+ * MJ-aware engine (e.g. `MJComputerUseEngine`) runs its controller/judge prompts under this user
146
+ * (prompt-run logging, model access, credential resolution). Transport-neutral here — typed only as
147
+ * {@link UserInfo}, carrying no computer-use SDK types.
148
+ */
149
+ ContextUser?: UserInfo;
150
+ /**
151
+ * Optional parent `MJ: AI Agent Runs` id for observability. When set (with {@link AgentRunStepID}), an
152
+ * MJ-aware goal engine links the run's prompt runs to this agent run and nests them under the step.
153
+ */
154
+ AgentRunID?: string;
155
+ /**
156
+ * Optional parent `MJ: AI Agent Run Steps` id (the goal's step). When set (with {@link AgentRunID}), an
157
+ * MJ-aware goal engine nests a child `Prompt` step per prompt under it — grouping the goal's many prompt
158
+ * runs beneath a single step in the realtime agent run.
159
+ */
160
+ AgentRunStepID?: string;
161
+ }
162
+ /**
163
+ * The outcome of an autonomous, goal-driven browser run — either the computer-use loop
164
+ * ({@link IRemoteBrowserSession.RunComputerUseGoal}) or a backend's native AI control
165
+ * ({@link IRemoteBrowserSession.InvokeNativeAIControl}), unified by {@link resolveControlStrategy}.
166
+ */
167
+ export interface RemoteBrowserGoalResult {
168
+ /** Whether the goal was achieved. */
169
+ Success: boolean;
170
+ /** Which control strategy executed the goal. */
171
+ Strategy?: RemoteBrowserControlStrategy;
172
+ /** The page URL when the run ended, when known. */
173
+ CurrentUrl?: string;
174
+ /** A terminal status label (e.g. `'Completed'`, `'MaxStepsReached'`, `'Impossible'`, `'Error'`). */
175
+ Status?: string;
176
+ /** Number of perceive-act steps executed (computer-use strategy). */
177
+ StepCount?: number;
178
+ /** Human-readable detail (judge feedback / error message). */
179
+ Detail?: string;
180
+ }
181
+ /**
182
+ * A single encoded viewport frame emitted by the live screencast (CDP `Page.startScreencast` for
183
+ * self-host, or a provider live-view stream). Frames feed the channel's ScreenOut media track when
184
+ * screen-sharing the browser into a meeting, or a panel in the MJ console.
185
+ */
186
+ export interface RemoteBrowserScreencastFrame {
187
+ /** The frame image, Base64-encoded (typically JPEG/PNG per the backend). */
188
+ DataBase64: string;
189
+ /** Frame width in pixels. */
190
+ Width: number;
191
+ /** Frame height in pixels. */
192
+ Height: number;
193
+ /** Monotonically increasing sequence number for ordering / drop detection. */
194
+ SequenceNumber: number;
195
+ }
196
+ /**
197
+ * A single encoded chunk of audio streamed FROM the remote browser to the user — the soundtrack of
198
+ * whatever the browser is playing (e.g. a YouTube video the co-agent is demoing). The sibling of
199
+ * {@link RemoteBrowserScreencastFrame} for audio: chunks feed the channel's client-side audio player
200
+ * (a MediaSource fed `audio/webm;codecs=opus`).
201
+ *
202
+ * The default (Self-Hosted Chrome) capture path produces `'webm-opus'` chunks via an in-page
203
+ * `MediaRecorder`; `'opus'` / `'pcm16'` are reserved for future backend capture paths (e.g. a
204
+ * server-side virtual audio sink).
205
+ */
206
+ export interface RemoteBrowserAudioChunk {
207
+ /** The encoded audio data, Base64-encoded (no `data:` prefix). */
208
+ DataBase64: string;
209
+ /** The codec / container of {@link DataBase64}. `'webm-opus'` for the default in-page recorder. */
210
+ Codec: 'webm-opus' | 'opus' | 'pcm16';
211
+ /** Sample rate in Hz (typically 48000 for the Opus path). */
212
+ SampleRate: number;
213
+ /** Channel count (1 = mono, 2 = stereo). */
214
+ Channels: number;
215
+ /** Monotonically increasing sequence number for ordering / drop detection / resync. */
216
+ SequenceNumber: number;
217
+ /** Approximate duration of this chunk in milliseconds, when known. */
218
+ DurationMs?: number;
219
+ }
220
+ /**
221
+ * A keyboard modifier key that can be held while a human input occurs. These ride on pointer clicks
222
+ * (so Shift-click text selection / Ctrl-click new-tab semantics relay faithfully) AND on key presses
223
+ * (so combos like Ctrl/Cmd+A select-all, Cmd+C / Cmd+V relay faithfully). `'Meta'` is the Command key
224
+ * on macOS / the Windows key elsewhere. Platform-agnostic at this layer — the CDP mapper translates to
225
+ * Playwright/CDP modifier syntax.
226
+ */
227
+ export type RemoteBrowserModifierKey = 'Shift' | 'Control' | 'Alt' | 'Meta';
228
+ /**
229
+ * A human pointer move into the browser viewport.
230
+ */
231
+ export interface RemoteBrowserPointerMoveInput {
232
+ Kind: 'pointer-move';
233
+ /** Viewport X coordinate. */
234
+ X: number;
235
+ /** Viewport Y coordinate. */
236
+ Y: number;
237
+ }
238
+ /**
239
+ * A human pointer click into the browser viewport.
240
+ */
241
+ export interface RemoteBrowserPointerClickInput {
242
+ Kind: 'pointer-click';
243
+ /** Viewport X coordinate. */
244
+ X: number;
245
+ /** Viewport Y coordinate. */
246
+ Y: number;
247
+ /** Which mouse button was used (defaults to `'left'` when omitted). */
248
+ Button?: 'left' | 'middle' | 'right';
249
+ /**
250
+ * Modifier keys held during the click (e.g. `['Shift']` for shift-click text selection). Omitted /
251
+ * empty means no modifiers.
252
+ */
253
+ Modifiers?: RemoteBrowserModifierKey[];
254
+ }
255
+ /**
256
+ * A human pointer-button press (mouse-down) at a viewport point WITHOUT a release — the start of a
257
+ * click-drag. Pairs with {@link RemoteBrowserPointerUpInput} (and any intervening
258
+ * {@link RemoteBrowserPointerMoveInput}s) to relay a drag, e.g. click-drag text selection in a field.
259
+ */
260
+ export interface RemoteBrowserPointerDownInput {
261
+ Kind: 'pointer-down';
262
+ /** Viewport X coordinate. */
263
+ X: number;
264
+ /** Viewport Y coordinate. */
265
+ Y: number;
266
+ /** Which mouse button was pressed (defaults to `'left'` when omitted). */
267
+ Button?: 'left' | 'middle' | 'right';
268
+ /** Modifier keys held during the press. Omitted / empty means no modifiers. */
269
+ Modifiers?: RemoteBrowserModifierKey[];
270
+ }
271
+ /**
272
+ * A human pointer-button release (mouse-up) at a viewport point — the end of a click-drag started by a
273
+ * {@link RemoteBrowserPointerDownInput}.
274
+ */
275
+ export interface RemoteBrowserPointerUpInput {
276
+ Kind: 'pointer-up';
277
+ /** Viewport X coordinate. */
278
+ X: number;
279
+ /** Viewport Y coordinate. */
280
+ Y: number;
281
+ /** Which mouse button was released (defaults to `'left'` when omitted). */
282
+ Button?: 'left' | 'middle' | 'right';
283
+ /** Modifier keys held during the release. Omitted / empty means no modifiers. */
284
+ Modifiers?: RemoteBrowserModifierKey[];
285
+ }
286
+ /**
287
+ * A human key press routed into the browser during takeover.
288
+ */
289
+ export interface RemoteBrowserKeyInput {
290
+ Kind: 'key';
291
+ /** The key (or combination) pressed, in Playwright/CDP key syntax. */
292
+ Key: string;
293
+ /**
294
+ * Modifier keys held during the press (e.g. `['Control']` with `Key: 'a'` for select-all). The
295
+ * mapper composes these with `Key` into a single Playwright/CDP chord. Omitted / empty means the
296
+ * `Key` is pressed on its own.
297
+ */
298
+ Modifiers?: RemoteBrowserModifierKey[];
299
+ }
300
+ /**
301
+ * A block of text the human pastes INTO the browser during takeover — the human-input twin of the
302
+ * agent's {@link RemoteBrowserTypeAction}. The text is inserted into the remote page's currently-focused
303
+ * element (no key-by-key synthesis), which is how VNC/remote-desktop paste works: the viewer reads the
304
+ * LOCAL clipboard on a `paste` event and relays the text here, sidestepping the isolated remote clipboard.
305
+ */
306
+ export interface RemoteBrowserTextInput {
307
+ Kind: 'text';
308
+ /** The text inserted verbatim into the remote browser's focused element (for human paste). */
309
+ Text: string;
310
+ }
311
+ /**
312
+ * A human mouse-wheel / trackpad scroll into the browser viewport (Magic Mouse scroll, trackpad
313
+ * two-finger scroll, wheel). `X` / `Y` are the viewport pixel position the scroll occurred over (so a
314
+ * scroll targets the element under the pointer); `DeltaX` / `DeltaY` are the scroll deltas in pixels
315
+ * (positive `DeltaY` = down, positive `DeltaX` = right — matching the DOM `WheelEvent` convention).
316
+ */
317
+ export interface RemoteBrowserScrollInput {
318
+ Kind: 'scroll';
319
+ /** Viewport X coordinate the scroll occurred over. */
320
+ X: number;
321
+ /** Viewport Y coordinate the scroll occurred over. */
322
+ Y: number;
323
+ /** Horizontal scroll delta in pixels (positive = right). */
324
+ DeltaX: number;
325
+ /** Vertical scroll delta in pixels (positive = down). */
326
+ DeltaY: number;
327
+ }
328
+ /**
329
+ * The full human-takeover input vocabulary — a discriminated union over `Kind`. When a human "grabs
330
+ * the wheel" in `Collaborative` (or watches in `ViewOnly`) their pointer/keyboard/scroll events arrive
331
+ * as these and route into the backend browser via {@link IRemoteBrowserSession.RouteHumanInput}.
332
+ * Strongly typed with no `any`; narrow on `input.Kind`.
333
+ */
334
+ export type RemoteBrowserHumanInput = RemoteBrowserPointerMoveInput | RemoteBrowserPointerClickInput | RemoteBrowserPointerDownInput | RemoteBrowserPointerUpInput | RemoteBrowserKeyInput | RemoteBrowserTextInput | RemoteBrowserScrollInput;
335
+ /**
336
+ * A live handle to a single remote-browser session, returned by
337
+ * {@link import('./base-remote-browser-provider.js').BaseRemoteBrowserProvider.Connect}.
338
+ *
339
+ * The **core** methods (`GetCdpEndpoint`, `Navigate`, `ExecuteAction`, `CaptureScreenshot`,
340
+ * `GetCurrentUrl`, `Close`) rest on the universal CDP substrate every backend provides and are always
341
+ * available. The remaining methods are **capability-gated** (feature-gated, documented per-method):
342
+ * the engine checks the provider's `SupportedFeatures` flag before calling them, and a backend that
343
+ * cannot satisfy a feature should reject/throw {@link
344
+ * import('./capability-errors.js').RemoteBrowserCapabilityNotSupportedError}.
345
+ *
346
+ * This is a pure interface — the concrete implementation lives in the server package / drivers (which
347
+ * own the Playwright + CDP machinery).
348
+ */
349
+ export interface IRemoteBrowserSession {
350
+ /**
351
+ * Returns the Chrome DevTools Protocol endpoint for this session — the one primitive every
352
+ * backend exposes and the engine's computer-use loop connects to.
353
+ *
354
+ * @returns The CDP websocket/connect endpoint URL.
355
+ */
356
+ GetCdpEndpoint(): string;
357
+ /**
358
+ * Navigates the browser to a URL. Convenience over {@link IRemoteBrowserSession.ExecuteAction}
359
+ * with a `navigate` action.
360
+ *
361
+ * @param url The absolute URL to load.
362
+ * @returns The action result (success + resulting URL).
363
+ */
364
+ Navigate(url: string): Promise<RemoteBrowserActionResult>;
365
+ /**
366
+ * Executes a single agent {@link RemoteBrowserAction} over CDP.
367
+ *
368
+ * @param action The action to perform; narrow on `action.Kind`.
369
+ * @returns The action result.
370
+ */
371
+ ExecuteAction(action: RemoteBrowserAction): Promise<RemoteBrowserActionResult>;
372
+ /**
373
+ * Captures a one-off screenshot of the current viewport.
374
+ *
375
+ * @returns The screenshot image, Base64-encoded.
376
+ */
377
+ CaptureScreenshot(): Promise<string>;
378
+ /**
379
+ * Returns the browser's current URL synchronously (last known to the session).
380
+ *
381
+ * @returns The current page URL.
382
+ */
383
+ GetCurrentUrl(): string;
384
+ /**
385
+ * Closes the session and releases the backend browser/container. Idempotent — safe to call once
386
+ * teardown has already begun.
387
+ *
388
+ * @returns A promise that resolves once the session is fully torn down.
389
+ */
390
+ Close(): Promise<void>;
391
+ /**
392
+ * Returns an embeddable live-view URL so humans can watch the browser without MJ encoding frames.
393
+ *
394
+ * **Capability-gated** by `LiveView`. Backends without it reject with {@link
395
+ * import('./capability-errors.js').RemoteBrowserCapabilityNotSupportedError}.
396
+ *
397
+ * @returns The live-view URL.
398
+ */
399
+ GetLiveViewUrl(): Promise<string>;
400
+ /**
401
+ * Begins streaming encoded viewport frames to `onFrame` (the source for the channel's ScreenOut
402
+ * track).
403
+ *
404
+ * **Capability-gated** by `ScreenStreaming`. Backends without it reject.
405
+ *
406
+ * @param onFrame Invoked with each encoded {@link RemoteBrowserScreencastFrame}.
407
+ * @returns A promise that resolves once the screencast has started.
408
+ */
409
+ StartScreencast(onFrame: (frame: RemoteBrowserScreencastFrame) => void): Promise<void>;
410
+ /**
411
+ * Stops a screencast previously started with {@link IRemoteBrowserSession.StartScreencast}.
412
+ *
413
+ * **Capability-gated** by `ScreenStreaming`. Backends without it reject.
414
+ *
415
+ * @returns A promise that resolves once streaming has stopped.
416
+ */
417
+ StopScreencast(): Promise<void>;
418
+ /**
419
+ * Begins streaming the remote browser's tab audio to `onChunk` (the source for the channel's
420
+ * client-side audio player) — so a co-agent demoing a video/audio site is HEARD, not just seen.
421
+ *
422
+ * **Capability-gated by BACKEND IMPLEMENTATION** (v1): a backend without an audio-capture mechanism
423
+ * rejects with {@link import('./capability-errors.js').RemoteBrowserCapabilityNotSupportedError}, exactly
424
+ * like a non-streaming backend rejects {@link IRemoteBrowserSession.StartScreencast}. (A future
425
+ * metadata `AudioStreaming` feature flag is a documented fast-follow; v1 gates by whether the backend
426
+ * provides capture.)
427
+ *
428
+ * @param onChunk Invoked with each encoded {@link RemoteBrowserAudioChunk}.
429
+ * @returns A promise that resolves once the audio stream has started.
430
+ */
431
+ StartAudioStream(onChunk: (chunk: RemoteBrowserAudioChunk) => void): Promise<void>;
432
+ /**
433
+ * Stops an audio stream previously started with {@link IRemoteBrowserSession.StartAudioStream}.
434
+ *
435
+ * **Capability-gated by BACKEND IMPLEMENTATION** (v1). Backends without audio capture reject.
436
+ *
437
+ * @returns A promise that resolves once streaming has stopped.
438
+ */
439
+ StopAudioStream(): Promise<void>;
440
+ /**
441
+ * Routes a human takeover input (pointer move/click, key, scroll) into the backend browser.
442
+ *
443
+ * **Capability-gated** by `HumanTakeover` — only valid in `Collaborative` (and pointer-only
444
+ * observation in `ViewOnly`). Backends without it throw.
445
+ *
446
+ * @param input The human input event; narrow on `input.Kind`.
447
+ */
448
+ RouteHumanInput(input: RemoteBrowserHumanInput): void;
449
+ /**
450
+ * Reads the remote page's CURRENT text selection — the copy-out half of human clipboard support. The
451
+ * viewer captures a local `copy` / Cmd+C, calls this to read what the human has selected on the remote
452
+ * page, and writes the returned text to the LOCAL clipboard (again sidestepping the isolated remote
453
+ * clipboard, the mirror of the {@link RemoteBrowserTextInput} paste path).
454
+ *
455
+ * **Capability-gated** by `HumanTakeover` (copy-out belongs to the human-control path). Degrades
456
+ * gracefully: backends that cannot read the selection — or a page with nothing selected — resolve to
457
+ * `''` rather than throwing, so a best-effort copy never breaks the live view.
458
+ *
459
+ * @returns The selected text, or `''` when nothing is selected / the selection can't be read.
460
+ */
461
+ GetSelectionText(): Promise<string>;
462
+ /**
463
+ * Delegates a high-level natural-language intent to the backend's own AI-control harness (e.g.
464
+ * Browserbase Stagehand) instead of MJ's computer-use loop.
465
+ *
466
+ * **Capability-gated** by `NativeAIControl`. Backends without it reject.
467
+ *
468
+ * @param intent The natural-language intent (e.g. `'log in with the test account'`).
469
+ * @returns The action result.
470
+ */
471
+ InvokeNativeAIControl(intent: string): Promise<RemoteBrowserActionResult>;
472
+ /**
473
+ * Runs an autonomous, goal-driven browser loop (MJ's computer-use) against THIS session's live
474
+ * browser — the agent sets a high-level goal ("log in and download the latest invoice") and the
475
+ * computer-use vision/action model plans + executes it, instead of the caller issuing granular
476
+ * actions. The CDP layer hands its own (already-attached) `PlaywrightBrowserAdapter` to the engine,
477
+ * so the loop drives the very page the human is watching (no second browser/CDP attach).
478
+ *
479
+ * This is the default `'ComputerUse'` control strategy; the `'NativeAI'` strategy uses
480
+ * {@link IRemoteBrowserSession.InvokeNativeAIControl} instead. {@link resolveControlStrategy} picks.
481
+ *
482
+ * @param goal The natural-language goal.
483
+ * @param options Optional start URL, step cap, model overrides, and model-blind context.
484
+ * @returns The goal outcome (success, status, step count, final URL).
485
+ */
486
+ RunComputerUseGoal(goal: string, options?: RunComputerUseGoalOptions): Promise<RemoteBrowserGoalResult>;
487
+ }
488
+ //# sourceMappingURL=remote-browser-session.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"remote-browser-session.d.ts","sourceRoot":"","sources":["../src/remote-browser-session.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AACrD,OAAO,KAAK,EAAE,4BAA4B,EAAE,MAAM,WAAW,CAAC;AAM9D;;GAEG;AACH,MAAM,WAAW,2BAA2B;IAC1C,IAAI,EAAE,UAAU,CAAC;IACjB,gCAAgC;IAChC,GAAG,EAAE,MAAM,CAAC;CACb;AAED;;;GAGG;AACH,MAAM,WAAW,wBAAwB;IACvC,IAAI,EAAE,OAAO,CAAC;IACd,8DAA8D;IAC9D,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,yEAAyE;IACzE,CAAC,CAAC,EAAE,MAAM,CAAC;IACX,yEAAyE;IACzE,CAAC,CAAC,EAAE,MAAM,CAAC;CACZ;AAED;;;GAGG;AACH,MAAM,WAAW,uBAAuB;IACtC,IAAI,EAAE,MAAM,CAAC;IACb,wBAAwB;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,oDAAoD;IACpD,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACrC,IAAI,EAAE,KAAK,CAAC;IACZ,uEAAuE;IACvE,GAAG,EAAE,MAAM,CAAC;CACb;AAED;;;;GAIG;AACH,MAAM,WAAW,yBAAyB;IACxC,IAAI,EAAE,QAAQ,CAAC;IACf,+DAA+D;IAC/D,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,4DAA4D;IAC5D,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,4EAA4E;IAC5E,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,uBAAuB;IACtC,IAAI,EAAE,MAAM,CAAC;CACd;AAED;;GAEG;AACH,MAAM,WAAW,0BAA0B;IACzC,IAAI,EAAE,SAAS,CAAC;CACjB;AAED;;;GAGG;AACH,MAAM,WAAW,uBAAuB;IACtC,IAAI,EAAE,MAAM,CAAC;IACb,kCAAkC;IAClC,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,qEAAqE;IACrE,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;;;GAIG;AACH,MAAM,MAAM,mBAAmB,GAC3B,2BAA2B,GAC3B,wBAAwB,GACxB,uBAAuB,GACvB,sBAAsB,GACtB,yBAAyB,GACzB,uBAAuB,GACvB,0BAA0B,GAC1B,uBAAuB,CAAC;AAE5B;;GAEG;AACH,MAAM,WAAW,yBAAyB;IACxC,iDAAiD;IACjD,OAAO,EAAE,OAAO,CAAC;IACjB,iDAAiD;IACjD,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,uFAAuF;IACvF,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,+FAA+F;AAC/F,MAAM,WAAW,yBAAyB;IACxC,2BAA2B;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,yFAAyF;IACzF,OAAO,EAAE,MAAM,CAAC;IAChB,6CAA6C;IAC7C,GAAG,CAAC,EAAE,MAAM,CAAC;CACd;AAED;;;;GAIG;AACH,MAAM,WAAW,yBAAyB;IACxC,+DAA+D;IAC/D,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,2DAA2D;IAC3D,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,iGAAiG;IACjG,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,4EAA4E;IAC5E,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB;;;;OAIG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAClC,2FAA2F;IAC3F,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,yBAAyB,KAAK,IAAI,CAAC;IAC3D,iFAAiF;IACjF,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB;;;;;OAKG;IACH,WAAW,CAAC,EAAE,QAAQ,CAAC;IAEvB;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB;;;;OAIG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED;;;;GAIG;AACH,MAAM,WAAW,uBAAuB;IACtC,qCAAqC;IACrC,OAAO,EAAE,OAAO,CAAC;IACjB,gDAAgD;IAChD,QAAQ,CAAC,EAAE,4BAA4B,CAAC;IACxC,mDAAmD;IACnD,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,oGAAoG;IACpG,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,qEAAqE;IACrE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,8DAA8D;IAC9D,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAMD;;;;GAIG;AACH,MAAM,WAAW,4BAA4B;IAC3C,4EAA4E;IAC5E,UAAU,EAAE,MAAM,CAAC;IACnB,6BAA6B;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,8BAA8B;IAC9B,MAAM,EAAE,MAAM,CAAC;IACf,8EAA8E;IAC9E,cAAc,EAAE,MAAM,CAAC;CACxB;AAMD;;;;;;;;;GASG;AACH,MAAM,WAAW,uBAAuB;IACtC,kEAAkE;IAClE,UAAU,EAAE,MAAM,CAAC;IACnB,mGAAmG;IACnG,KAAK,EAAE,WAAW,GAAG,MAAM,GAAG,OAAO,CAAC;IACtC,6DAA6D;IAC7D,UAAU,EAAE,MAAM,CAAC;IACnB,4CAA4C;IAC5C,QAAQ,EAAE,MAAM,CAAC;IACjB,uFAAuF;IACvF,cAAc,EAAE,MAAM,CAAC;IACvB,sEAAsE;IACtE,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAMD;;;;;;GAMG;AACH,MAAM,MAAM,wBAAwB,GAAG,OAAO,GAAG,SAAS,GAAG,KAAK,GAAG,MAAM,CAAC;AAE5E;;GAEG;AACH,MAAM,WAAW,6BAA6B;IAC5C,IAAI,EAAE,cAAc,CAAC;IACrB,6BAA6B;IAC7B,CAAC,EAAE,MAAM,CAAC;IACV,6BAA6B;IAC7B,CAAC,EAAE,MAAM,CAAC;CACX;AAED;;GAEG;AACH,MAAM,WAAW,8BAA8B;IAC7C,IAAI,EAAE,eAAe,CAAC;IACtB,6BAA6B;IAC7B,CAAC,EAAE,MAAM,CAAC;IACV,6BAA6B;IAC7B,CAAC,EAAE,MAAM,CAAC;IACV,uEAAuE;IACvE,MAAM,CAAC,EAAE,MAAM,GAAG,QAAQ,GAAG,OAAO,CAAC;IACrC;;;OAGG;IACH,SAAS,CAAC,EAAE,wBAAwB,EAAE,CAAC;CACxC;AAED;;;;GAIG;AACH,MAAM,WAAW,6BAA6B;IAC5C,IAAI,EAAE,cAAc,CAAC;IACrB,6BAA6B;IAC7B,CAAC,EAAE,MAAM,CAAC;IACV,6BAA6B;IAC7B,CAAC,EAAE,MAAM,CAAC;IACV,0EAA0E;IAC1E,MAAM,CAAC,EAAE,MAAM,GAAG,QAAQ,GAAG,OAAO,CAAC;IACrC,+EAA+E;IAC/E,SAAS,CAAC,EAAE,wBAAwB,EAAE,CAAC;CACxC;AAED;;;GAGG;AACH,MAAM,WAAW,2BAA2B;IAC1C,IAAI,EAAE,YAAY,CAAC;IACnB,6BAA6B;IAC7B,CAAC,EAAE,MAAM,CAAC;IACV,6BAA6B;IAC7B,CAAC,EAAE,MAAM,CAAC;IACV,2EAA2E;IAC3E,MAAM,CAAC,EAAE,MAAM,GAAG,QAAQ,GAAG,OAAO,CAAC;IACrC,iFAAiF;IACjF,SAAS,CAAC,EAAE,wBAAwB,EAAE,CAAC;CACxC;AAED;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC,IAAI,EAAE,KAAK,CAAC;IACZ,sEAAsE;IACtE,GAAG,EAAE,MAAM,CAAC;IACZ;;;;OAIG;IACH,SAAS,CAAC,EAAE,wBAAwB,EAAE,CAAC;CACxC;AAED;;;;;GAKG;AACH,MAAM,WAAW,sBAAsB;IACrC,IAAI,EAAE,MAAM,CAAC;IACb,8FAA8F;IAC9F,IAAI,EAAE,MAAM,CAAC;CACd;AAED;;;;;GAKG;AACH,MAAM,WAAW,wBAAwB;IACvC,IAAI,EAAE,QAAQ,CAAC;IACf,sDAAsD;IACtD,CAAC,EAAE,MAAM,CAAC;IACV,sDAAsD;IACtD,CAAC,EAAE,MAAM,CAAC;IACV,4DAA4D;IAC5D,MAAM,EAAE,MAAM,CAAC;IACf,yDAAyD;IACzD,MAAM,EAAE,MAAM,CAAC;CAChB;AAED;;;;;GAKG;AACH,MAAM,MAAM,uBAAuB,GAC/B,6BAA6B,GAC7B,8BAA8B,GAC9B,6BAA6B,GAC7B,2BAA2B,GAC3B,qBAAqB,GACrB,sBAAsB,GACtB,wBAAwB,CAAC;AAM7B;;;;;;;;;;;;;GAaG;AACH,MAAM,WAAW,qBAAqB;IAGpC;;;;;OAKG;IACH,cAAc,IAAI,MAAM,CAAC;IAEzB;;;;;;OAMG;IACH,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,yBAAyB,CAAC,CAAC;IAE1D;;;;;OAKG;IACH,aAAa,CAAC,MAAM,EAAE,mBAAmB,GAAG,OAAO,CAAC,yBAAyB,CAAC,CAAC;IAE/E;;;;OAIG;IACH,iBAAiB,IAAI,OAAO,CAAC,MAAM,CAAC,CAAC;IAErC;;;;OAIG;IACH,aAAa,IAAI,MAAM,CAAC;IAExB;;;;;OAKG;IACH,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAIvB;;;;;;;OAOG;IACH,cAAc,IAAI,OAAO,CAAC,MAAM,CAAC,CAAC;IAElC;;;;;;;;OAQG;IACH,eAAe,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,4BAA4B,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAEvF;;;;;;OAMG;IACH,cAAc,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAEhC;;;;;;;;;;;;OAYG;IACH,gBAAgB,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,uBAAuB,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAEnF;;;;;;OAMG;IACH,eAAe,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAEjC;;;;;;;OAOG;IACH,eAAe,CAAC,KAAK,EAAE,uBAAuB,GAAG,IAAI,CAAC;IAEtD;;;;;;;;;;;OAWG;IACH,gBAAgB,IAAI,OAAO,CAAC,MAAM,CAAC,CAAC;IAEpC;;;;;;;;OAQG;IACH,qBAAqB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,yBAAyB,CAAC,CAAC;IAE1E;;;;;;;;;;;;;OAaG;IACH,kBAAkB,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,yBAAyB,GAAG,OAAO,CAAC,uBAAuB,CAAC,CAAC;CACzG"}
@@ -0,0 +1,15 @@
1
+ /**
2
+ * The live-session contract for the Remote Browser channel and the self-contained, strongly-typed
3
+ * action / input / output payloads it carries.
4
+ *
5
+ * These types are defined **self-contained here** (no dependency on `@memberjunction/computer-use` or
6
+ * Playwright) so this Base package stays universal — usable client *and* server. The server package
7
+ * and the backend drivers (which depend on `@memberjunction/computer-use`) *implement*
8
+ * {@link IRemoteBrowserSession} against a real CDP-connected browser; this package only declares the
9
+ * shape both sides agree on.
10
+ *
11
+ * @see `/plans/realtime/realtime-bridges-architecture.md` §4d (the Remote Browser channel) and §4d-i
12
+ * (the one-primitive-CDP build decision).
13
+ */
14
+ export {};
15
+ //# sourceMappingURL=remote-browser-session.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"remote-browser-session.js","sourceRoot":"","sources":["../src/remote-browser-session.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG"}
package/package.json CHANGED
@@ -1,10 +1,35 @@
1
1
  {
2
2
  "name": "@memberjunction/remote-browser-base",
3
- "version": "0.0.1",
4
- "description": "OIDC trusted publishing setup package for @memberjunction/remote-browser-base",
5
- "keywords": [
6
- "oidc",
7
- "trusted-publishing",
8
- "setup"
9
- ]
3
+ "type": "module",
4
+ "version": "5.42.0",
5
+ "description": "MemberJunction: Universal (client+server) base layer for the Remote Browser channel — the provider registry cache, the capability-gated BaseRemoteBrowserProvider driver contract, the CDP-backed live-session interface, and the control mode/strategy helpers that the server engine and backend drivers build on.",
6
+ "main": "dist/index.js",
7
+ "types": "dist/index.d.ts",
8
+ "files": [
9
+ "/dist"
10
+ ],
11
+ "scripts": {
12
+ "start": "ts-node-dev src/index.ts",
13
+ "build": "tsc && tsc-alias -f",
14
+ "test": "vitest run",
15
+ "test:watch": "vitest"
16
+ },
17
+ "author": "MemberJunction.com",
18
+ "license": "ISC",
19
+ "dependencies": {
20
+ "@memberjunction/core": "5.42.0",
21
+ "@memberjunction/global": "5.42.0",
22
+ "@memberjunction/core-entities": "5.42.0",
23
+ "rxjs": "^7.8.2"
24
+ },
25
+ "devDependencies": {
26
+ "@types/node": "24.10.11",
27
+ "ts-node-dev": "^2.0.0",
28
+ "typescript": "^5.9.3",
29
+ "vitest": "^4.0.18"
30
+ },
31
+ "repository": {
32
+ "type": "git",
33
+ "url": "https://github.com/MemberJunction/MJ"
34
+ }
10
35
  }