@opengeni/codemode 0.6.1 → 0.6.2-canary.36079201993001

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,7 +1,8 @@
1
- import type { AttachedBrowserDevice, AuthRun, AuthRunListResponse, AuthRunMutationResponse, BrowserAction, BrowserActionReceipt, BrowserClipboard, BrowserDiagnosticBatch, BrowserDiagnosticKind, BrowserIdentity, BrowserIdentityListResponse, BrowserIdentityMutationResponse, BrowserLocator, BrowserObservation, BrowserPermission, BrowserPermissionSetting, BrowserRevisionListResponse, BrowserSession, BrowserSessionMutationResponse, BrowserTargetListResponse, ComputerAction, ComputerActionReceipt, ComputerClipboard, ComputerLocator, ComputerObservation, ComputerSession, ComputerSessionMutationResponse, ComputerTargetListResponse, ExternalAuthRunRequest, ExternalAuthRunResponse, InteractionPlacement, ProtectedAuthFillRequest, ProtectedAuthFillResponse, PublishBrowserRevisionResponse, ReportAuthRunRequest, RequestHumanInteractionToolInput, RequestHumanInteractionToolOutput, SiteAuthConnection, SiteAuthConnectionListResponse, StartAuthRunRequest, UpdateBrowserIdentityRequest, VerifyAuthRunRequest } from "@opengeni/contracts";
1
+ import type { AttachedBrowserDevice, AuthRun, AuthRunListResponse, AuthRunMutationResponse, BrowserAction, BrowserActionReceipt, BrowserClipboard, BrowserDiagnosticBatch, BrowserDiagnosticKind, BrowserDomReadLocator, BrowserDomReadResponse, BrowserDomSafeAttribute, BrowserIdentity, BrowserIdentityListResponse, BrowserIdentityMutationResponse, BrowserLocator, BrowserObservation, BrowserPermission, BrowserPermissionSetting, BrowserRevisionListResponse, BrowserSession, BrowserSessionMutationResponse, BrowserTargetListResponse, ComputerAction, ComputerActionReceipt, ComputerClipboard, ComputerLocator, ComputerObservation, ComputerSession, ComputerSessionMutationResponse, ComputerTargetListResponse, ExternalAuthRunRequest, ExternalAuthRunResponse, InteractionPlacement, ProtectedAuthFillRequest, ProtectedAuthFillResponse, PublishBrowserRevisionResponse, ReportAuthRunRequest, RequestHumanInteractionToolInput, RequestHumanInteractionToolOutput, SiteAuthConnection, SiteAuthConnectionListResponse, StartAuthRunRequest, UpdateBrowserIdentityRequest, VerifyAuthRunRequest } from "@opengeni/contracts";
2
2
  import type { CodemodeCallOptions, CodemodeClient } from "./index.js";
3
3
  import { type CodemodeClientProvider } from "./environment.js";
4
4
  import { CodemodeArtifactCollection } from "./artifacts.js";
5
+ import { type CodemodeLocalImage } from "./images.js";
5
6
  export { CodemodeToolExecutionError } from "./structured.js";
6
7
  export type InteractionDiscovery = {
7
8
  browserRevision: number;
@@ -35,6 +36,111 @@ export type BrowserActionFences = {
35
36
  expectedDocumentGeneration?: string | null | undefined;
36
37
  expectedFrameId?: string | null | undefined;
37
38
  };
39
+ export type CodemodeBrowserScreenshotOptions = {
40
+ /** Capture the entire scrollable page when supported by the browser. */
41
+ fullPage?: boolean | undefined;
42
+ /** JPEG quality, 1–100. Reduce it if a large image exceeds the journal bound. */
43
+ quality?: number | undefined;
44
+ /** Save the still at this exact path. Existing files are never overwritten. */
45
+ saveTo?: string | undefined;
46
+ };
47
+ export type CodemodeBrowserScreenshotMetadata = Readonly<{
48
+ kind: "browser_screenshot";
49
+ browserSessionId: string;
50
+ targetId: string;
51
+ frameId: string;
52
+ targetGeneration: string;
53
+ documentGeneration: string;
54
+ mediaType: "image/png" | "image/jpeg";
55
+ width: number;
56
+ height: number;
57
+ deviceScaleFactor: number;
58
+ scrollX: number;
59
+ scrollY: number;
60
+ capturedAt: string;
61
+ fullPage: boolean;
62
+ }>;
63
+ export type CodemodeBrowserScreenshot = CodemodeLocalImage & Readonly<{
64
+ /** Small structured frame metadata; pixel bytes stay in the local image file. */
65
+ metadata: CodemodeBrowserScreenshotMetadata;
66
+ }>;
67
+ export type CodemodeBrowserAgentNode = Readonly<{
68
+ ref: string;
69
+ role: string;
70
+ depth: number;
71
+ name?: string;
72
+ description?: string;
73
+ value?: string | {
74
+ redacted: true;
75
+ reason: string;
76
+ };
77
+ states: readonly string[];
78
+ actions: readonly string[];
79
+ }>;
80
+ export type CodemodeBrowserAgentView = Readonly<{
81
+ kind: "compact";
82
+ sourceKind: "snapshot" | "diff" | "none";
83
+ nodes: readonly CodemodeBrowserAgentNode[];
84
+ sourceNodeCount: number;
85
+ omittedNodeCount: number;
86
+ removedRefCount: number;
87
+ clippedFieldCount: number;
88
+ maxNodes: number;
89
+ maxNodeBytes: number;
90
+ }>;
91
+ export type CodemodeBrowserAgentObservation = BrowserObservation & {
92
+ /** Present on the compact default; absent when the full tree was requested. */
93
+ agentView?: CodemodeBrowserAgentView;
94
+ };
95
+ export type CodemodeBrowserActionReceipt = Omit<BrowserActionReceipt, "observation"> & {
96
+ observation: CodemodeBrowserAgentObservation | null;
97
+ };
98
+ export type CodemodeBrowserActionOptions = BrowserActionFences & {
99
+ view?: "compact" | "full" | "none" | undefined;
100
+ };
101
+ export type CodemodeBrowserAxReadOptions = Readonly<{
102
+ mode?: "matches" | "count" | "subtree";
103
+ ref?: string;
104
+ scopeRef?: string;
105
+ role?: string;
106
+ nameContains?: string;
107
+ textContains?: string;
108
+ state?: string;
109
+ action?: string;
110
+ limit?: number;
111
+ }>;
112
+ export type CodemodeBrowserDomReadOptions = BrowserActionFences & Readonly<{
113
+ mode: "dom";
114
+ dom: {
115
+ kind: "element";
116
+ locator: BrowserDomReadLocator;
117
+ attributes?: readonly BrowserDomSafeAttribute[];
118
+ maxChars?: number;
119
+ } | {
120
+ kind: "count";
121
+ selector: string;
122
+ };
123
+ }>;
124
+ export type CodemodeBrowserReadOptions = CodemodeBrowserAxReadOptions | CodemodeBrowserDomReadOptions;
125
+ export type CodemodeBrowserAxReadResult = Readonly<{
126
+ browserSessionId: string;
127
+ targetId: string;
128
+ observationId: string;
129
+ targetGeneration: string;
130
+ documentGeneration: string | null;
131
+ frameId: string | null;
132
+ source: "accessibility";
133
+ mode: "matches" | "count" | "subtree";
134
+ scopeFound: boolean;
135
+ nodes: readonly CodemodeBrowserAgentNode[];
136
+ totalMatches: number;
137
+ omittedMatches: number;
138
+ clippedFieldCount: number;
139
+ maxNodeBytes: number;
140
+ }>;
141
+ export type CodemodeBrowserReadResult = CodemodeBrowserAxReadResult | (BrowserDomReadResponse & {
142
+ source: "dom";
143
+ });
38
144
  export type ComputerActionFences = {
39
145
  expectedTargetGeneration?: string | undefined;
40
146
  expectedObservationId?: string | null | undefined;
@@ -83,10 +189,17 @@ export declare class CodemodeBrowser {
83
189
  readonly clipboard: CodemodeBrowserClipboard;
84
190
  constructor(client: CodemodeClientProvider, id: string);
85
191
  refresh(callOptions?: CodemodeCallOptions): Promise<BrowserSession>;
86
- observe(targetId?: string, callOptions?: CodemodeCallOptions): Promise<BrowserObservation>;
87
- act(action: BrowserAction, options?: BrowserActionFences & {
192
+ observe(targetId?: string, callOptions?: CodemodeCallOptions): Promise<CodemodeBrowserAgentObservation>;
193
+ observeFull(targetId?: string, callOptions?: CodemodeCallOptions): Promise<BrowserObservation>;
194
+ read(options?: CodemodeBrowserReadOptions & {
88
195
  targetId?: string | undefined;
89
- }, callOptions?: CodemodeCallOptions): Promise<BrowserActionReceipt>;
196
+ }, callOptions?: CodemodeCallOptions): Promise<CodemodeBrowserReadResult>;
197
+ screenshot(options?: CodemodeBrowserScreenshotOptions & {
198
+ targetId?: string | undefined;
199
+ }, callOptions?: CodemodeCallOptions): Promise<CodemodeBrowserScreenshot>;
200
+ act(action: BrowserAction, options?: CodemodeBrowserActionOptions & {
201
+ targetId?: string | undefined;
202
+ }, callOptions?: CodemodeCallOptions): Promise<CodemodeBrowserActionReceipt>;
90
203
  diagnostics(targetId?: string, options?: {
91
204
  kinds?: BrowserDiagnosticKind[] | undefined;
92
205
  after?: number | undefined;
@@ -143,99 +256,12 @@ export declare class CodemodeBrowserTab {
143
256
  readonly browserSessionId: string;
144
257
  readonly id: string;
145
258
  constructor(client: CodemodeClientProvider, browserSessionId: string, id: string);
146
- observe(callOptions?: CodemodeCallOptions): Promise<BrowserObservation>;
147
- act(action: BrowserAction, fences?: BrowserActionFences, callOptions?: CodemodeCallOptions): Promise<BrowserActionReceipt>;
148
- navigate(url: string, callOptions?: CodemodeCallOptions): Promise<{
149
- protocolVersion: 1;
150
- operationId: string;
151
- browserSessionId: string;
152
- controllerGeneration: string;
153
- targetId: string;
154
- state: "completed" | "dispatched" | "failed" | "outcome_unknown" | "prepared";
155
- dispatchedAt: string | null;
156
- settledAt: string | null;
157
- observation: {
158
- protocolVersion: 1;
159
- observationId: string;
160
- browserSessionId: string;
161
- target: {
162
- id: string;
163
- browserSessionId: string;
164
- controllerGeneration: string;
165
- targetGeneration: string;
166
- documentGeneration: string | null;
167
- kind: "background_page" | "page" | "popup" | "worker";
168
- title: string;
169
- url: string;
170
- selected: boolean;
171
- attached: boolean;
172
- createdAt: string;
173
- };
174
- frameId: string | null;
175
- semantic: {
176
- kind: "snapshot";
177
- roots: import("@opengeni/contracts").InteractionSemanticNodeValue[];
178
- nodeCount: number;
179
- } | {
180
- kind: "diff";
181
- baseObservationId: string;
182
- removedRefs: string[];
183
- changed: import("@opengeni/contracts").InteractionSemanticNodeValue[];
184
- } | null;
185
- screenshot: {
186
- available: true;
187
- artifactId: string;
188
- kind: "assistant_completion" | "computer_screenshot" | "event_media" | "file" | "generated_image" | "generated_video" | "internal_update" | "tool_result";
189
- contentType: string;
190
- originalBytes: number;
191
- sha256: string;
192
- retainedAt: string;
193
- dimensions?: {
194
- width: number;
195
- height: number;
196
- } | undefined;
197
- retention: {
198
- policy: "workspace_file";
199
- expiresAt: null;
200
- } | {
201
- policy: "session_screenshot";
202
- expiresAt: string;
203
- };
204
- retrieval: {
205
- method: "GET";
206
- path: string;
207
- acceptRanges: "bytes";
208
- maxRangeBytes: number;
209
- };
210
- } | null;
211
- focusedRef: string | null;
212
- changedRegions: {
213
- x: number;
214
- y: number;
215
- width: number;
216
- height: number;
217
- }[];
218
- diagnostics: {
219
- consoleErrorCount: number;
220
- failedRequestCount: number;
221
- downloadCount: number;
222
- pageErrorCount: number;
223
- };
224
- dialog: {
225
- type: "alert" | "beforeunload" | "confirm" | "prompt";
226
- message: string;
227
- defaultPrompt: string;
228
- openedAt: string;
229
- } | null;
230
- observedAt: string;
231
- } | null;
232
- error: {
233
- code: "attempt_stale" | "controller_lost" | "controller_stale" | "document_stale" | "driver_failed" | "frame_stale" | "invalid_action" | "locator_ambiguous" | "locator_not_found" | "machine_locked" | "observation_stale" | "operation_conflict" | "outcome_unknown" | "permission_denied" | "resource_not_found" | "resource_unavailable" | "target_not_found" | "target_stale" | "timeout" | "unsupported";
234
- message: string;
235
- retryable: boolean;
236
- details?: Record<string, import("zod").JSONType> | undefined;
237
- } | null;
238
- }>;
259
+ observe(callOptions?: CodemodeCallOptions): Promise<CodemodeBrowserAgentObservation>;
260
+ observeFull(callOptions?: CodemodeCallOptions): Promise<BrowserObservation>;
261
+ read(options?: CodemodeBrowserReadOptions, callOptions?: CodemodeCallOptions): Promise<CodemodeBrowserReadResult>;
262
+ screenshot(options?: CodemodeBrowserScreenshotOptions, callOptions?: CodemodeCallOptions): Promise<CodemodeBrowserScreenshot>;
263
+ act(action: BrowserAction, fences?: CodemodeBrowserActionOptions, callOptions?: CodemodeCallOptions): Promise<CodemodeBrowserActionReceipt>;
264
+ navigate(url: string, callOptions?: CodemodeCallOptions): Promise<CodemodeBrowserActionReceipt>;
239
265
  setPermission(permission: BrowserPermission, setting: BrowserPermissionSetting, fences?: BrowserActionFences, callOptions?: CodemodeCallOptions): Promise<BrowserActionReceipt>;
240
266
  requestHuman(reason: string, options?: {
241
267
  kind?: "manual_login" | "mfa" | "external_action" | "confirmation" | "other";
@@ -259,734 +285,14 @@ export declare class CodemodeBrowserLocator {
259
285
  constructor(tab: CodemodeBrowserTab, locator: BrowserLocator);
260
286
  click(options?: {
261
287
  button?: "left" | "right" | "middle";
262
- }, callOptions?: CodemodeCallOptions): Promise<{
263
- protocolVersion: 1;
264
- operationId: string;
265
- browserSessionId: string;
266
- controllerGeneration: string;
267
- targetId: string;
268
- state: "completed" | "dispatched" | "failed" | "outcome_unknown" | "prepared";
269
- dispatchedAt: string | null;
270
- settledAt: string | null;
271
- observation: {
272
- protocolVersion: 1;
273
- observationId: string;
274
- browserSessionId: string;
275
- target: {
276
- id: string;
277
- browserSessionId: string;
278
- controllerGeneration: string;
279
- targetGeneration: string;
280
- documentGeneration: string | null;
281
- kind: "background_page" | "page" | "popup" | "worker";
282
- title: string;
283
- url: string;
284
- selected: boolean;
285
- attached: boolean;
286
- createdAt: string;
287
- };
288
- frameId: string | null;
289
- semantic: {
290
- kind: "snapshot";
291
- roots: import("@opengeni/contracts").InteractionSemanticNodeValue[];
292
- nodeCount: number;
293
- } | {
294
- kind: "diff";
295
- baseObservationId: string;
296
- removedRefs: string[];
297
- changed: import("@opengeni/contracts").InteractionSemanticNodeValue[];
298
- } | null;
299
- screenshot: {
300
- available: true;
301
- artifactId: string;
302
- kind: "assistant_completion" | "computer_screenshot" | "event_media" | "file" | "generated_image" | "generated_video" | "internal_update" | "tool_result";
303
- contentType: string;
304
- originalBytes: number;
305
- sha256: string;
306
- retainedAt: string;
307
- dimensions?: {
308
- width: number;
309
- height: number;
310
- } | undefined;
311
- retention: {
312
- policy: "workspace_file";
313
- expiresAt: null;
314
- } | {
315
- policy: "session_screenshot";
316
- expiresAt: string;
317
- };
318
- retrieval: {
319
- method: "GET";
320
- path: string;
321
- acceptRanges: "bytes";
322
- maxRangeBytes: number;
323
- };
324
- } | null;
325
- focusedRef: string | null;
326
- changedRegions: {
327
- x: number;
328
- y: number;
329
- width: number;
330
- height: number;
331
- }[];
332
- diagnostics: {
333
- consoleErrorCount: number;
334
- failedRequestCount: number;
335
- downloadCount: number;
336
- pageErrorCount: number;
337
- };
338
- dialog: {
339
- type: "alert" | "beforeunload" | "confirm" | "prompt";
340
- message: string;
341
- defaultPrompt: string;
342
- openedAt: string;
343
- } | null;
344
- observedAt: string;
345
- } | null;
346
- error: {
347
- code: "attempt_stale" | "controller_lost" | "controller_stale" | "document_stale" | "driver_failed" | "frame_stale" | "invalid_action" | "locator_ambiguous" | "locator_not_found" | "machine_locked" | "observation_stale" | "operation_conflict" | "outcome_unknown" | "permission_denied" | "resource_not_found" | "resource_unavailable" | "target_not_found" | "target_stale" | "timeout" | "unsupported";
348
- message: string;
349
- retryable: boolean;
350
- details?: Record<string, import("zod").JSONType> | undefined;
351
- } | null;
352
- }>;
353
- doubleClick(callOptions?: CodemodeCallOptions): Promise<{
354
- protocolVersion: 1;
355
- operationId: string;
356
- browserSessionId: string;
357
- controllerGeneration: string;
358
- targetId: string;
359
- state: "completed" | "dispatched" | "failed" | "outcome_unknown" | "prepared";
360
- dispatchedAt: string | null;
361
- settledAt: string | null;
362
- observation: {
363
- protocolVersion: 1;
364
- observationId: string;
365
- browserSessionId: string;
366
- target: {
367
- id: string;
368
- browserSessionId: string;
369
- controllerGeneration: string;
370
- targetGeneration: string;
371
- documentGeneration: string | null;
372
- kind: "background_page" | "page" | "popup" | "worker";
373
- title: string;
374
- url: string;
375
- selected: boolean;
376
- attached: boolean;
377
- createdAt: string;
378
- };
379
- frameId: string | null;
380
- semantic: {
381
- kind: "snapshot";
382
- roots: import("@opengeni/contracts").InteractionSemanticNodeValue[];
383
- nodeCount: number;
384
- } | {
385
- kind: "diff";
386
- baseObservationId: string;
387
- removedRefs: string[];
388
- changed: import("@opengeni/contracts").InteractionSemanticNodeValue[];
389
- } | null;
390
- screenshot: {
391
- available: true;
392
- artifactId: string;
393
- kind: "assistant_completion" | "computer_screenshot" | "event_media" | "file" | "generated_image" | "generated_video" | "internal_update" | "tool_result";
394
- contentType: string;
395
- originalBytes: number;
396
- sha256: string;
397
- retainedAt: string;
398
- dimensions?: {
399
- width: number;
400
- height: number;
401
- } | undefined;
402
- retention: {
403
- policy: "workspace_file";
404
- expiresAt: null;
405
- } | {
406
- policy: "session_screenshot";
407
- expiresAt: string;
408
- };
409
- retrieval: {
410
- method: "GET";
411
- path: string;
412
- acceptRanges: "bytes";
413
- maxRangeBytes: number;
414
- };
415
- } | null;
416
- focusedRef: string | null;
417
- changedRegions: {
418
- x: number;
419
- y: number;
420
- width: number;
421
- height: number;
422
- }[];
423
- diagnostics: {
424
- consoleErrorCount: number;
425
- failedRequestCount: number;
426
- downloadCount: number;
427
- pageErrorCount: number;
428
- };
429
- dialog: {
430
- type: "alert" | "beforeunload" | "confirm" | "prompt";
431
- message: string;
432
- defaultPrompt: string;
433
- openedAt: string;
434
- } | null;
435
- observedAt: string;
436
- } | null;
437
- error: {
438
- code: "attempt_stale" | "controller_lost" | "controller_stale" | "document_stale" | "driver_failed" | "frame_stale" | "invalid_action" | "locator_ambiguous" | "locator_not_found" | "machine_locked" | "observation_stale" | "operation_conflict" | "outcome_unknown" | "permission_denied" | "resource_not_found" | "resource_unavailable" | "target_not_found" | "target_stale" | "timeout" | "unsupported";
439
- message: string;
440
- retryable: boolean;
441
- details?: Record<string, import("zod").JSONType> | undefined;
442
- } | null;
443
- }>;
444
- hover(callOptions?: CodemodeCallOptions): Promise<{
445
- protocolVersion: 1;
446
- operationId: string;
447
- browserSessionId: string;
448
- controllerGeneration: string;
449
- targetId: string;
450
- state: "completed" | "dispatched" | "failed" | "outcome_unknown" | "prepared";
451
- dispatchedAt: string | null;
452
- settledAt: string | null;
453
- observation: {
454
- protocolVersion: 1;
455
- observationId: string;
456
- browserSessionId: string;
457
- target: {
458
- id: string;
459
- browserSessionId: string;
460
- controllerGeneration: string;
461
- targetGeneration: string;
462
- documentGeneration: string | null;
463
- kind: "background_page" | "page" | "popup" | "worker";
464
- title: string;
465
- url: string;
466
- selected: boolean;
467
- attached: boolean;
468
- createdAt: string;
469
- };
470
- frameId: string | null;
471
- semantic: {
472
- kind: "snapshot";
473
- roots: import("@opengeni/contracts").InteractionSemanticNodeValue[];
474
- nodeCount: number;
475
- } | {
476
- kind: "diff";
477
- baseObservationId: string;
478
- removedRefs: string[];
479
- changed: import("@opengeni/contracts").InteractionSemanticNodeValue[];
480
- } | null;
481
- screenshot: {
482
- available: true;
483
- artifactId: string;
484
- kind: "assistant_completion" | "computer_screenshot" | "event_media" | "file" | "generated_image" | "generated_video" | "internal_update" | "tool_result";
485
- contentType: string;
486
- originalBytes: number;
487
- sha256: string;
488
- retainedAt: string;
489
- dimensions?: {
490
- width: number;
491
- height: number;
492
- } | undefined;
493
- retention: {
494
- policy: "workspace_file";
495
- expiresAt: null;
496
- } | {
497
- policy: "session_screenshot";
498
- expiresAt: string;
499
- };
500
- retrieval: {
501
- method: "GET";
502
- path: string;
503
- acceptRanges: "bytes";
504
- maxRangeBytes: number;
505
- };
506
- } | null;
507
- focusedRef: string | null;
508
- changedRegions: {
509
- x: number;
510
- y: number;
511
- width: number;
512
- height: number;
513
- }[];
514
- diagnostics: {
515
- consoleErrorCount: number;
516
- failedRequestCount: number;
517
- downloadCount: number;
518
- pageErrorCount: number;
519
- };
520
- dialog: {
521
- type: "alert" | "beforeunload" | "confirm" | "prompt";
522
- message: string;
523
- defaultPrompt: string;
524
- openedAt: string;
525
- } | null;
526
- observedAt: string;
527
- } | null;
528
- error: {
529
- code: "attempt_stale" | "controller_lost" | "controller_stale" | "document_stale" | "driver_failed" | "frame_stale" | "invalid_action" | "locator_ambiguous" | "locator_not_found" | "machine_locked" | "observation_stale" | "operation_conflict" | "outcome_unknown" | "permission_denied" | "resource_not_found" | "resource_unavailable" | "target_not_found" | "target_stale" | "timeout" | "unsupported";
530
- message: string;
531
- retryable: boolean;
532
- details?: Record<string, import("zod").JSONType> | undefined;
533
- } | null;
534
- }>;
535
- fill(value: string, callOptions?: CodemodeCallOptions): Promise<{
536
- protocolVersion: 1;
537
- operationId: string;
538
- browserSessionId: string;
539
- controllerGeneration: string;
540
- targetId: string;
541
- state: "completed" | "dispatched" | "failed" | "outcome_unknown" | "prepared";
542
- dispatchedAt: string | null;
543
- settledAt: string | null;
544
- observation: {
545
- protocolVersion: 1;
546
- observationId: string;
547
- browserSessionId: string;
548
- target: {
549
- id: string;
550
- browserSessionId: string;
551
- controllerGeneration: string;
552
- targetGeneration: string;
553
- documentGeneration: string | null;
554
- kind: "background_page" | "page" | "popup" | "worker";
555
- title: string;
556
- url: string;
557
- selected: boolean;
558
- attached: boolean;
559
- createdAt: string;
560
- };
561
- frameId: string | null;
562
- semantic: {
563
- kind: "snapshot";
564
- roots: import("@opengeni/contracts").InteractionSemanticNodeValue[];
565
- nodeCount: number;
566
- } | {
567
- kind: "diff";
568
- baseObservationId: string;
569
- removedRefs: string[];
570
- changed: import("@opengeni/contracts").InteractionSemanticNodeValue[];
571
- } | null;
572
- screenshot: {
573
- available: true;
574
- artifactId: string;
575
- kind: "assistant_completion" | "computer_screenshot" | "event_media" | "file" | "generated_image" | "generated_video" | "internal_update" | "tool_result";
576
- contentType: string;
577
- originalBytes: number;
578
- sha256: string;
579
- retainedAt: string;
580
- dimensions?: {
581
- width: number;
582
- height: number;
583
- } | undefined;
584
- retention: {
585
- policy: "workspace_file";
586
- expiresAt: null;
587
- } | {
588
- policy: "session_screenshot";
589
- expiresAt: string;
590
- };
591
- retrieval: {
592
- method: "GET";
593
- path: string;
594
- acceptRanges: "bytes";
595
- maxRangeBytes: number;
596
- };
597
- } | null;
598
- focusedRef: string | null;
599
- changedRegions: {
600
- x: number;
601
- y: number;
602
- width: number;
603
- height: number;
604
- }[];
605
- diagnostics: {
606
- consoleErrorCount: number;
607
- failedRequestCount: number;
608
- downloadCount: number;
609
- pageErrorCount: number;
610
- };
611
- dialog: {
612
- type: "alert" | "beforeunload" | "confirm" | "prompt";
613
- message: string;
614
- defaultPrompt: string;
615
- openedAt: string;
616
- } | null;
617
- observedAt: string;
618
- } | null;
619
- error: {
620
- code: "attempt_stale" | "controller_lost" | "controller_stale" | "document_stale" | "driver_failed" | "frame_stale" | "invalid_action" | "locator_ambiguous" | "locator_not_found" | "machine_locked" | "observation_stale" | "operation_conflict" | "outcome_unknown" | "permission_denied" | "resource_not_found" | "resource_unavailable" | "target_not_found" | "target_stale" | "timeout" | "unsupported";
621
- message: string;
622
- retryable: boolean;
623
- details?: Record<string, import("zod").JSONType> | undefined;
624
- } | null;
625
- }>;
626
- type(text: string, callOptions?: CodemodeCallOptions): Promise<{
627
- protocolVersion: 1;
628
- operationId: string;
629
- browserSessionId: string;
630
- controllerGeneration: string;
631
- targetId: string;
632
- state: "completed" | "dispatched" | "failed" | "outcome_unknown" | "prepared";
633
- dispatchedAt: string | null;
634
- settledAt: string | null;
635
- observation: {
636
- protocolVersion: 1;
637
- observationId: string;
638
- browserSessionId: string;
639
- target: {
640
- id: string;
641
- browserSessionId: string;
642
- controllerGeneration: string;
643
- targetGeneration: string;
644
- documentGeneration: string | null;
645
- kind: "background_page" | "page" | "popup" | "worker";
646
- title: string;
647
- url: string;
648
- selected: boolean;
649
- attached: boolean;
650
- createdAt: string;
651
- };
652
- frameId: string | null;
653
- semantic: {
654
- kind: "snapshot";
655
- roots: import("@opengeni/contracts").InteractionSemanticNodeValue[];
656
- nodeCount: number;
657
- } | {
658
- kind: "diff";
659
- baseObservationId: string;
660
- removedRefs: string[];
661
- changed: import("@opengeni/contracts").InteractionSemanticNodeValue[];
662
- } | null;
663
- screenshot: {
664
- available: true;
665
- artifactId: string;
666
- kind: "assistant_completion" | "computer_screenshot" | "event_media" | "file" | "generated_image" | "generated_video" | "internal_update" | "tool_result";
667
- contentType: string;
668
- originalBytes: number;
669
- sha256: string;
670
- retainedAt: string;
671
- dimensions?: {
672
- width: number;
673
- height: number;
674
- } | undefined;
675
- retention: {
676
- policy: "workspace_file";
677
- expiresAt: null;
678
- } | {
679
- policy: "session_screenshot";
680
- expiresAt: string;
681
- };
682
- retrieval: {
683
- method: "GET";
684
- path: string;
685
- acceptRanges: "bytes";
686
- maxRangeBytes: number;
687
- };
688
- } | null;
689
- focusedRef: string | null;
690
- changedRegions: {
691
- x: number;
692
- y: number;
693
- width: number;
694
- height: number;
695
- }[];
696
- diagnostics: {
697
- consoleErrorCount: number;
698
- failedRequestCount: number;
699
- downloadCount: number;
700
- pageErrorCount: number;
701
- };
702
- dialog: {
703
- type: "alert" | "beforeunload" | "confirm" | "prompt";
704
- message: string;
705
- defaultPrompt: string;
706
- openedAt: string;
707
- } | null;
708
- observedAt: string;
709
- } | null;
710
- error: {
711
- code: "attempt_stale" | "controller_lost" | "controller_stale" | "document_stale" | "driver_failed" | "frame_stale" | "invalid_action" | "locator_ambiguous" | "locator_not_found" | "machine_locked" | "observation_stale" | "operation_conflict" | "outcome_unknown" | "permission_denied" | "resource_not_found" | "resource_unavailable" | "target_not_found" | "target_stale" | "timeout" | "unsupported";
712
- message: string;
713
- retryable: boolean;
714
- details?: Record<string, import("zod").JSONType> | undefined;
715
- } | null;
716
- }>;
717
- press(key: string, callOptions?: CodemodeCallOptions): Promise<{
718
- protocolVersion: 1;
719
- operationId: string;
720
- browserSessionId: string;
721
- controllerGeneration: string;
722
- targetId: string;
723
- state: "completed" | "dispatched" | "failed" | "outcome_unknown" | "prepared";
724
- dispatchedAt: string | null;
725
- settledAt: string | null;
726
- observation: {
727
- protocolVersion: 1;
728
- observationId: string;
729
- browserSessionId: string;
730
- target: {
731
- id: string;
732
- browserSessionId: string;
733
- controllerGeneration: string;
734
- targetGeneration: string;
735
- documentGeneration: string | null;
736
- kind: "background_page" | "page" | "popup" | "worker";
737
- title: string;
738
- url: string;
739
- selected: boolean;
740
- attached: boolean;
741
- createdAt: string;
742
- };
743
- frameId: string | null;
744
- semantic: {
745
- kind: "snapshot";
746
- roots: import("@opengeni/contracts").InteractionSemanticNodeValue[];
747
- nodeCount: number;
748
- } | {
749
- kind: "diff";
750
- baseObservationId: string;
751
- removedRefs: string[];
752
- changed: import("@opengeni/contracts").InteractionSemanticNodeValue[];
753
- } | null;
754
- screenshot: {
755
- available: true;
756
- artifactId: string;
757
- kind: "assistant_completion" | "computer_screenshot" | "event_media" | "file" | "generated_image" | "generated_video" | "internal_update" | "tool_result";
758
- contentType: string;
759
- originalBytes: number;
760
- sha256: string;
761
- retainedAt: string;
762
- dimensions?: {
763
- width: number;
764
- height: number;
765
- } | undefined;
766
- retention: {
767
- policy: "workspace_file";
768
- expiresAt: null;
769
- } | {
770
- policy: "session_screenshot";
771
- expiresAt: string;
772
- };
773
- retrieval: {
774
- method: "GET";
775
- path: string;
776
- acceptRanges: "bytes";
777
- maxRangeBytes: number;
778
- };
779
- } | null;
780
- focusedRef: string | null;
781
- changedRegions: {
782
- x: number;
783
- y: number;
784
- width: number;
785
- height: number;
786
- }[];
787
- diagnostics: {
788
- consoleErrorCount: number;
789
- failedRequestCount: number;
790
- downloadCount: number;
791
- pageErrorCount: number;
792
- };
793
- dialog: {
794
- type: "alert" | "beforeunload" | "confirm" | "prompt";
795
- message: string;
796
- defaultPrompt: string;
797
- openedAt: string;
798
- } | null;
799
- observedAt: string;
800
- } | null;
801
- error: {
802
- code: "attempt_stale" | "controller_lost" | "controller_stale" | "document_stale" | "driver_failed" | "frame_stale" | "invalid_action" | "locator_ambiguous" | "locator_not_found" | "machine_locked" | "observation_stale" | "operation_conflict" | "outcome_unknown" | "permission_denied" | "resource_not_found" | "resource_unavailable" | "target_not_found" | "target_stale" | "timeout" | "unsupported";
803
- message: string;
804
- retryable: boolean;
805
- details?: Record<string, import("zod").JSONType> | undefined;
806
- } | null;
807
- }>;
808
- select(values: string[], callOptions?: CodemodeCallOptions): Promise<{
809
- protocolVersion: 1;
810
- operationId: string;
811
- browserSessionId: string;
812
- controllerGeneration: string;
813
- targetId: string;
814
- state: "completed" | "dispatched" | "failed" | "outcome_unknown" | "prepared";
815
- dispatchedAt: string | null;
816
- settledAt: string | null;
817
- observation: {
818
- protocolVersion: 1;
819
- observationId: string;
820
- browserSessionId: string;
821
- target: {
822
- id: string;
823
- browserSessionId: string;
824
- controllerGeneration: string;
825
- targetGeneration: string;
826
- documentGeneration: string | null;
827
- kind: "background_page" | "page" | "popup" | "worker";
828
- title: string;
829
- url: string;
830
- selected: boolean;
831
- attached: boolean;
832
- createdAt: string;
833
- };
834
- frameId: string | null;
835
- semantic: {
836
- kind: "snapshot";
837
- roots: import("@opengeni/contracts").InteractionSemanticNodeValue[];
838
- nodeCount: number;
839
- } | {
840
- kind: "diff";
841
- baseObservationId: string;
842
- removedRefs: string[];
843
- changed: import("@opengeni/contracts").InteractionSemanticNodeValue[];
844
- } | null;
845
- screenshot: {
846
- available: true;
847
- artifactId: string;
848
- kind: "assistant_completion" | "computer_screenshot" | "event_media" | "file" | "generated_image" | "generated_video" | "internal_update" | "tool_result";
849
- contentType: string;
850
- originalBytes: number;
851
- sha256: string;
852
- retainedAt: string;
853
- dimensions?: {
854
- width: number;
855
- height: number;
856
- } | undefined;
857
- retention: {
858
- policy: "workspace_file";
859
- expiresAt: null;
860
- } | {
861
- policy: "session_screenshot";
862
- expiresAt: string;
863
- };
864
- retrieval: {
865
- method: "GET";
866
- path: string;
867
- acceptRanges: "bytes";
868
- maxRangeBytes: number;
869
- };
870
- } | null;
871
- focusedRef: string | null;
872
- changedRegions: {
873
- x: number;
874
- y: number;
875
- width: number;
876
- height: number;
877
- }[];
878
- diagnostics: {
879
- consoleErrorCount: number;
880
- failedRequestCount: number;
881
- downloadCount: number;
882
- pageErrorCount: number;
883
- };
884
- dialog: {
885
- type: "alert" | "beforeunload" | "confirm" | "prompt";
886
- message: string;
887
- defaultPrompt: string;
888
- openedAt: string;
889
- } | null;
890
- observedAt: string;
891
- } | null;
892
- error: {
893
- code: "attempt_stale" | "controller_lost" | "controller_stale" | "document_stale" | "driver_failed" | "frame_stale" | "invalid_action" | "locator_ambiguous" | "locator_not_found" | "machine_locked" | "observation_stale" | "operation_conflict" | "outcome_unknown" | "permission_denied" | "resource_not_found" | "resource_unavailable" | "target_not_found" | "target_stale" | "timeout" | "unsupported";
894
- message: string;
895
- retryable: boolean;
896
- details?: Record<string, import("zod").JSONType> | undefined;
897
- } | null;
898
- }>;
899
- check(checked?: boolean, callOptions?: CodemodeCallOptions): Promise<{
900
- protocolVersion: 1;
901
- operationId: string;
902
- browserSessionId: string;
903
- controllerGeneration: string;
904
- targetId: string;
905
- state: "completed" | "dispatched" | "failed" | "outcome_unknown" | "prepared";
906
- dispatchedAt: string | null;
907
- settledAt: string | null;
908
- observation: {
909
- protocolVersion: 1;
910
- observationId: string;
911
- browserSessionId: string;
912
- target: {
913
- id: string;
914
- browserSessionId: string;
915
- controllerGeneration: string;
916
- targetGeneration: string;
917
- documentGeneration: string | null;
918
- kind: "background_page" | "page" | "popup" | "worker";
919
- title: string;
920
- url: string;
921
- selected: boolean;
922
- attached: boolean;
923
- createdAt: string;
924
- };
925
- frameId: string | null;
926
- semantic: {
927
- kind: "snapshot";
928
- roots: import("@opengeni/contracts").InteractionSemanticNodeValue[];
929
- nodeCount: number;
930
- } | {
931
- kind: "diff";
932
- baseObservationId: string;
933
- removedRefs: string[];
934
- changed: import("@opengeni/contracts").InteractionSemanticNodeValue[];
935
- } | null;
936
- screenshot: {
937
- available: true;
938
- artifactId: string;
939
- kind: "assistant_completion" | "computer_screenshot" | "event_media" | "file" | "generated_image" | "generated_video" | "internal_update" | "tool_result";
940
- contentType: string;
941
- originalBytes: number;
942
- sha256: string;
943
- retainedAt: string;
944
- dimensions?: {
945
- width: number;
946
- height: number;
947
- } | undefined;
948
- retention: {
949
- policy: "workspace_file";
950
- expiresAt: null;
951
- } | {
952
- policy: "session_screenshot";
953
- expiresAt: string;
954
- };
955
- retrieval: {
956
- method: "GET";
957
- path: string;
958
- acceptRanges: "bytes";
959
- maxRangeBytes: number;
960
- };
961
- } | null;
962
- focusedRef: string | null;
963
- changedRegions: {
964
- x: number;
965
- y: number;
966
- width: number;
967
- height: number;
968
- }[];
969
- diagnostics: {
970
- consoleErrorCount: number;
971
- failedRequestCount: number;
972
- downloadCount: number;
973
- pageErrorCount: number;
974
- };
975
- dialog: {
976
- type: "alert" | "beforeunload" | "confirm" | "prompt";
977
- message: string;
978
- defaultPrompt: string;
979
- openedAt: string;
980
- } | null;
981
- observedAt: string;
982
- } | null;
983
- error: {
984
- code: "attempt_stale" | "controller_lost" | "controller_stale" | "document_stale" | "driver_failed" | "frame_stale" | "invalid_action" | "locator_ambiguous" | "locator_not_found" | "machine_locked" | "observation_stale" | "operation_conflict" | "outcome_unknown" | "permission_denied" | "resource_not_found" | "resource_unavailable" | "target_not_found" | "target_stale" | "timeout" | "unsupported";
985
- message: string;
986
- retryable: boolean;
987
- details?: Record<string, import("zod").JSONType> | undefined;
988
- } | null;
989
- }>;
288
+ }, callOptions?: CodemodeCallOptions): Promise<CodemodeBrowserActionReceipt>;
289
+ doubleClick(callOptions?: CodemodeCallOptions): Promise<CodemodeBrowserActionReceipt>;
290
+ hover(callOptions?: CodemodeCallOptions): Promise<CodemodeBrowserActionReceipt>;
291
+ fill(value: string, callOptions?: CodemodeCallOptions): Promise<CodemodeBrowserActionReceipt>;
292
+ type(text: string, callOptions?: CodemodeCallOptions): Promise<CodemodeBrowserActionReceipt>;
293
+ press(key: string, callOptions?: CodemodeCallOptions): Promise<CodemodeBrowserActionReceipt>;
294
+ select(values: string[], callOptions?: CodemodeCallOptions): Promise<CodemodeBrowserActionReceipt>;
295
+ check(checked?: boolean, callOptions?: CodemodeCallOptions): Promise<CodemodeBrowserActionReceipt>;
990
296
  }
991
297
  export declare class CodemodeAuth {
992
298
  private readonly client;
@@ -1167,7 +473,7 @@ export declare class CodemodeComputerLocator {
1167
473
  screenshot: {
1168
474
  available: true;
1169
475
  artifactId: string;
1170
- kind: "assistant_completion" | "computer_screenshot" | "event_media" | "file" | "generated_image" | "generated_video" | "internal_update" | "tool_result";
476
+ kind: "assistant_completion" | "browser_screenshot" | "computer_screenshot" | "event_media" | "file" | "generated_image" | "generated_video" | "internal_update" | "tool_result";
1171
477
  contentType: string;
1172
478
  originalBytes: number;
1173
479
  sha256: string;
@@ -1250,7 +556,7 @@ export declare class CodemodeComputerLocator {
1250
556
  screenshot: {
1251
557
  available: true;
1252
558
  artifactId: string;
1253
- kind: "assistant_completion" | "computer_screenshot" | "event_media" | "file" | "generated_image" | "generated_video" | "internal_update" | "tool_result";
559
+ kind: "assistant_completion" | "browser_screenshot" | "computer_screenshot" | "event_media" | "file" | "generated_image" | "generated_video" | "internal_update" | "tool_result";
1254
560
  contentType: string;
1255
561
  originalBytes: number;
1256
562
  sha256: string;
@@ -1333,7 +639,7 @@ export declare class CodemodeComputerLocator {
1333
639
  screenshot: {
1334
640
  available: true;
1335
641
  artifactId: string;
1336
- kind: "assistant_completion" | "computer_screenshot" | "event_media" | "file" | "generated_image" | "generated_video" | "internal_update" | "tool_result";
642
+ kind: "assistant_completion" | "browser_screenshot" | "computer_screenshot" | "event_media" | "file" | "generated_image" | "generated_video" | "internal_update" | "tool_result";
1337
643
  contentType: string;
1338
644
  originalBytes: number;
1339
645
  sha256: string;
@@ -1416,7 +722,7 @@ export declare class CodemodeComputerLocator {
1416
722
  screenshot: {
1417
723
  available: true;
1418
724
  artifactId: string;
1419
- kind: "assistant_completion" | "computer_screenshot" | "event_media" | "file" | "generated_image" | "generated_video" | "internal_update" | "tool_result";
725
+ kind: "assistant_completion" | "browser_screenshot" | "computer_screenshot" | "event_media" | "file" | "generated_image" | "generated_video" | "internal_update" | "tool_result";
1420
726
  contentType: string;
1421
727
  originalBytes: number;
1422
728
  sha256: string;
@@ -1499,7 +805,7 @@ export declare class CodemodeComputerLocator {
1499
805
  screenshot: {
1500
806
  available: true;
1501
807
  artifactId: string;
1502
- kind: "assistant_completion" | "computer_screenshot" | "event_media" | "file" | "generated_image" | "generated_video" | "internal_update" | "tool_result";
808
+ kind: "assistant_completion" | "browser_screenshot" | "computer_screenshot" | "event_media" | "file" | "generated_image" | "generated_video" | "internal_update" | "tool_result";
1503
809
  contentType: string;
1504
810
  originalBytes: number;
1505
811
  sha256: string;
@@ -1582,7 +888,7 @@ export declare class CodemodeComputerLocator {
1582
888
  screenshot: {
1583
889
  available: true;
1584
890
  artifactId: string;
1585
- kind: "assistant_completion" | "computer_screenshot" | "event_media" | "file" | "generated_image" | "generated_video" | "internal_update" | "tool_result";
891
+ kind: "assistant_completion" | "browser_screenshot" | "computer_screenshot" | "event_media" | "file" | "generated_image" | "generated_video" | "internal_update" | "tool_result";
1586
892
  contentType: string;
1587
893
  originalBytes: number;
1588
894
  sha256: string;