@motebit/protocol 1.2.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -2
- package/dist/artifact-type.d.ts +144 -0
- package/dist/artifact-type.d.ts.map +1 -0
- package/dist/artifact-type.js +107 -0
- package/dist/artifact-type.js.map +1 -0
- package/dist/audience.d.ts +108 -0
- package/dist/audience.d.ts.map +1 -0
- package/dist/audience.js +104 -0
- package/dist/audience.js.map +1 -0
- package/dist/co-browse.d.ts +369 -0
- package/dist/co-browse.d.ts.map +1 -0
- package/dist/co-browse.js +64 -0
- package/dist/co-browse.js.map +1 -0
- package/dist/computer-use.d.ts +463 -3
- package/dist/computer-use.d.ts.map +1 -1
- package/dist/computer-use.js +40 -0
- package/dist/computer-use.js.map +1 -1
- package/dist/event-type.d.ts +62 -0
- package/dist/event-type.d.ts.map +1 -0
- package/dist/event-type.js +123 -0
- package/dist/event-type.js.map +1 -0
- package/dist/index.d.ts +257 -20
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +84 -1
- package/dist/index.js.map +1 -1
- package/dist/memory-events.d.ts +13 -0
- package/dist/memory-events.d.ts.map +1 -1
- package/dist/money.d.ts +33 -0
- package/dist/money.d.ts.map +1 -0
- package/dist/money.js +41 -0
- package/dist/money.js.map +1 -0
- package/dist/perception.d.ts +347 -0
- package/dist/perception.d.ts.map +1 -0
- package/dist/perception.js +9 -0
- package/dist/perception.js.map +1 -0
- package/dist/retention-policy.d.ts +8 -1
- package/dist/retention-policy.d.ts.map +1 -1
- package/dist/retention-policy.js +18 -0
- package/dist/retention-policy.js.map +1 -1
- package/dist/routing.d.ts +266 -0
- package/dist/routing.d.ts.map +1 -0
- package/dist/routing.js +88 -0
- package/dist/routing.js.map +1 -0
- package/dist/sensitivity.d.ts +123 -0
- package/dist/sensitivity.d.ts.map +1 -0
- package/dist/sensitivity.js +154 -0
- package/dist/sensitivity.js.map +1 -0
- package/dist/settlement-asset.d.ts +92 -0
- package/dist/settlement-asset.d.ts.map +1 -0
- package/dist/settlement-asset.js +82 -0
- package/dist/settlement-asset.js.map +1 -0
- package/dist/settlement-mode.d.ts +144 -13
- package/dist/settlement-mode.d.ts.map +1 -1
- package/dist/settlement-mode.js +45 -1
- package/dist/settlement-mode.js.map +1 -1
- package/dist/transparency.d.ts +116 -0
- package/dist/transparency.d.ts.map +1 -0
- package/dist/transparency.js +67 -0
- package/dist/transparency.js.map +1 -0
- package/package.json +1 -1
package/dist/computer-use.d.ts
CHANGED
|
@@ -37,6 +37,7 @@
|
|
|
37
37
|
* producer (cloud-browser surface, federation peer replaying an audit log)
|
|
38
38
|
* or a non-desktop-shim consumer exercises the format in anger.
|
|
39
39
|
*/
|
|
40
|
+
import type { ControlState } from "./co-browse.js";
|
|
40
41
|
/**
|
|
41
42
|
* A point in primary-display logical pixel coordinates. `(0, 0)` = top-left.
|
|
42
43
|
* @alpha
|
|
@@ -157,17 +158,102 @@ export interface ScrollAction {
|
|
|
157
158
|
readonly dx: number;
|
|
158
159
|
readonly dy: number;
|
|
159
160
|
}
|
|
161
|
+
/**
|
|
162
|
+
* Navigate the active browser context to `url`. Cloud-browser-only in
|
|
163
|
+
* v1: the headless Playwright runtime in `services/browser-sandbox`
|
|
164
|
+
* has no address-bar UI for `key`/`type` to drive, so the spec
|
|
165
|
+
* promotion path
|
|
166
|
+
* (`spec/computer-use-v1.md` §"No new wire-format actions… real-usage-
|
|
167
|
+
* driven, not speculative") fired when the AI hit "navigate to
|
|
168
|
+
* tesla.com" against the cloud-browser dispatcher.
|
|
169
|
+
*
|
|
170
|
+
* Desktop dispatcher (`apps/desktop/src-tauri/src/computer_use.rs`)
|
|
171
|
+
* does NOT implement this action — OS-level computer-use has no
|
|
172
|
+
* notion of "the active browser context"; the user is in control of
|
|
173
|
+
* which app is focused. The dispatcher-parity check at
|
|
174
|
+
* `scripts/check-computer-use-dispatcher-parity.ts` carries an
|
|
175
|
+
* ALLOWLIST entry naming desktop as deferred until an OS-level
|
|
176
|
+
* navigation use-case proves itself.
|
|
177
|
+
* @alpha
|
|
178
|
+
*/
|
|
179
|
+
export interface NavigateAction {
|
|
180
|
+
readonly kind: "navigate";
|
|
181
|
+
/**
|
|
182
|
+
* Target URL. Implementations SHOULD normalize relative-looking
|
|
183
|
+
* inputs (`example.com` → `https://example.com`) but MAY reject
|
|
184
|
+
* malformed inputs with a `not_supported` failure.
|
|
185
|
+
*/
|
|
186
|
+
readonly url: string;
|
|
187
|
+
}
|
|
188
|
+
/**
|
|
189
|
+
* element-1 — click on a structurally-addressed element. Server
|
|
190
|
+
* resolves `element_id` (issued by a prior `read_page` extraction
|
|
191
|
+
* via the stamped `data-motebit-id` attribute), scrolls into view,
|
|
192
|
+
* clicks the center of the element's bounding rect.
|
|
193
|
+
*
|
|
194
|
+
* Prefer this over coordinate-based `click` when the target was
|
|
195
|
+
* discovered via `read_page` — durable against viewport, zoom, and
|
|
196
|
+
* layout shifts. Coordinate `click` remains for purely-visual tasks
|
|
197
|
+
* (clicking on a position seen in pixels).
|
|
198
|
+
*
|
|
199
|
+
* On staleness (page navigated since read_page, page reloaded,
|
|
200
|
+
* element removed by JS) the result fails with `element_not_found`
|
|
201
|
+
* and the AI re-reads to refresh the id space.
|
|
202
|
+
* @alpha
|
|
203
|
+
*/
|
|
204
|
+
export interface ClickElementAction {
|
|
205
|
+
readonly kind: "click_element";
|
|
206
|
+
readonly element_id: string;
|
|
207
|
+
}
|
|
208
|
+
/**
|
|
209
|
+
* element-1 — focus a structurally-addressed element without
|
|
210
|
+
* clicking. Useful for setting up `type` calls without the side-
|
|
211
|
+
* effects of a click (some fields open dropdowns / dialogs on
|
|
212
|
+
* click; focus-only avoids them).
|
|
213
|
+
* @alpha
|
|
214
|
+
*/
|
|
215
|
+
export interface FocusElementAction {
|
|
216
|
+
readonly kind: "focus_element";
|
|
217
|
+
readonly element_id: string;
|
|
218
|
+
}
|
|
219
|
+
/**
|
|
220
|
+
* element-1 — semantic-intent type. Server focuses the addressed
|
|
221
|
+
* element first (so keystrokes can't be swallowed by the page's
|
|
222
|
+
* focus state), optionally clears existing value, then types. The
|
|
223
|
+
* result envelope carries `text_appeared` truth-feedback so the AI
|
|
224
|
+
* doesn't confabulate "I typed it" when the keystrokes went
|
|
225
|
+
* nowhere.
|
|
226
|
+
*
|
|
227
|
+
* Composes the two-step click+type workflow into one semantic
|
|
228
|
+
* action — closes the action-truth gap from the witnessed
|
|
229
|
+
* 2026-05-08 smoke (AI typed without focus → keystrokes
|
|
230
|
+
* swallowed → AI claimed success).
|
|
231
|
+
* @alpha
|
|
232
|
+
*/
|
|
233
|
+
export interface TypeIntoAction {
|
|
234
|
+
readonly kind: "type_into";
|
|
235
|
+
readonly element_id: string;
|
|
236
|
+
readonly text: string;
|
|
237
|
+
/** Per-character delay in ms; same shape as the lower-level `type` action. */
|
|
238
|
+
readonly per_char_delay_ms?: number;
|
|
239
|
+
/**
|
|
240
|
+
* If true (default), the field's current value is cleared before
|
|
241
|
+
* typing. Mirrors the human "type fresh into this box" intent. Set
|
|
242
|
+
* to false to append to existing text.
|
|
243
|
+
*/
|
|
244
|
+
readonly clear_first?: boolean;
|
|
245
|
+
}
|
|
160
246
|
/**
|
|
161
247
|
* Full action taxonomy. Every concrete action the motebit can request is
|
|
162
248
|
* one of these. Exhaustive discriminated union on `kind`.
|
|
163
249
|
* @alpha
|
|
164
250
|
*/
|
|
165
|
-
export type ComputerAction = ScreenshotAction | CursorPositionAction | ClickAction | DoubleClickAction | MouseMoveAction | DragAction | TypeAction | KeyAction | ScrollAction;
|
|
251
|
+
export type ComputerAction = ScreenshotAction | CursorPositionAction | ClickAction | DoubleClickAction | MouseMoveAction | DragAction | TypeAction | KeyAction | ScrollAction | NavigateAction | ClickElementAction | FocusElementAction | TypeIntoAction;
|
|
166
252
|
/**
|
|
167
253
|
* Discriminator values — useful for JSON Schema enum and runtime validation.
|
|
168
254
|
* @alpha
|
|
169
255
|
*/
|
|
170
|
-
export declare const COMPUTER_ACTION_KINDS: readonly ["screenshot", "cursor_position", "click", "double_click", "mouse_move", "drag", "type", "key", "scroll"];
|
|
256
|
+
export declare const COMPUTER_ACTION_KINDS: readonly ["screenshot", "cursor_position", "click", "double_click", "mouse_move", "drag", "type", "key", "scroll", "navigate", "click_element", "focus_element", "type_into"];
|
|
171
257
|
/** @alpha */
|
|
172
258
|
export type ComputerActionKind = (typeof COMPUTER_ACTION_KINDS)[number];
|
|
173
259
|
/**
|
|
@@ -259,6 +345,193 @@ export interface CursorPositionObservation {
|
|
|
259
345
|
readonly y: number;
|
|
260
346
|
readonly captured_at: number;
|
|
261
347
|
}
|
|
348
|
+
/**
|
|
349
|
+
* A single heading extracted from the page's DOM. `level` is the
|
|
350
|
+
* HTML heading level 1-6; `text` is the visible text content. The
|
|
351
|
+
* server returns headings in document order so the AI can rebuild
|
|
352
|
+
* the page's outline without parsing markup.
|
|
353
|
+
* @alpha
|
|
354
|
+
*/
|
|
355
|
+
export interface ReadPageHeading {
|
|
356
|
+
readonly level: number;
|
|
357
|
+
readonly text: string;
|
|
358
|
+
}
|
|
359
|
+
/**
|
|
360
|
+
* A single visible link extracted from the page's DOM. `text` is
|
|
361
|
+
* the anchor's visible label (innerText); `href` is the absolute
|
|
362
|
+
* URL the link points to. Server skips empty-label links and
|
|
363
|
+
* `javascript:` / fragment-only hrefs.
|
|
364
|
+
* @alpha
|
|
365
|
+
*/
|
|
366
|
+
export interface ReadPageLink {
|
|
367
|
+
readonly text: string;
|
|
368
|
+
readonly href: string;
|
|
369
|
+
}
|
|
370
|
+
/**
|
|
371
|
+
* A single typeable input extracted from the page's DOM —
|
|
372
|
+
* `<input>` (text-shaped types: text, search, email, url, tel,
|
|
373
|
+
* number, password) and `<textarea>`. Server stamps a
|
|
374
|
+
* `data-motebit-id="motebit-N"` attribute during extraction so
|
|
375
|
+
* subsequent `type_into` / `focus_element` actions can address
|
|
376
|
+
* the element by `element_id` instead of coordinates.
|
|
377
|
+
*
|
|
378
|
+
* **Why element-addressing over coordinates.** Coordinates are
|
|
379
|
+
* fragile against viewport size, zoom, layout shifts, dynamic
|
|
380
|
+
* loading. Structural identity is stable for the lifetime of the
|
|
381
|
+
* page. Production browser-agent platforms (Browserbase, the
|
|
382
|
+
* Playwright codegen flow, Anthropic's computer-use cookbook)
|
|
383
|
+
* converge on the same primitive — the AI says "type into this
|
|
384
|
+
* input" referring to a thing it discovered via read_page, not
|
|
385
|
+
* "type at (612, 348)."
|
|
386
|
+
*
|
|
387
|
+
* The element_id is opaque to the AI — server-issued, server-
|
|
388
|
+
* resolved. Stamps are scoped to the read_page response that
|
|
389
|
+
* returned them; a subsequent read_page may issue different ids
|
|
390
|
+
* for the same elements (the prior stamps are cleared and re-
|
|
391
|
+
* stamped). Cross-read stability is a future slice.
|
|
392
|
+
*
|
|
393
|
+
* Stamps don't survive page navigation; an `element_not_found`
|
|
394
|
+
* result tells the AI to re-read.
|
|
395
|
+
* @alpha
|
|
396
|
+
*/
|
|
397
|
+
export interface ReadPageInput {
|
|
398
|
+
/** Server-issued opaque identifier for subsequent typed actions. */
|
|
399
|
+
readonly element_id: string;
|
|
400
|
+
/** DOM tag — `input` or `textarea`. */
|
|
401
|
+
readonly tag: "input" | "textarea";
|
|
402
|
+
/**
|
|
403
|
+
* For `<input>`, the `type` attribute (text, search, email, url,
|
|
404
|
+
* tel, number, password, …). For `<textarea>`, always `"textarea"`.
|
|
405
|
+
* Useful to the AI for choosing what to type and whether the
|
|
406
|
+
* field is sensitive.
|
|
407
|
+
*/
|
|
408
|
+
readonly input_type: string;
|
|
409
|
+
/** `name` attribute when present. */
|
|
410
|
+
readonly name?: string;
|
|
411
|
+
/** `placeholder` attribute when present. */
|
|
412
|
+
readonly placeholder?: string;
|
|
413
|
+
/** `aria-label` when present (or the closest `<label>` text). */
|
|
414
|
+
readonly aria_label?: string;
|
|
415
|
+
/**
|
|
416
|
+
* Current value of the field at extraction time, capped at 256
|
|
417
|
+
* characters. Lets the AI reason about whether to clear before
|
|
418
|
+
* typing, and verify intent across turns.
|
|
419
|
+
*/
|
|
420
|
+
readonly value?: string;
|
|
421
|
+
}
|
|
422
|
+
/**
|
|
423
|
+
* A single button-shaped clickable element — `<button>`,
|
|
424
|
+
* `<input type="submit">`, `<input type="button">`,
|
|
425
|
+
* `<input type="reset">`, plus `<a>` tags styled as buttons (when
|
|
426
|
+
* the element has `role="button"` or carries a button-shaped
|
|
427
|
+
* accessible name without an href). Server stamps the same
|
|
428
|
+
* `data-motebit-id` so `click_element({element_id})` resolves
|
|
429
|
+
* unambiguously.
|
|
430
|
+
*
|
|
431
|
+
* `<a>` tags WITH a navigation href stay in `ReadPageLink`. A
|
|
432
|
+
* single element MAY appear in both `links` and `buttons` if it
|
|
433
|
+
* has both a navigation href and a button-shaped role; the AI
|
|
434
|
+
* picks the addressing it needs.
|
|
435
|
+
* @alpha
|
|
436
|
+
*/
|
|
437
|
+
export interface ReadPageButton {
|
|
438
|
+
readonly element_id: string;
|
|
439
|
+
/** DOM tag — `button`, `input`, or `a`. */
|
|
440
|
+
readonly tag: "button" | "input" | "a";
|
|
441
|
+
/** Visible label text (innerText or `aria-label` when innerText is empty). */
|
|
442
|
+
readonly text: string;
|
|
443
|
+
/**
|
|
444
|
+
* For `<input>` button variants, the `type` attribute
|
|
445
|
+
* (`submit` / `button` / `reset`). Absent for `<button>` and
|
|
446
|
+
* `<a role="button">`.
|
|
447
|
+
*/
|
|
448
|
+
readonly input_type?: string;
|
|
449
|
+
}
|
|
450
|
+
/**
|
|
451
|
+
* Wire format for the `read_page` tool's result. Returned by
|
|
452
|
+
* `services/browser-sandbox`'s `POST /sessions/:id/read-page` and
|
|
453
|
+
* read unchanged by `CloudBrowserDispatcher.readPage()`. Lands in
|
|
454
|
+
* the AI loop as the tool result; passes through `projectForAi`
|
|
455
|
+
* unchanged (no `bytes_base64` field to strip).
|
|
456
|
+
*
|
|
457
|
+
* Sizes are bounded server-side to keep the AI context tractable:
|
|
458
|
+
* - `text` truncated at 8KB; `text_truncated` set when the cut
|
|
459
|
+
* fired so the AI knows it's reading a prefix.
|
|
460
|
+
* - `headings` capped at 100 entries.
|
|
461
|
+
* - `links` capped at 100 entries.
|
|
462
|
+
*
|
|
463
|
+
* The bounds are conservative defaults the surface MAY tune in
|
|
464
|
+
* future. Beyond a soft cap, `text_truncated: true` tells the AI
|
|
465
|
+
* to ask for a more targeted observation rather than scrolling
|
|
466
|
+
* through a 50KB blob.
|
|
467
|
+
* @alpha
|
|
468
|
+
*/
|
|
469
|
+
export interface ReadPageResult {
|
|
470
|
+
readonly kind: "read_page";
|
|
471
|
+
readonly session_id: string;
|
|
472
|
+
readonly url: string;
|
|
473
|
+
readonly title: string;
|
|
474
|
+
readonly text: string;
|
|
475
|
+
readonly text_truncated: boolean;
|
|
476
|
+
readonly headings: ReadonlyArray<ReadPageHeading>;
|
|
477
|
+
readonly links: ReadonlyArray<ReadPageLink>;
|
|
478
|
+
/**
|
|
479
|
+
* element-1 — typeable input fields with server-issued
|
|
480
|
+
* `element_id` for `focus_element` / `type_into` actions. Capped
|
|
481
|
+
* at 100 entries server-side. Empty when the page has no
|
|
482
|
+
* typeable fields.
|
|
483
|
+
*/
|
|
484
|
+
readonly inputs: ReadonlyArray<ReadPageInput>;
|
|
485
|
+
/**
|
|
486
|
+
* element-1 — button-shaped clickable elements with server-
|
|
487
|
+
* issued `element_id` for `click_element` actions. Capped at
|
|
488
|
+
* 100 entries server-side. Empty when the page has no button-
|
|
489
|
+
* shaped elements (text-content pages, document viewers).
|
|
490
|
+
*/
|
|
491
|
+
readonly buttons: ReadonlyArray<ReadPageButton>;
|
|
492
|
+
/**
|
|
493
|
+
* Typed-truth hint naming the page's primary submit button by
|
|
494
|
+
* `element_id` — present when `buttons[]` contains a discoverable
|
|
495
|
+
* submit-class element. The runtime detects it via two signals in
|
|
496
|
+
* order:
|
|
497
|
+
*
|
|
498
|
+
* 1. **HTML semantic**: any button with `input_type === "submit"`
|
|
499
|
+
* (`<input type="submit">`). Most reliable; this is the
|
|
500
|
+
* browser's own statement that the element submits a form.
|
|
501
|
+
* 2. **Label heuristic** (fallback): a `<button>` whose visible
|
|
502
|
+
* label matches a submit-class word — `Search` / `Submit` /
|
|
503
|
+
* `Send` / `Sign in` / `Log in` / `Continue` / `Go` /
|
|
504
|
+
* `Subscribe` / `Next` / `Save` / `Post`. Case-insensitive,
|
|
505
|
+
* whole-label-or-prefix match. The label heuristic produces
|
|
506
|
+
* false positives on rare pages with non-submit buttons
|
|
507
|
+
* labeled "Send" (a contact-icon UI); the AI's reading still
|
|
508
|
+
* benefits from the hint because `click_element` on a
|
|
509
|
+
* non-submit button is a no-op rather than a wrong action.
|
|
510
|
+
*
|
|
511
|
+
* AI usage: when this field is present, prefer
|
|
512
|
+
* `click_element(submit_button_id)` over `key("Enter")` for form
|
|
513
|
+
* submission. The runtime invariant is the doctrine's preferred
|
|
514
|
+
* mechanism (no focus race, no global-keystroke ambiguity) — the
|
|
515
|
+
* wire field carries the "right tool for this page" signal that
|
|
516
|
+
* the AI's selection would otherwise have to derive from
|
|
517
|
+
* unstructured prompt teaching. Doctrine:
|
|
518
|
+
* `docs/doctrine/runtime-invariants-over-prompt-rules.md` —
|
|
519
|
+
* exemplar of B→A graduation via typed-truth conversion.
|
|
520
|
+
*
|
|
521
|
+
* Absent when:
|
|
522
|
+
* - The page has no `buttons[]` at all (text-content pages).
|
|
523
|
+
* - `buttons[]` contains no submit-class element (a search
|
|
524
|
+
* results page with only navigation chips, e.g.).
|
|
525
|
+
* - The page uses non-button submit affordances (custom widgets,
|
|
526
|
+
* `<a>` tags posting via JS) — the heuristic stays conservative
|
|
527
|
+
* to avoid false-confident hints.
|
|
528
|
+
*
|
|
529
|
+
* When absent, the AI falls back to its existing reasoning over
|
|
530
|
+
* `buttons[]` directly. This is additive, not replacing.
|
|
531
|
+
*/
|
|
532
|
+
readonly submit_button_id?: string;
|
|
533
|
+
readonly extracted_at: number;
|
|
534
|
+
}
|
|
262
535
|
/**
|
|
263
536
|
* Signed event emitted when a computer-use session begins. Carries the
|
|
264
537
|
* primary display's dimensions and scaling factor so the AI knows the
|
|
@@ -304,9 +577,196 @@ export interface ComputerSessionClosed {
|
|
|
304
577
|
* user_preempted — physical user input interrupted mid-dispatch
|
|
305
578
|
* platform_blocked — OS blocked synthetic input (secure password field, elevation boundary)
|
|
306
579
|
* not_supported — surface cannot execute computer use at all
|
|
580
|
+
* frame_stale — the page navigated underneath the action; the
|
|
581
|
+
* executor's frame reference is no longer valid.
|
|
582
|
+
* Distinct from session_closed (the session is
|
|
583
|
+
* still open; only the frame changed). The
|
|
584
|
+
* cloud executor retries once before surfacing
|
|
585
|
+
* this so transient redirects (Google's
|
|
586
|
+
* anti-cache `?zx=…` redirect, OAuth round-
|
|
587
|
+
* trips, AJAX-driven URL rewrites) don't reach
|
|
588
|
+
* the AI as failure. When the AI sees this
|
|
589
|
+
* reason, the page has moved twice and a
|
|
590
|
+
* re-read is required.
|
|
307
591
|
* @alpha
|
|
308
592
|
*/
|
|
309
|
-
export declare const COMPUTER_FAILURE_REASONS: readonly ["policy_denied", "approval_required", "approval_expired", "permission_denied", "session_closed", "target_not_found", "target_obscured", "user_preempted", "platform_blocked", "not_supported"];
|
|
593
|
+
export declare const COMPUTER_FAILURE_REASONS: readonly ["policy_denied", "approval_required", "approval_expired", "permission_denied", "session_closed", "target_not_found", "target_obscured", "user_preempted", "platform_blocked", "not_supported", "not_in_control", "frame_stale"];
|
|
310
594
|
/** @alpha */
|
|
311
595
|
export type ComputerFailureReason = (typeof COMPUTER_FAILURE_REASONS)[number];
|
|
596
|
+
/**
|
|
597
|
+
* One CDP-decoded screencast frame as it lands on the wire.
|
|
598
|
+
*
|
|
599
|
+
* `jpeg_base64` — the image payload, base64-encoded JPEG. Quality is
|
|
600
|
+
* tuned by the service (60% at v1.3) for slab register, not fidelity.
|
|
601
|
+
*
|
|
602
|
+
* `timestamp` — wall-clock ms from CDP's frame metadata (normalized;
|
|
603
|
+
* CDP returns seconds). Frames may arrive slightly out of order
|
|
604
|
+
* under load; consumers should latest-wins on `timestamp`.
|
|
605
|
+
*
|
|
606
|
+
* `device_width` / `device_height` — logical-pixel dimensions of the
|
|
607
|
+
* captured viewport. The slab uses these to compute the right
|
|
608
|
+
* aspect ratio when rendering into a `<canvas>` or `<img>`.
|
|
609
|
+
* @alpha
|
|
610
|
+
*/
|
|
611
|
+
export interface ScreencastFrame {
|
|
612
|
+
readonly jpeg_base64: string;
|
|
613
|
+
readonly timestamp: number;
|
|
614
|
+
readonly device_width: number;
|
|
615
|
+
readonly device_height: number;
|
|
616
|
+
}
|
|
617
|
+
/**
|
|
618
|
+
* Subscribe-shape source of `ScreencastFrame`s. Producers (apps) own
|
|
619
|
+
* the publish path (dispatcher → bus); consumers (the slab's live-
|
|
620
|
+
* browser renderer) subscribe via `subscribe(cb)` and dispose via the
|
|
621
|
+
* returned unsubscribe.
|
|
622
|
+
*
|
|
623
|
+
* Why subscribe-shape rather than push-callback in payload. A slab
|
|
624
|
+
* item's `payload` is signed-eligible state; embedding a closure
|
|
625
|
+
* makes the payload non-serializable. `ScreencastFrameSource` is a
|
|
626
|
+
* minimal observer interface — easy to fake in tests, easy to
|
|
627
|
+
* implement (one Set + iterate). The frame stream itself is in-
|
|
628
|
+
* memory only and never goes through the audit log.
|
|
629
|
+
* @alpha
|
|
630
|
+
*/
|
|
631
|
+
export interface ScreencastFrameSource {
|
|
632
|
+
subscribe(callback: (frame: ScreencastFrame) => void): () => void;
|
|
633
|
+
}
|
|
634
|
+
/**
|
|
635
|
+
* Per-action structural roll-up entry. The runtime appends one of these
|
|
636
|
+
* to the in-session log on every `executeAction` call, regardless of
|
|
637
|
+
* outcome. The canonical JSON of the array (in dispatch order) is
|
|
638
|
+
* hashed into `ComputerSessionReceipt.actions_hash` at session close.
|
|
639
|
+
*
|
|
640
|
+
* Carries kind + timing + outcome only — never targets, args, screenshot
|
|
641
|
+
* bytes, OCR text, or any payload that could leak content. The
|
|
642
|
+
* `failure_reason` is present iff `outcome === "failure"`.
|
|
643
|
+
*
|
|
644
|
+
* Why a separate type rather than reusing `ComputerActionRequest`:
|
|
645
|
+
* the request type carries action-specific payloads (target points,
|
|
646
|
+
* type strings, drag deltas). The receipt commits to *structure*, not
|
|
647
|
+
* *intent* — the per-action ToolInvocationReceipt already commits to
|
|
648
|
+
* the args via `args_hash`. Splitting the two keeps the session
|
|
649
|
+
* receipt's privacy invariant (no leak surface) compositional with
|
|
650
|
+
* the per-action receipt's audit invariant.
|
|
651
|
+
* @alpha
|
|
652
|
+
*/
|
|
653
|
+
export interface ComputerSessionActionRecord {
|
|
654
|
+
readonly kind: ComputerActionKind;
|
|
655
|
+
readonly started_at: number;
|
|
656
|
+
readonly completed_at: number;
|
|
657
|
+
readonly outcome: "success" | "failure";
|
|
658
|
+
readonly failure_reason?: ComputerFailureReason;
|
|
659
|
+
/**
|
|
660
|
+
* Co-browse Slice 1 — when `failure_reason === "not_in_control"`,
|
|
661
|
+
* this field carries the literal `ControlState` at dispatch time
|
|
662
|
+
* (e.g. `{kind: "user"}` or `{kind: "paused", previousDriver:
|
|
663
|
+
* "motebit"}`). The runtime stamps it on the action ledger; the
|
|
664
|
+
* field flows into `actions_hash` so any retroactive edit to the
|
|
665
|
+
* recorded state breaks the session-receipt signature.
|
|
666
|
+
*
|
|
667
|
+
* Present iff `failure_reason === "not_in_control"` — control
|
|
668
|
+
* state at non-control denials would be category noise. Absent on
|
|
669
|
+
* success and on every other failure mode.
|
|
670
|
+
*
|
|
671
|
+
* @alpha
|
|
672
|
+
*/
|
|
673
|
+
readonly control_state_at_denial?: ControlState;
|
|
674
|
+
}
|
|
675
|
+
/**
|
|
676
|
+
* Body of a `ComputerSessionReceipt` *before* it is signed. The signer
|
|
677
|
+
* stamps `suite` and `signature`. Embedded as the input shape of the
|
|
678
|
+
* runtime's session summarizer; the signer (`@motebit/crypto`'s
|
|
679
|
+
* `signComputerSessionReceipt`) consumes this and emits a
|
|
680
|
+
* `ComputerSessionReceipt`.
|
|
681
|
+
* @alpha
|
|
682
|
+
*/
|
|
683
|
+
export interface SignableComputerSessionReceipt {
|
|
684
|
+
readonly receipt_id: string;
|
|
685
|
+
readonly session_id: string;
|
|
686
|
+
readonly motebit_id: string;
|
|
687
|
+
readonly public_key?: string;
|
|
688
|
+
/**
|
|
689
|
+
* Embodiment mode the session ran under (`"virtual_browser"`,
|
|
690
|
+
* `"desktop_drive"`, etc.). Free-typed string here for the same
|
|
691
|
+
* reason `ToolDefinition.embodimentMode` is — the canonical
|
|
692
|
+
* `EmbodimentMode` union lives in `@motebit/render-engine` and
|
|
693
|
+
* promoting it into the protocol layer is a separate slice.
|
|
694
|
+
* Production callers always pass a member of `EMBODIMENT_MODES`;
|
|
695
|
+
* verifiers should treat unknown strings as opaque-but-valid.
|
|
696
|
+
*/
|
|
697
|
+
readonly embodiment_mode: string;
|
|
698
|
+
readonly display_width: number;
|
|
699
|
+
readonly display_height: number;
|
|
700
|
+
readonly scaling_factor: number;
|
|
701
|
+
readonly opened_at: number;
|
|
702
|
+
readonly closed_at: number;
|
|
703
|
+
/** Free-text closure code from `ComputerSessionClosed.reason`. */
|
|
704
|
+
readonly close_reason?: string;
|
|
705
|
+
/** Total actions dispatched during the session. */
|
|
706
|
+
readonly action_count: number;
|
|
707
|
+
/** Action outcome counts. `success + failure === action_count`. */
|
|
708
|
+
readonly outcomes_summary: {
|
|
709
|
+
readonly success: number;
|
|
710
|
+
readonly failure: number;
|
|
711
|
+
};
|
|
712
|
+
/**
|
|
713
|
+
* Per-failure-reason counts. Closed key set
|
|
714
|
+
* (`ComputerFailureReason`); absent keys mean zero. Sum of values
|
|
715
|
+
* equals `outcomes_summary.failure`.
|
|
716
|
+
*/
|
|
717
|
+
readonly failure_breakdown: {
|
|
718
|
+
readonly [K in ComputerFailureReason]?: number;
|
|
719
|
+
};
|
|
720
|
+
/**
|
|
721
|
+
* True if `ComputerSessionManager.halt()` fired during this session
|
|
722
|
+
* (spec §3.3 user-floor primitive). A halted session that resumed
|
|
723
|
+
* and continued still has `was_halted: true` — the flag commits to
|
|
724
|
+
* "the user paused at least once," not to terminal state.
|
|
725
|
+
*/
|
|
726
|
+
readonly was_halted: boolean;
|
|
727
|
+
/**
|
|
728
|
+
* Highest sensitivity tier observed across all action observations
|
|
729
|
+
* during the session. Closed `SensitivityLevel` union by
|
|
730
|
+
* convention — encoded as the string value (e.g. `"financial"`).
|
|
731
|
+
* The runtime's observation classifier sets this; absence implies
|
|
732
|
+
* `"none"` (no observation rose above the floor).
|
|
733
|
+
*/
|
|
734
|
+
readonly max_sensitivity: string;
|
|
735
|
+
/**
|
|
736
|
+
* SHA-256 hex digest of JCS-canonicalized
|
|
737
|
+
* `ReadonlyArray<ComputerSessionActionRecord>` in dispatch order.
|
|
738
|
+
* Verifiers with the per-action records recompute and match;
|
|
739
|
+
* verifiers without the records still get the signature's commit
|
|
740
|
+
* to the digest itself.
|
|
741
|
+
*/
|
|
742
|
+
readonly actions_hash: string;
|
|
743
|
+
}
|
|
744
|
+
/**
|
|
745
|
+
* Signed proof that a computer-use session ran with this exact
|
|
746
|
+
* shape. Sibling of `ExecutionReceipt` and `ToolInvocationReceipt`.
|
|
747
|
+
*
|
|
748
|
+
* - Self-verifiable: a third party with the signer's public key
|
|
749
|
+
* can verify without contacting any relay.
|
|
750
|
+
* - Structural-only: never carries observation bytes, OCR text,
|
|
751
|
+
* action targets, or any content that could leak. The
|
|
752
|
+
* `actions_hash` binds to a roll-up; per-action
|
|
753
|
+
* `ToolInvocationReceipt`s carry the args/result hashes.
|
|
754
|
+
* - Issued at session close. The runtime emits one per
|
|
755
|
+
* `closeSession()` call (including idempotent replays of an
|
|
756
|
+
* already-closed session).
|
|
757
|
+
* - Composes with delegation: a session whose outer task was
|
|
758
|
+
* delegated produces a `ComputerSessionReceipt` AND a
|
|
759
|
+
* `DelegationReceipt`. Verifiers can chain the two by
|
|
760
|
+
* `motebit_id` + timeframe.
|
|
761
|
+
* @alpha
|
|
762
|
+
*/
|
|
763
|
+
export interface ComputerSessionReceipt extends SignableComputerSessionReceipt {
|
|
764
|
+
/**
|
|
765
|
+
* Cryptosuite discriminator. Always `"motebit-jcs-ed25519-b64-v1"`
|
|
766
|
+
* today. Widening requires a registry change in `SuiteId` + a new
|
|
767
|
+
* dispatch arm in `@motebit/crypto`, not a wire-format break.
|
|
768
|
+
*/
|
|
769
|
+
readonly suite: "motebit-jcs-ed25519-b64-v1";
|
|
770
|
+
readonly signature: string;
|
|
771
|
+
}
|
|
312
772
|
//# sourceMappingURL=computer-use.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"computer-use.d.ts","sourceRoot":"","sources":["../src/computer-use.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsCG;
|
|
1
|
+
{"version":3,"file":"computer-use.d.ts","sourceRoot":"","sources":["../src/computer-use.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsCG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAInD;;;GAGG;AACH,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,kBAAkB;IACjC,4EAA4E;IAC5E,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IACvB,wCAAwC;IACxC,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IACxB;;;;OAIG;IACH,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAID;;;GAGG;AACH,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,IAAI,EAAE,YAAY,CAAC;CAC7B;AAED;;;GAGG;AACH,MAAM,WAAW,oBAAoB;IACnC,QAAQ,CAAC,IAAI,EAAE,iBAAiB,CAAC;CAClC;AAED;;;GAGG;AACH,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,aAAa,CAAC;IAC/B,2DAA2D;IAC3D,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IACzB,mDAAmD;IACnD,QAAQ,CAAC,SAAS,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACvC,QAAQ,CAAC,WAAW,CAAC,EAAE,kBAAkB,CAAC;CAC3C;AAED;;;GAGG;AACH,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,IAAI,EAAE,cAAc,CAAC;IAC9B,QAAQ,CAAC,MAAM,EAAE,aAAa,CAAC;IAC/B,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,SAAS,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACvC,QAAQ,CAAC,WAAW,CAAC,EAAE,kBAAkB,CAAC;CAC3C;AAED;;;GAGG;AACH,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,IAAI,EAAE,YAAY,CAAC;IAC5B,QAAQ,CAAC,MAAM,EAAE,aAAa,CAAC;IAC/B,QAAQ,CAAC,WAAW,CAAC,EAAE,kBAAkB,CAAC;CAC3C;AAED;;;GAGG;AACH,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,aAAa,CAAC;IAC7B,QAAQ,CAAC,EAAE,EAAE,aAAa,CAAC;IAC3B,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,SAAS,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACvC,qFAAqF;IACrF,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,WAAW,CAAC,EAAE,kBAAkB,CAAC;CAC3C;AAED;;;GAGG;AACH,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB;;;OAGG;IACH,QAAQ,CAAC,iBAAiB,CAAC,EAAE,MAAM,CAAC;CACrC;AAED;;;GAGG;AACH,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC;IACrB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;CACtB;AAED;;;GAGG;AACH,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;IACxB,QAAQ,CAAC,MAAM,EAAE,aAAa,CAAC;IAC/B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;CACrB;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;IAC1B;;;;OAIG;IACH,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;CACtB;AAED;;;;;;;;;;;;;;;GAeG;AACH,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,IAAI,EAAE,eAAe,CAAC;IAC/B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;CAC7B;AAED;;;;;;GAMG;AACH,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,IAAI,EAAE,eAAe,CAAC;IAC/B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;CAC7B;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,IAAI,EAAE,WAAW,CAAC;IAC3B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,8EAA8E;IAC9E,QAAQ,CAAC,iBAAiB,CAAC,EAAE,MAAM,CAAC;IACpC;;;;OAIG;IACH,QAAQ,CAAC,WAAW,CAAC,EAAE,OAAO,CAAC;CAChC;AAED;;;;GAIG;AACH,MAAM,MAAM,cAAc,GACtB,gBAAgB,GAChB,oBAAoB,GACpB,WAAW,GACX,iBAAiB,GACjB,eAAe,GACf,UAAU,GACV,UAAU,GACV,SAAS,GACT,YAAY,GACZ,cAAc,GACd,kBAAkB,GAClB,kBAAkB,GAClB,cAAc,CAAC;AAEnB;;;GAGG;AACH,eAAO,MAAM,qBAAqB,+KAcxB,CAAC;AAEX,aAAa;AACb,MAAM,MAAM,kBAAkB,GAAG,CAAC,OAAO,qBAAqB,CAAC,CAAC,MAAM,CAAC,CAAC;AAIxE;;;;;GAKG;AACH,MAAM,WAAW,qBAAqB;IACpC,0CAA0C;IAC1C,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,MAAM,EAAE,cAAc,CAAC;CACjC;AAID;;;;GAIG;AACH,MAAM,MAAM,yBAAyB,GAAG,qBAAqB,GAAG,yBAAyB,CAAC;AAE1F;;;;;GAKG;AACH,MAAM,WAAW,iBAAiB;IAChC,oEAAoE;IACpE,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B;;;;;OAKG;IACH,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,wEAAwE;IACxE,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,CAAC;IACjC,kEAAkE;IAClE,QAAQ,CAAC,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAC3C;;;;;OAKG;IACH,QAAQ,CAAC,yBAAyB,CAAC,EAAE,MAAM,CAAC;CAC7C;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,IAAI,EAAE,YAAY,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,4DAA4D;IAC5D,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,0CAA0C;IAC1C,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,wBAAwB;IACxB,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,qCAAqC;IACrC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,sCAAsC;IACtC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,8BAA8B;IAC9B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,SAAS,EAAE,iBAAiB,CAAC;IACtC;;;;OAIG;IACH,QAAQ,CAAC,sBAAsB,CAAC,EAAE,MAAM,CAAC;IACzC,6EAA6E;IAC7E,QAAQ,CAAC,0BAA0B,CAAC,EAAE,MAAM,CAAC;CAC9C;AAED;;;GAGG;AACH,MAAM,WAAW,yBAAyB;IACxC,QAAQ,CAAC,IAAI,EAAE,iBAAiB,CAAC;IACjC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;CAC9B;AAsBD;;;;;;GAMG;AACH,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;CACvB;AAED;;;;;;GAMG;AACH,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;CACvB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,MAAM,WAAW,aAAa;IAC5B,oEAAoE;IACpE,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,uCAAuC;IACvC,QAAQ,CAAC,GAAG,EAAE,OAAO,GAAG,UAAU,CAAC;IACnC;;;;;OAKG;IACH,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,qCAAqC;IACrC,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IACvB,4CAA4C;IAC5C,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B,iEAAiE;IACjE,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAC7B;;;;OAIG;IACH,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;CACzB;AAED;;;;;;;;;;;;;;GAcG;AACH,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,2CAA2C;IAC3C,QAAQ,CAAC,GAAG,EAAE,QAAQ,GAAG,OAAO,GAAG,GAAG,CAAC;IACvC,8EAA8E;IAC9E,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB;;;;OAIG;IACH,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;CAC9B;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,IAAI,EAAE,WAAW,CAAC;IAC3B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAC;IACjC,QAAQ,CAAC,QAAQ,EAAE,aAAa,CAAC,eAAe,CAAC,CAAC;IAClD,QAAQ,CAAC,KAAK,EAAE,aAAa,CAAC,YAAY,CAAC,CAAC;IAC5C;;;;;OAKG;IACH,QAAQ,CAAC,MAAM,EAAE,aAAa,CAAC,aAAa,CAAC,CAAC;IAC9C;;;;;OAKG;IACH,QAAQ,CAAC,OAAO,EAAE,aAAa,CAAC,cAAc,CAAC,CAAC;IAChD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAuCG;IACH,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IACnC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;CAC/B;AAID;;;;;;;;;;GAUG;AACH,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC5B;AAED;;;GAGG;AACH,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,yEAAyE;IACzE,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;CAC1B;AAID;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,eAAO,MAAM,wBAAwB,2OAoC3B,CAAC;AAEX,aAAa;AACb,MAAM,MAAM,qBAAqB,GAAG,CAAC,OAAO,wBAAwB,CAAC,CAAC,MAAM,CAAC,CAAC;AAmC9E;;;;;;;;;;;;;;GAcG;AACH,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;CAChC;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,WAAW,qBAAqB;IACpC,SAAS,CAAC,QAAQ,EAAE,CAAC,KAAK,EAAE,eAAe,KAAK,IAAI,GAAG,MAAM,IAAI,CAAC;CACnE;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,WAAW,2BAA2B;IAC1C,QAAQ,CAAC,IAAI,EAAE,kBAAkB,CAAC;IAClC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,OAAO,EAAE,SAAS,GAAG,SAAS,CAAC;IACxC,QAAQ,CAAC,cAAc,CAAC,EAAE,qBAAqB,CAAC;IAChD;;;;;;;;;;;;;OAaG;IACH,QAAQ,CAAC,uBAAuB,CAAC,EAAE,YAAY,CAAC;CACjD;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,8BAA8B;IAC7C,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAC7B;;;;;;;;OAQG;IACH,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,kEAAkE;IAClE,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC;IAC/B,mDAAmD;IACnD,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,mEAAmE;IACnE,QAAQ,CAAC,gBAAgB,EAAE;QACzB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;QACzB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;KAC1B,CAAC;IACF;;;;OAIG;IACH,QAAQ,CAAC,iBAAiB,EAAE;QAAE,QAAQ,EAAE,CAAC,IAAI,qBAAqB,CAAC,CAAC,EAAE,MAAM;KAAE,CAAC;IAC/E;;;;;OAKG;IACH,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC;IAC7B;;;;;;OAMG;IACH,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC;;;;;;OAMG;IACH,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;CAC/B;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,WAAW,sBAAuB,SAAQ,8BAA8B;IAC5E;;;;OAIG;IACH,QAAQ,CAAC,KAAK,EAAE,4BAA4B,CAAC;IAC7C,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC5B"}
|
package/dist/computer-use.js
CHANGED
|
@@ -51,6 +51,10 @@ export const COMPUTER_ACTION_KINDS = [
|
|
|
51
51
|
"type",
|
|
52
52
|
"key",
|
|
53
53
|
"scroll",
|
|
54
|
+
"navigate",
|
|
55
|
+
"click_element",
|
|
56
|
+
"focus_element",
|
|
57
|
+
"type_into",
|
|
54
58
|
];
|
|
55
59
|
// ── Outcome taxonomy ─────────────────────────────────────────────────
|
|
56
60
|
/**
|
|
@@ -69,6 +73,17 @@ export const COMPUTER_ACTION_KINDS = [
|
|
|
69
73
|
* user_preempted — physical user input interrupted mid-dispatch
|
|
70
74
|
* platform_blocked — OS blocked synthetic input (secure password field, elevation boundary)
|
|
71
75
|
* not_supported — surface cannot execute computer use at all
|
|
76
|
+
* frame_stale — the page navigated underneath the action; the
|
|
77
|
+
* executor's frame reference is no longer valid.
|
|
78
|
+
* Distinct from session_closed (the session is
|
|
79
|
+
* still open; only the frame changed). The
|
|
80
|
+
* cloud executor retries once before surfacing
|
|
81
|
+
* this so transient redirects (Google's
|
|
82
|
+
* anti-cache `?zx=…` redirect, OAuth round-
|
|
83
|
+
* trips, AJAX-driven URL rewrites) don't reach
|
|
84
|
+
* the AI as failure. When the AI sees this
|
|
85
|
+
* reason, the page has moved twice and a
|
|
86
|
+
* re-read is required.
|
|
72
87
|
* @alpha
|
|
73
88
|
*/
|
|
74
89
|
export const COMPUTER_FAILURE_REASONS = [
|
|
@@ -82,5 +97,30 @@ export const COMPUTER_FAILURE_REASONS = [
|
|
|
82
97
|
"user_preempted",
|
|
83
98
|
"platform_blocked",
|
|
84
99
|
"not_supported",
|
|
100
|
+
// Co-browse Slice 1 — fired when the session manager's optional
|
|
101
|
+
// `coBrowseControl` machine reports `state.kind !== "motebit"` at
|
|
102
|
+
// dispatch time. Distinct from `user_preempted` (which fires for
|
|
103
|
+
// active halt) and `policy_denied` (governance-driven). The
|
|
104
|
+
// accompanying `control_state_at_denial` field on the per-action
|
|
105
|
+
// record carries the literal ControlState so verifiers replaying
|
|
106
|
+
// the log answer "what state were we in" without cross-referencing
|
|
107
|
+
// adjacent co_browse_control_changed events.
|
|
108
|
+
"not_in_control",
|
|
109
|
+
// Typed-truth-perception (motebit-computer.md §"Typed truth on
|
|
110
|
+
// results"): the page navigated mid-action so the executor's frame
|
|
111
|
+
// reference became stale. Playwright surfaces this as
|
|
112
|
+
// `Execution context was destroyed`, `frame was detached`,
|
|
113
|
+
// `Target closed`, or `Target page, context or browser has been
|
|
114
|
+
// closed`. Before this entry, those errors fell through the route
|
|
115
|
+
// handler's general catch into `platform_blocked` — the AI received
|
|
116
|
+
// an opaque server-fault and verbally interpreted it as "keystrokes
|
|
117
|
+
// aren't landing." With this reason typed, the executor retries
|
|
118
|
+
// once (recovers transient redirects without the AI seeing the
|
|
119
|
+
// race) and surfaces the failure with a name the prompt teaches
|
|
120
|
+
// against: re-read the page before retrying. Doctrine:
|
|
121
|
+
// motebit-computer.md §"Typed truth on results" + the
|
|
122
|
+
// typed-truth-perception triple (wire field + prompt clause +
|
|
123
|
+
// dispatch enforcement).
|
|
124
|
+
"frame_stale",
|
|
85
125
|
];
|
|
86
126
|
//# sourceMappingURL=computer-use.js.map
|
package/dist/computer-use.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"computer-use.js","sourceRoot":"","sources":["../src/computer-use.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsCG;
|
|
1
|
+
{"version":3,"file":"computer-use.js","sourceRoot":"","sources":["../src/computer-use.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsCG;AAwPH;;;GAGG;AACH,MAAM,CAAC,MAAM,qBAAqB,GAAG;IACnC,YAAY;IACZ,iBAAiB;IACjB,OAAO;IACP,cAAc;IACd,YAAY;IACZ,MAAM;IACN,MAAM;IACN,KAAK;IACL,QAAQ;IACR,UAAU;IACV,eAAe;IACf,eAAe;IACf,WAAW;CACH,CAAC;AA4VX,wEAAwE;AAExE;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAG;IACtC,eAAe;IACf,mBAAmB;IACnB,kBAAkB;IAClB,mBAAmB;IACnB,gBAAgB;IAChB,kBAAkB;IAClB,iBAAiB;IACjB,gBAAgB;IAChB,kBAAkB;IAClB,eAAe;IACf,gEAAgE;IAChE,kEAAkE;IAClE,iEAAiE;IACjE,4DAA4D;IAC5D,iEAAiE;IACjE,iEAAiE;IACjE,mEAAmE;IACnE,6CAA6C;IAC7C,gBAAgB;IAChB,+DAA+D;IAC/D,mEAAmE;IACnE,sDAAsD;IACtD,2DAA2D;IAC3D,gEAAgE;IAChE,kEAAkE;IAClE,oEAAoE;IACpE,oEAAoE;IACpE,gEAAgE;IAChE,+DAA+D;IAC/D,gEAAgE;IAChE,uDAAuD;IACvD,sDAAsD;IACtD,8DAA8D;IAC9D,yBAAyB;IACzB,aAAa;CACL,CAAC"}
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `EventType` canonical-registry tooling.
|
|
3
|
+
*
|
|
4
|
+
* The `EventType` enum (in `./index.ts`) is the closed vocabulary of
|
|
5
|
+
* event-log entry discriminators that flow through every motebit's
|
|
6
|
+
* append-only event substrate. Every `EventLogEntry` carries an
|
|
7
|
+
* `event_type` field; sync peers, federation participants, audit
|
|
8
|
+
* verifiers, and consolidation cycles dispatch on it. Cross-
|
|
9
|
+
* implementation drift would break interop — a motebit emitting an
|
|
10
|
+
* unknown event_type to a peer would surface as a silent
|
|
11
|
+
* type-narrowing failure on the receiving side.
|
|
12
|
+
*
|
|
13
|
+
* Promoted to a registered registry per
|
|
14
|
+
* `docs/doctrine/registry-pattern-canonical.md` on 2026-05-14 — the
|
|
15
|
+
* sixth instance after `SuiteId`, `TokenAudience`,
|
|
16
|
+
* `ContentArtifactType`, `TaskShape`, and `SensitivityLevel`. The
|
|
17
|
+
* arc validates the meta-gate's claim that adding a sixth registry
|
|
18
|
+
* is template growth, not new design.
|
|
19
|
+
*
|
|
20
|
+
* Same shape as `audience.ts`, `artifact-type.ts`, `routing.ts`'s
|
|
21
|
+
* `ALL_TASK_SHAPES`/`isTaskShape` pair, and `sensitivity.ts`'s
|
|
22
|
+
* `ALL_SENSITIVITY_LEVELS`/`isSensitivityLevel` pair.
|
|
23
|
+
*
|
|
24
|
+
* Pure deterministic data + type guard. Permissive-floor primitive
|
|
25
|
+
* per `packages/protocol/CLAUDE.md` rule 1 (closed-registry tooling
|
|
26
|
+
* is structural, not policy).
|
|
27
|
+
*/
|
|
28
|
+
import type { EventType } from "./index.js";
|
|
29
|
+
/**
|
|
30
|
+
* Canonical iteration order over `EventType`, frozen. The single
|
|
31
|
+
* source of truth for "every event type" — drift gates, exhaustive
|
|
32
|
+
* switches, sync filters, and the protocol's registry-coverage gate
|
|
33
|
+
* (`check-event-type-canonical`) all enumerate through this array.
|
|
34
|
+
*
|
|
35
|
+
* Ordered in declaration order from the enum in `index.ts`. The
|
|
36
|
+
* gate's sibling-alignment block verifies the array mirrors the
|
|
37
|
+
* enum exactly — a registry append in the enum without a
|
|
38
|
+
* corresponding array append fails CI.
|
|
39
|
+
*
|
|
40
|
+
* Same shape as `ALL_SUITE_IDS`, `ALL_TOKEN_AUDIENCES`,
|
|
41
|
+
* `ALL_CONTENT_ARTIFACT_TYPES`, `ALL_TASK_SHAPES`,
|
|
42
|
+
* `ALL_SENSITIVITY_LEVELS`. Adding an event type is intentional
|
|
43
|
+
* protocol-level work: new enum entry + new entry here + gate
|
|
44
|
+
* reference update + spec/event-log entry if wire-format-relevant.
|
|
45
|
+
*
|
|
46
|
+
* Values are the enum's string literals (not enum members) to avoid
|
|
47
|
+
* the init-order cycle the file's `import type` already documents.
|
|
48
|
+
*/
|
|
49
|
+
export declare const ALL_EVENT_TYPES: readonly EventType[];
|
|
50
|
+
/**
|
|
51
|
+
* Type guard — narrows `unknown` to `EventType`. Drift-gate-driven
|
|
52
|
+
* literal scanners use this to validate values pulled from
|
|
53
|
+
* wire-format payloads; consumers that derive event types from
|
|
54
|
+
* external sources (sync intake, federation peer payloads) call
|
|
55
|
+
* this before dispatching so an unchecked cast is a fail-open path
|
|
56
|
+
* the type system can't catch.
|
|
57
|
+
*
|
|
58
|
+
* Same shape as `isSuiteId`, `isTokenAudience`,
|
|
59
|
+
* `isContentArtifactType`, `isTaskShape`, `isSensitivityLevel`.
|
|
60
|
+
*/
|
|
61
|
+
export declare function isEventType(value: unknown): value is EventType;
|
|
62
|
+
//# sourceMappingURL=event-type.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"event-type.d.ts","sourceRoot":"","sources":["../src/event-type.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAOH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAE5C;;;;;;;;;;;;;;;;;;;GAmBG;AACH,eAAO,MAAM,eAAe,EAAE,SAAS,SAAS,EA4D/B,CAAC;AAElB;;;;;;;;;;GAUG;AACH,wBAAgB,WAAW,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,SAAS,CAE9D"}
|