@opengeni/contracts 0.50.0 → 1.0.0

package/src/interaction.ts

@@ -45,6 +45,13 @@ const opaqueGeneration = z
   .min(1)
   .max(256)
   .regex(/^[A-Za-z0-9][A-Za-z0-9._:-]*$/u);
+// Native bridge generations use unpadded base64url; both - and _ can lead.
+// Keep the bytes unchanged: generation equality fences old connections.
+const attachedBridgeGeneration = z
+  .string()
+  .min(1)
+  .max(256)
+  .regex(/^[A-Za-z0-9_-][A-Za-z0-9._:-]*$/u);
 const boundedOpaqueId = z
   .string()
   .min(1)
@@ -202,6 +209,11 @@ export const BrowserSessionCapabilities = z
   .strict();
 export type BrowserSessionCapabilities = z.infer<typeof BrowserSessionCapabilities>;
 
+/** Versioned persistent discriminator for experimental disposable contexts. */
+export { EPHEMERAL_CHROMIUM_DRIVER_ID, browserSessionStorageMode } from "./browser-storage";
+export const BrowserStorageMode = z.enum(["private_profile", "ephemeral_context"]);
+export type BrowserStorageMode = z.infer<typeof BrowserStorageMode>;
+
 export const BrowserSession = z
   .object({
     id: z.string().uuid(),
@@ -309,7 +321,7 @@ export type AttachedBrowserDeviceAnnouncement = z.infer<typeof AttachedBrowserDe
 
 export const AttachedBrowserInventorySnapshot = z
   .object({
-    bridgeGeneration: opaqueGeneration,
+    bridgeGeneration: attachedBridgeGeneration,
     revision: z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER),
     devices: z.array(AttachedBrowserDeviceAnnouncement).max(10_000),
   })
@@ -364,9 +376,25 @@ export const AttachedBrowserDevice = z
   });
 export type AttachedBrowserDevice = z.infer<typeof AttachedBrowserDevice>;
 
+/** One enrolled machine agent currently reporting its browser-bridge inventory.
+ * A bridge with zero devices is operational but has no Chrome profile connected
+ * through the OpenGeni extension yet. */
+export const AttachedBrowserBridge = z
+  .object({
+    enrollmentId: z.string().uuid(),
+    state: z.enum(["online", "offline"]),
+    bridgeGeneration: attachedBridgeGeneration,
+    inventoryRevision: z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER),
+    connectedProfileCount: z.number().int().nonnegative().max(10_000),
+    lastSeenAt: z.string().datetime({ offset: true }),
+  })
+  .strict();
+export type AttachedBrowserBridge = z.infer<typeof AttachedBrowserBridge>;
+
 export const AttachedBrowserDeviceListResponse = z
   .object({
     revision: z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER),
+    bridges: z.array(AttachedBrowserBridge).max(10_000),
     devices: z.array(AttachedBrowserDevice).max(10_000),
   })
   .strict();
@@ -1800,6 +1828,18 @@ export const BrowserObservation = z
     frameId: opaqueGeneration.nullable(),
     semantic: z.union([InteractionSemanticSnapshot, InteractionSemanticDiff]).nullable(),
     screenshot: RetainedArtifactReferenceSchema.nullable(),
+    viewport: z
+      .object({
+        width: z.number().int().positive(),
+        height: z.number().int().positive(),
+        visualWidth: z.number().positive(),
+        visualHeight: z.number().positive(),
+        deviceScaleFactor: z.number().positive(),
+        maxTouchPoints: z.number().int().nonnegative(),
+      })
+      .strict()
+      .nullable()
+      .optional(),
     focusedRef: boundedOpaqueId.nullable(),
     changedRegions: z.array(InteractionRect).max(INTERACTION_MAX_CHANGED_NODES),
     diagnostics: InteractionDiagnosticSummary,
@@ -1868,6 +1908,90 @@ export const BrowserLocator = z.discriminatedUnion("kind", [
 ]);
 export type BrowserLocator = z.infer<typeof BrowserLocator>;
 
+/** Cheap causal state for a target. This is intentionally independent of an
+ * accessibility observation, which can require a full tree for every frame. */
+export const BrowserTargetState = z
+  .object({
+    browserSessionId: z.string().uuid(),
+    controllerGeneration: opaqueGeneration,
+    targetId: boundedOpaqueId,
+    targetGeneration: opaqueGeneration,
+    documentGeneration: opaqueGeneration.nullable(),
+    frameId: opaqueGeneration.nullable(),
+  })
+  .strict();
+export type BrowserTargetState = z.infer<typeof BrowserTargetState>;
+
+export const BrowserDomSafeAttribute = z.enum([
+  "href",
+  "src",
+  "alt",
+  "title",
+  "role",
+  "aria-label",
+  "aria-expanded",
+  "aria-checked",
+  "aria-selected",
+  "placeholder",
+  "type",
+  "name",
+  "data-testid",
+]);
+export type BrowserDomSafeAttribute = z.infer<typeof BrowserDomSafeAttribute>;
+
+export const BrowserDomReadSelector = z
+  .string()
+  .min(1)
+  .max(8_192)
+  .regex(/^[A-Za-z0-9_#.,\s>*-]+$/u);
+
+/** DOM extraction avoids content-matching locators, which can probe redacted text. */
+export const BrowserDomReadLocator = z.discriminatedUnion("kind", [
+  z.object({ kind: z.literal("ref"), ref: boundedOpaqueId }).strict(),
+  z.object({ kind: z.literal("placeholder"), text: z.string().min(1).max(8_192) }).strict(),
+  z.object({ kind: z.literal("test_id"), value: z.string().min(1).max(2_048) }).strict(),
+  z.object({ kind: z.literal("css"), selector: BrowserDomReadSelector }).strict(),
+]);
+export type BrowserDomReadLocator = z.infer<typeof BrowserDomReadLocator>;
+
+const BrowserDomReadFence = z.object({
+  expectedTargetGeneration: opaqueGeneration,
+  expectedDocumentGeneration: opaqueGeneration,
+  expectedFrameId: opaqueGeneration,
+});
+
+/** A fixed, read-only query. The caller never supplies executable JavaScript.
+ * CSS reads accept only simple tag, class, id, descendant, and child selectors
+ * so predicate queries cannot probe protected attribute values. CSS count is
+ * scoped to the target's main document; child frames and shadow roots are not
+ * part of its count. */
+export const BrowserDomReadRequest = z.discriminatedUnion("kind", [
+  BrowserDomReadFence.extend({
+    kind: z.literal("element"),
+    locator: BrowserDomReadLocator,
+    attributes: z.array(BrowserDomSafeAttribute).max(13).optional(),
+    maxChars: z.number().int().min(1).max(32_768).optional(),
+  }).strict(),
+  BrowserDomReadFence.extend({
+    kind: z.literal("count"),
+    selector: BrowserDomReadSelector,
+  }).strict(),
+]);
+export type BrowserDomReadRequest = z.infer<typeof BrowserDomReadRequest>;
+
+export const BrowserDomReadResponse = BrowserTargetState.extend({
+  kind: z.enum(["element", "count"]),
+  count: z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER),
+  text: z.string().max(32_768).nullable().optional(),
+  value: z.string().max(32_768).nullable().optional(),
+  attributes: z
+    .partialRecord(BrowserDomSafeAttribute, z.string().max(32_768).nullable())
+    .optional(),
+  redacted: z.enum(["password", "payment", "private", "policy"]).nullable().optional(),
+  truncated: z.boolean(),
+}).strict();
+export type BrowserDomReadResponse = z.infer<typeof BrowserDomReadResponse>;
+
 /** Provider-neutral locator for native accessibility trees. Native automation
  * identifiers are intentionally distinct from DOM selectors/test ids: AX,
  * AT-SPI, and UIA adapters resolve them inside the exact observed target. */
@@ -1926,6 +2050,17 @@ export type BrowserPermissionSetting = z.infer<typeof BrowserPermissionSetting>;
 
 const browserActionVariants = [
   z.object({ type: z.literal("navigate"), url: boundedUrl }).strict(),
+  z.object({ type: z.literal("history"), direction: z.enum(["back", "forward"]) }).strict(),
+  z.object({ type: z.literal("activate") }).strict(),
+  z
+    .object({
+      type: z.literal("viewport"),
+      width: z.number().int().min(240).max(4_096),
+      height: z.number().int().min(240).max(4_096),
+      mobile: z.boolean(),
+      deviceScaleFactor: z.number().min(0.5).max(4).optional(),
+    })
+    .strict(),
   z
     .object({
       type: z.literal("click"),
@@ -2146,6 +2281,8 @@ export const BrowserActionBatch = z
   .object({
     type: z.literal("batch"),
     actions: z.array(BrowserAction).min(1).max(INTERACTION_MAX_ACTIONS_PER_BATCH),
+    /** Revalidate the original document before each action; never follow navigation. */
+    fenceEachAction: z.literal(true).optional(),
   })
   .strict()
   .superRefine((batch, context) => {
@@ -2162,6 +2299,41 @@ export const BrowserActionBatch = z
   });
 export type BrowserActionBatch = z.infer<typeof BrowserActionBatch>;
 
+/** Secret-safe projection of a connected-machine control failure carried in a
+ * standard API error envelope. The outer envelope owns HTTP status, retryability,
+ * and public request correlation; this preserves the exact failing control class
+ * and optional inner request id without exposing subjects, local paths, or tokens. */
+export const InteractionControlFailureCode = z.enum([
+  "unknown",
+  "unsupported",
+  "os",
+  "not_found",
+  "consent_required",
+  "timeout",
+  "draining",
+  "protocol",
+  "stream",
+  "agent_offline",
+  "fenced",
+  "payload_too_large",
+]);
+export type InteractionControlFailureCode = z.infer<typeof InteractionControlFailureCode>;
+
+export const InteractionControlFailureDetails = z
+  .object({
+    interactionLayer: z.literal("connected_machine"),
+    interactionSurface: z.enum(["browser", "computer"]),
+    controlFailureCode: InteractionControlFailureCode,
+    controlRequestId: z
+      .string()
+      .min(1)
+      .max(128)
+      .regex(/^[A-Za-z0-9._:-]+$/u)
+      .optional(),
+  })
+  .strict();
+export type InteractionControlFailureDetails = z.infer<typeof InteractionControlFailureDetails>;
+
 export const InteractionError = z
   .object({
     code: z.enum([
@@ -2205,8 +2377,10 @@ export const BrowserActionCommand = z
     expectedFrameId: opaqueGeneration.nullable(),
     actor: InteractionActor,
     /** Human live-control surfaces already receive the resulting pixels. They
-     * may omit the expensive semantic snapshot; agent actions keep it. */
-    observationMode: z.enum(["full", "none"]).optional(),
+     * may omit the expensive semantic snapshot; agent actions keep it. The
+     * negotiated input mode only observes a focused native dropdown after a
+     * pointer click, and otherwise returns no observation. */
+    observationMode: z.enum(["full", "none", "input"]).optional(),
     action: z.union([BrowserAction, BrowserActionBatch]),
   })
   .strict();
@@ -2334,6 +2508,7 @@ export const CreateBrowserSessionRequest = z
     name: z.string().trim().min(1).max(200).optional(),
     initialUrl: boundedUrl.optional(),
     headless: z.boolean().default(true),
+    storageMode: BrowserStorageMode.default("private_profile"),
     /** Managed engine choice. Attached Chrome continues to derive its engine
      * from the selected device rather than accepting an impersonated value. */
     engine: z.enum(["chromium", "lightpanda"]).default("chromium"),
@@ -2345,6 +2520,23 @@ export const CreateBrowserSessionRequest = z
   })
   .strict()
   .superRefine((value, context) => {
+    if (
+      value.storageMode === "ephemeral_context" &&
+      (!value.headless ||
+        value.engine !== "chromium" ||
+        value.identityId ||
+        value.baseRevisionId ||
+        value.networkRouteId ||
+        value.linkedComputerSessionId ||
+        (value.placement && value.placement.kind !== "sandbox_group"))
+    ) {
+      context.addIssue({
+        code: "custom",
+        path: ["storageMode"],
+        message:
+          "ephemeral contexts require headless managed sandbox Chromium without identity, revision, route or Computer",
+      });
+    }
     if (value.baseRevisionId && !value.identityId) {
       context.addIssue({
         code: "custom",
@@ -2502,11 +2694,16 @@ const DirectInteractionFrameStreamAttachment = z
   })
   .strict();
 
+/** Native tunnels offer `binary` + RFB + bearer (3). Docker and OpenSandbox
+ * hide those on the API interaction frame-proxy, so the viewer sees `binary` +
+ * the proxy grant (2). */
 const DirectComputerRfbAttachment = z
   .object({
     kind: z.literal("direct_rfb"),
     url: boundedUrl,
-    protocols: z.array(z.string().min(1).max(2_048)).length(3),
+    protocols: z.array(z.string().min(1).max(2_048)).min(2).max(3),
+    /** Absent on an older API means pixel viewing only. */
+    inputAllowed: z.boolean().default(false),
   })
   .strict();
 
@@ -2554,6 +2751,8 @@ export const BrowserSessionAttachment = z
     controllerGeneration: opaqueGeneration,
     targetId: boundedOpaqueId,
     stream: BrowserFrameStreamAttachment,
+    fencedInputBatches: z.literal(true).optional(),
+    focusedInputObservations: z.literal(true).optional(),
     expiresAt: z.string().datetime({ offset: true }),
   })
   .strict();
@@ -2578,7 +2777,7 @@ export const BrowserActionRequest = z
     expectedTargetGeneration: opaqueGeneration,
     expectedDocumentGeneration: opaqueGeneration.nullable(),
     expectedFrameId: opaqueGeneration.nullable(),
-    observationMode: z.enum(["full", "none"]).default("full"),
+    observationMode: z.enum(["full", "none", "input"]).default("full"),
     action: z.union([BrowserAction, BrowserActionBatch]),
   })
   .strict();
@@ -2742,6 +2941,8 @@ export const ComputerSessionCapabilities = z
     keyboardInput: z.boolean(),
     clipboard: z.boolean(),
     backgroundActions: z.boolean(),
+    /** Raw window input stays in the background. Absent means foreground-only. */
+    backgroundInput: z.boolean().optional(),
     parallelApps: z.boolean(),
   })
   .strict();
@@ -3106,12 +3307,26 @@ export const ComputerSessionAttachment = z
     computerSessionId: z.string().uuid(),
     controllerGeneration: opaqueGeneration,
     targetId: boundedOpaqueId,
+    // Viewer posture only. Canonical actions still require live source authority.
+    // Older API responses omit it; their existing action authorization remains.
+    inputAllowed: z.boolean().optional(),
     stream: ComputerFrameStreamAttachment,
     expiresAt: z.string().datetime({ offset: true }),
   })
   .strict();
 export type ComputerSessionAttachment = z.infer<typeof ComputerSessionAttachment>;
 
+/** Caller-specific human input posture, independent of media/native capability.
+ * It is a current-controller hint; every action still reauthorizes. */
+export const ComputerSessionInputPosture = z
+  .object({
+    computerSessionId: z.string().uuid(),
+    controllerGeneration: opaqueGeneration,
+    inputAllowed: z.boolean(),
+  })
+  .strict();
+export type ComputerSessionInputPosture = z.infer<typeof ComputerSessionInputPosture>;
+
 export const ComputerSessionAttachmentRequest = z
   .object({
     targetId: boundedOpaqueId,