@opengeni/sdk 0.57.0 → 1.0.0

package/src/interaction.ts

@@ -1,4 +1,17 @@
 import type { OpenGeniRequestOptions } from "./client";
+import { browserSessionStorageMode } from "@opengeni/contracts/browser-storage";
+import type {
+  BrowserDomReadRequest,
+  BrowserDomReadResponse,
+  BrowserTargetState,
+} from "@opengeni/contracts";
+export type {
+  BrowserDomReadRequest,
+  BrowserDomReadResponse,
+  BrowserDomSafeAttribute,
+  BrowserTargetState,
+} from "@opengeni/contracts";
+import { OpenGeniApiError } from "./errors";
 import type { RetainedArtifactReference } from "./types";
 
 /** Public Browser/Computer protocol version. Driver and provider details stay private. */
@@ -28,6 +41,83 @@ export type InteractionJsonValue =
   | InteractionJsonValue[]
   | { [key: string]: InteractionJsonValue };
 
+export type InteractionControlFailureCode =
+  | "unknown"
+  | "unsupported"
+  | "os"
+  | "not_found"
+  | "consent_required"
+  | "timeout"
+  | "draining"
+  | "protocol"
+  | "stream"
+  | "agent_offline"
+  | "fenced"
+  | "payload_too_large";
+
+export type InteractionControlFailure = {
+  layer: "connected_machine";
+  surface: "browser" | "computer";
+  code: InteractionControlFailureCode;
+  retryable: boolean;
+  outcomeUnknown: boolean;
+  requestId: string | undefined;
+  controlRequestId: string | undefined;
+  message: string;
+};
+
+/** Decodes the bounded typed control-failure details emitted by OpenGeni. Raw
+ * provider/OS detail is intentionally absent; both correlation ids are opaque. */
+export function interactionControlFailureFromError(
+  error: unknown,
+): InteractionControlFailure | null {
+  if (!(error instanceof OpenGeniApiError)) return null;
+  const details = error.details;
+  if (!details || details.interactionLayer !== "connected_machine") return null;
+  const surface = details.interactionSurface;
+  const code = details.controlFailureCode;
+  if ((surface !== "browser" && surface !== "computer") || !isControlFailureCode(code)) {
+    return null;
+  }
+  const controlRequestId = boundedOpaqueCorrelation(details.controlRequestId);
+  return {
+    layer: "connected_machine",
+    surface,
+    code,
+    retryable: error.retryable,
+    outcomeUnknown: error.outcomeUnknown,
+    requestId: error.correlationId,
+    controlRequestId,
+    message: error.message,
+  };
+}
+
+function isControlFailureCode(value: unknown): value is InteractionControlFailureCode {
+  return (
+    typeof value === "string" &&
+    [
+      "unknown",
+      "unsupported",
+      "os",
+      "not_found",
+      "consent_required",
+      "timeout",
+      "draining",
+      "protocol",
+      "stream",
+      "agent_offline",
+      "fenced",
+      "payload_too_large",
+    ].includes(value)
+  );
+}
+
+function boundedOpaqueCorrelation(value: unknown): string | undefined {
+  return typeof value === "string" && value.length <= 128 && /^[A-Za-z0-9._:-]+$/u.test(value)
+    ? value
+    : undefined;
+}
+
 export type InteractionPlacement =
   | { kind: "sandbox_group"; sandboxGroupId: string }
   | { kind: "connected_machine"; sandboxId: string }
@@ -71,8 +161,18 @@ export type AttachedBrowserDevice = {
   updatedAt: string;
 };
 
+export type AttachedBrowserBridge = {
+  enrollmentId: string;
+  state: "online" | "offline";
+  bridgeGeneration: string;
+  inventoryRevision: number;
+  connectedProfileCount: number;
+  lastSeenAt: string;
+};
+
 export type AttachedBrowserDeviceListResponse = {
   revision: number;
+  bridges: AttachedBrowserBridge[];
   devices: AttachedBrowserDevice[];
 };
 
@@ -714,6 +814,17 @@ export type BrowserObservation = {
   frameId: string | null;
   semantic: InteractionSemanticSnapshot | InteractionSemanticDiff | null;
   screenshot: RetainedArtifactReference | null;
+  viewport?:
+    | {
+        width: number;
+        height: number;
+        visualWidth: number;
+        visualHeight: number;
+        deviceScaleFactor: number;
+        maxTouchPoints: number;
+      }
+    | null
+    | undefined;
   focusedRef: string | null;
   changedRegions: InteractionRect[];
   diagnostics: InteractionDiagnosticSummary;
@@ -775,6 +886,15 @@ export type BrowserPermissionSetting = "granted" | "denied" | "prompt";
 
 export type BrowserAction =
   | { type: "navigate"; url: string }
+  | { type: "history"; direction: "back" | "forward" }
+  | { type: "activate" }
+  | {
+      type: "viewport";
+      width: number;
+      height: number;
+      mobile: boolean;
+      deviceScaleFactor?: number | undefined;
+    }
   | {
       type: "click";
       locator: BrowserLocator;
@@ -831,7 +951,11 @@ export type BrowserAction =
       timeoutMs?: number | undefined;
     };
 
-export type BrowserActionBatch = { type: "batch"; actions: BrowserAction[] };
+export type BrowserActionBatch = {
+  type: "batch";
+  actions: BrowserAction[];
+  fenceEachAction?: true | undefined;
+};
 
 export type InteractionError = {
   code:
@@ -886,6 +1010,7 @@ export type CreateBrowserSessionRequest = {
   name?: string | undefined;
   initialUrl?: string | undefined;
   headless?: boolean | undefined;
+  storageMode?: "private_profile" | "ephemeral_context" | undefined;
   engine?: "chromium" | "lightpanda" | undefined;
   placement?: InteractionPlacement | undefined;
   identityId?: string | undefined;
@@ -930,13 +1055,16 @@ export type InteractionFrameStreamAttachment<RelayKind extends 3 | 4 = 3 | 4> =
 
 export type ComputerFrameStreamAttachment =
   | InteractionFrameStreamAttachment<4>
-  | { kind: "direct_rfb"; url: string; protocols: string[] };
+  | { kind: "direct_rfb"; url: string; protocols: string[]; inputAllowed: boolean };
 
 export type BrowserSessionAttachment = {
   browserSessionId: string;
   controllerGeneration: string;
   targetId: string;
   stream: InteractionFrameStreamAttachment<3>;
+  /** Negotiated from this attachment’s live controller, absent on older helpers. */
+  fencedInputBatches?: true | undefined;
+  focusedInputObservations?: true | undefined;
   expiresAt: string;
 };
 
@@ -952,7 +1080,10 @@ export type BrowserActionRequest = {
   expectedTargetGeneration: string;
   expectedDocumentGeneration: string | null;
   expectedFrameId: string | null;
-  observationMode?: "full" | "none" | undefined;
+  /** `input` is negotiated by focusedInputObservations on a live attachment.
+   * It returns optional native-select metadata after a pointer click, and null
+   * for ordinary input. Use full for agent observations. */
+  observationMode?: "full" | "none" | "input" | undefined;
   action: BrowserAction | BrowserActionBatch;
 };
 
@@ -986,6 +1117,8 @@ export type ComputerSessionCapabilities = {
   keyboardInput: boolean;
   clipboard: boolean;
   backgroundActions: boolean;
+  /** Raw window input stays in the background. Absent means foreground-only. */
+  backgroundInput?: boolean | undefined;
   parallelApps: boolean;
 };
 
@@ -1133,10 +1266,19 @@ export type ComputerSessionAttachment = {
   computerSessionId: string;
   controllerGeneration: string;
   targetId: string;
+  /** Viewer posture when attached; actions still reauthorize the live source. */
+  inputAllowed?: boolean | undefined;
   stream: ComputerFrameStreamAttachment;
   expiresAt: string;
 };
 
+/** Caller-specific hint only; every action still requires live authorization. */
+export type ComputerSessionInputPosture = {
+  computerSessionId: string;
+  controllerGeneration: string;
+  inputAllowed: boolean;
+};
+
 export type ComputerSessionAttachmentRequest = {
   targetId: string;
   expiresInSeconds?: number | undefined;
@@ -1387,6 +1529,26 @@ export interface InteractionTransport {
     targetId: string,
     options?: OpenGeniRequestOptions,
   ): Promise<BrowserObservation>;
+  getBrowserTargetState(
+    workspaceId: string,
+    browserSessionId: string,
+    targetId: string,
+    options?: OpenGeniRequestOptions,
+  ): Promise<BrowserTargetState>;
+  readBrowserDom(
+    workspaceId: string,
+    browserSessionId: string,
+    targetId: string,
+    request: BrowserDomReadRequest,
+    options?: OpenGeniRequestOptions,
+  ): Promise<BrowserDomReadResponse>;
+  captureBrowserTarget(
+    workspaceId: string,
+    browserSessionId: string,
+    targetId: string,
+    options?: OpenGeniRequestOptions,
+    captureOptions?: BrowserScreenshotOptions,
+  ): Promise<BrowserFrame>;
   actInBrowser(
     workspaceId: string,
     browserSessionId: string,
@@ -1449,6 +1611,11 @@ export interface InteractionTransport {
     computerSessionId: string,
     options?: OpenGeniRequestOptions,
   ): Promise<ComputerSession>;
+  getComputerInputPosture?(
+    workspaceId: string,
+    computerSessionId: string,
+    options?: OpenGeniRequestOptions,
+  ): Promise<ComputerSessionInputPosture>;
   readComputerClipboard(
     workspaceId: string,
     computerSessionId: string,
@@ -1470,6 +1637,12 @@ export interface InteractionTransport {
     targetId: string,
     options?: OpenGeniRequestOptions,
   ): Promise<ComputerObservation>;
+  captureComputerTarget(
+    workspaceId: string,
+    computerSessionId: string,
+    targetId: string,
+    options?: OpenGeniRequestOptions,
+  ): Promise<ComputerFrame>;
   actInComputer(
     workspaceId: string,
     computerSessionId: string,
@@ -1544,6 +1717,7 @@ export type CurrentOrOpenBrowserOptions = {
   name?: string | undefined;
   initialUrl?: string | undefined;
   headless?: boolean | undefined;
+  storageMode?: "private_profile" | "ephemeral_context" | undefined;
   placement?: InteractionPlacement | undefined;
   identityId?: string | undefined;
   baseRevisionId?: string | undefined;
@@ -1964,7 +2138,13 @@ export class BrowserSessionCollection {
   async currentOrOpen(options: CurrentOrOpenBrowserOptions): Promise<BrowserSessionResource> {
     const requestOptions = options.signal ? { signal: options.signal } : {};
     const listed = await this.list(options.workspaceId, requestOptions);
-    const current = newestRelevantBrowser(listed.sessions, options.associationSessionId);
+    const current = newestRelevantBrowser(
+      listed.sessions.filter(
+        (session) =>
+          browserSessionStorageMode(session) === (options.storageMode ?? "private_profile"),
+      ),
+      options.associationSessionId,
+    );
     if (current) {
       const resource = this.session(options.workspaceId, current.id);
       if (current.lifecycle === "suspended") {
@@ -1983,6 +2163,7 @@ export class BrowserSessionCollection {
         ...(options.name !== undefined ? { name: options.name } : {}),
         ...(options.initialUrl !== undefined ? { initialUrl: options.initialUrl } : {}),
         ...(options.headless !== undefined ? { headless: options.headless } : {}),
+        ...(options.storageMode !== undefined ? { storageMode: options.storageMode } : {}),
         ...(options.placement !== undefined ? { placement: options.placement } : {}),
         ...(options.identityId !== undefined ? { identityId: options.identityId } : {}),
         ...(options.baseRevisionId !== undefined ? { baseRevisionId: options.baseRevisionId } : {}),
@@ -2027,6 +2208,49 @@ export class BrowserSessionResource {
     return await this.transport.observeBrowserTarget(this.workspaceId, this.id, targetId, options);
   }
 
+  async targetState(
+    targetId: string,
+    options: OpenGeniRequestOptions = {},
+  ): Promise<BrowserTargetState> {
+    return await this.transport.getBrowserTargetState(this.workspaceId, this.id, targetId, options);
+  }
+
+  async readDom(
+    targetId: string,
+    request: BrowserDomReadRequest,
+    options: OpenGeniRequestOptions = {},
+  ): Promise<BrowserDomReadResponse> {
+    return await this.transport.readBrowserDom(
+      this.workspaceId,
+      this.id,
+      targetId,
+      request,
+      options,
+    );
+  }
+
+  async capture(
+    targetId: string,
+    options: OpenGeniRequestOptions = {},
+    captureOptions: BrowserScreenshotOptions = {},
+  ): Promise<BrowserFrame> {
+    return await this.transport.captureBrowserTarget(
+      this.workspaceId,
+      this.id,
+      targetId,
+      options,
+      captureOptions,
+    );
+  }
+
+  async screenshot(
+    targetId: string,
+    captureOptions: BrowserScreenshotOptions = {},
+    options: OpenGeniRequestOptions = {},
+  ): Promise<BrowserFrame> {
+    return await this.capture(targetId, options, captureOptions);
+  }
+
   async act(
     request: BrowserActionRequest,
     options: OpenGeniRequestOptions = {},
@@ -2314,6 +2538,10 @@ export class ComputerSessionResource {
     return await this.transport.observeComputerTarget(this.workspaceId, this.id, targetId, options);
   }
 
+  async capture(targetId: string, options: OpenGeniRequestOptions = {}): Promise<ComputerFrame> {
+    return await this.transport.captureComputerTarget(this.workspaceId, this.id, targetId, options);
+  }
+
   async act(
     request: ComputerActionRequest,
     options: OpenGeniRequestOptions = {},
@@ -2340,6 +2568,13 @@ export class ComputerSessionResource {
     return await this.transport.attachComputerSession(this.workspaceId, this.id, request, options);
   }
 
+  async inputPosture(options: OpenGeniRequestOptions = {}): Promise<ComputerSessionInputPosture> {
+    if (!this.transport.getComputerInputPosture) {
+      throw new Error("Computer input posture requires a current interaction transport.");
+    }
+    return await this.transport.getComputerInputPosture(this.workspaceId, this.id, options);
+  }
+
   async heartbeat(options: OpenGeniRequestOptions = {}): Promise<ComputerSessionHeartbeatResponse> {
     return await this.transport.heartbeatComputerSession(this.workspaceId, this.id, options);
   }
@@ -2407,6 +2642,14 @@ export type BrowserFrameMetadata = {
 
 export type BrowserFrame = BrowserFrameMetadata & { data: Uint8Array };
 
+export type BrowserScreenshotOptions = {
+  /** Capture the document content instead of the visible viewport. */
+  fullPage?: boolean;
+  format?: "jpeg" | "png";
+  /** JPEG quality, from 1 to 100. Ignored for PNG. */
+  quality?: number;
+};
+
 export type ComputerFrameMetadata = {
   frameId: string;
   computerSessionId: string;
@@ -2439,6 +2682,7 @@ export function browserFrameSocketUrl(
   else if (url.protocol !== "ws:" && url.protocol !== "wss:") {
     throw new Error("browser frame attachment did not contain an HTTP or WebSocket URL");
   }
+  upgradeMixedContentWebSocket(url);
   if (options.format !== undefined) url.searchParams.set("format", options.format);
   setBoundedIntegerQuery(url, "quality", options.quality, 1, 100);
   setBoundedIntegerQuery(url, "maxWidth", options.maxWidth, 1, 4_096);
@@ -2461,6 +2705,7 @@ export function computerFrameSocketUrl(
   else if (url.protocol !== "ws:" && url.protocol !== "wss:") {
     throw new Error("computer frame attachment did not contain an HTTP or WebSocket URL");
   }
+  upgradeMixedContentWebSocket(url);
   if (options.format !== undefined) url.searchParams.set("format", options.format);
   setBoundedIntegerQuery(url, "quality", options.quality, 1, 100);
   setBoundedIntegerQuery(url, "maxWidth", options.maxWidth, 1, 4_096);
@@ -2701,6 +2946,28 @@ export function parseComputerFrameMetadata(value: unknown): ComputerFrameMetadat
   return metadata;
 }
 
+export function decodeComputerFrameMetadataHeader(value: string): ComputerFrameMetadata {
+  if (!value || value.length > 32 * 1024 || !/^[A-Za-z0-9_-]+$/u.test(value)) {
+    throw new Error("computer frame metadata header is invalid");
+  }
+  const normalized = value.replace(/-/gu, "+").replace(/_/gu, "/");
+  const padded = normalized.padEnd(Math.ceil(normalized.length / 4) * 4, "=");
+  let decoded: string;
+  try {
+    decoded = atob(padded);
+  } catch {
+    throw new Error("computer frame metadata header is invalid");
+  }
+  const bytes = Uint8Array.from(decoded, (character) => character.charCodeAt(0));
+  let metadata: unknown;
+  try {
+    metadata = JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(bytes));
+  } catch {
+    throw new Error("computer frame metadata header is invalid");
+  }
+  return parseComputerFrameMetadata(metadata);
+}
+
 function newestRelevantBrowser(
   sessions: readonly BrowserSession[],
   associationSessionId: string,
@@ -2754,6 +3021,11 @@ function computerRelevance(session: ComputerSession, associationSessionId: strin
   return Math.max(association, Number.isFinite(used) ? used : 0);
 }
 
+function upgradeMixedContentWebSocket(url: URL): void {
+  const protocol = (globalThis as { location?: { protocol?: string } }).location?.protocol;
+  if (protocol === "https:" && url.protocol === "ws:") url.protocol = "wss:";
+}
+
 function setBoundedIntegerQuery(
   url: URL,
   name: string,

package/src/knowledge.ts

@@ -0,0 +1,72 @@
+export type {
+  AgentLearningMode,
+  AgentLearningCategory,
+  AgentLearningDefaults,
+  AgentLearningOverrides,
+  AgentLearningContext,
+  AgentLearningOverridePatch,
+  AgentLearningSettingsRecord,
+  AgentLearningOverrideRecord,
+  SaveAgentLearningSettingsRequest,
+  KnowledgeFilePreparationResult,
+  KnowledgeSavePreparationRequest,
+  KnowledgeSavePreparationResponse,
+  KnowledgeOriginalFileDownload,
+  KnowledgeEntryKind,
+  KnowledgeEntryScope,
+  KnowledgeIndexStatus,
+  KnowledgeEntryContent,
+  KnowledgeEntrySource,
+  KnowledgeEntryEvidence,
+  KnowledgeEntryRelationship,
+  KnowledgeEntryRecord,
+  KnowledgeEntrySummary,
+  KnowledgeEntrySaveRequest,
+  KnowledgeEntryReviewRequest,
+  KnowledgeEntryBatchReviewRequest,
+  KnowledgeEntryRestoreRequest,
+  KnowledgeEntryWriteReceipt,
+  KnowledgeEntryListRequest,
+  KnowledgeEntryListResponse,
+  KnowledgeReviewBatch,
+  KnowledgeReviewBatchListRequest,
+  KnowledgeReviewBatchListResponse,
+  AgentInstructionReviewRequest,
+  AgentInstructionReceipt,
+  AgentInstructionReviewItem,
+  AgentInstructionReviewListResponse,
+} from "@opengeni/contracts";
+
+import type { OpenGeniClient } from "./client";
+import type {
+  KnowledgeSavePreparationRequest,
+  KnowledgeSavePreparationResponse,
+  KnowledgeFilePreparationResult,
+} from "@opengeni/contracts";
+
+/** Agent-only source preparation is a focused transport surface, outside the browser client. */
+export async function prepareKnowledgeFile(
+  client: Pick<OpenGeniClient, "requestJson">,
+  workspaceId: string,
+  fileId: string,
+  purpose?: "evidence" | "reference",
+): Promise<KnowledgeFilePreparationResult> {
+  return client.requestJson(
+    "POST",
+    `/v1/workspaces/${encodeURIComponent(workspaceId)}/knowledge/files/${encodeURIComponent(fileId)}/prepare`,
+    purpose ? { purpose } : undefined,
+  );
+}
+
+/** Read-only collection/duplicate discovery for a live selected agent or human Knowledge reviewer. */
+export async function prepareKnowledgeSave(
+  client: Pick<OpenGeniClient, "requestJson">,
+  workspaceId: string,
+  request: KnowledgeSavePreparationRequest,
+): Promise<KnowledgeSavePreparationResponse> {
+  return client.requestJson(
+    "POST",
+    `/v1/workspaces/${encodeURIComponent(workspaceId)}/knowledge/entries/prepare-save`,
+    request,
+  );
+}