@opengeni/sdk 7.1.0 → 7.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. package/README.md +6 -0
  2. package/dist/artifacts.js +5 -5
  3. package/dist/browser.d.ts +1 -1
  4. package/dist/browser.js +3 -3
  5. package/dist/chat/index.js +6 -6
  6. package/dist/{chunk-MG4QFBKR.js → chunk-B7S4YVGX.js} +2 -2
  7. package/dist/{chunk-LSO66LQA.js → chunk-E5WAYGHT.js} +1 -1
  8. package/dist/chunk-E5WAYGHT.js.map +1 -0
  9. package/dist/{chunk-YHF66FWZ.js → chunk-MPN5UJWE.js} +2 -2
  10. package/dist/{chunk-Y7DRKEFZ.js → chunk-NP7DDXHK.js} +94 -5
  11. package/dist/chunk-NP7DDXHK.js.map +1 -0
  12. package/dist/{chunk-ZN6WWXAX.js → chunk-PCVFOBSV.js} +2 -2
  13. package/dist/{chunk-X6Y2JCAN.js → chunk-RAG3P2UV.js} +2 -2
  14. package/dist/{chunk-QTBAMHEF.js → chunk-YUOPVPZW.js} +25 -1
  15. package/dist/chunk-YUOPVPZW.js.map +1 -0
  16. package/dist/client.d.ts +10 -2
  17. package/dist/core.js +4 -4
  18. package/dist/document-authority.js +4 -4
  19. package/dist/editable-artifacts.js +1 -1
  20. package/dist/index.d.ts +1 -1
  21. package/dist/index.js +7 -7
  22. package/dist/interaction.d.ts +35 -0
  23. package/dist/interaction.js +1 -1
  24. package/dist/knowledge.d.ts +1 -1
  25. package/dist/knowledge.js.map +1 -1
  26. package/dist/session-message-search.d.ts +12 -0
  27. package/dist/site-browser-runtime.gen.d.ts +1 -1
  28. package/dist/site-document.js +1 -1
  29. package/dist/site-document.js.map +1 -1
  30. package/dist/site.js +4 -4
  31. package/dist/types.d.ts +3 -3
  32. package/package.json +2 -2
  33. package/src/browser.ts +2 -0
  34. package/src/client.ts +146 -2
  35. package/src/index.ts +2 -0
  36. package/src/interaction.ts +102 -0
  37. package/src/knowledge.ts +1 -0
  38. package/src/session-message-search.ts +8 -0
  39. package/src/site-browser-runtime.gen.ts +1 -1
  40. package/src/types.ts +5 -0
  41. package/dist/chunk-LSO66LQA.js.map +0 -1
  42. package/dist/chunk-QTBAMHEF.js.map +0 -1
  43. package/dist/chunk-Y7DRKEFZ.js.map +0 -1
  44. /package/dist/{chunk-MG4QFBKR.js.map → chunk-B7S4YVGX.js.map} +0 -0
  45. /package/dist/{chunk-YHF66FWZ.js.map → chunk-MPN5UJWE.js.map} +0 -0
  46. /package/dist/{chunk-ZN6WWXAX.js.map → chunk-PCVFOBSV.js.map} +0 -0
  47. /package/dist/{chunk-X6Y2JCAN.js.map → chunk-RAG3P2UV.js.map} +0 -0
package/dist/site.js CHANGED
@@ -18,11 +18,11 @@ import {
18
18
  sanitizeOpenGeniSiteToolCallRequest,
19
19
  serveSiteHttp,
20
20
  siteSessionPath
21
- } from "./chunk-YHF66FWZ.js";
21
+ } from "./chunk-MPN5UJWE.js";
22
22
  import "./chunk-FMLDFJ2Q.js";
23
- import "./chunk-Y7DRKEFZ.js";
24
- import "./chunk-LSO66LQA.js";
25
- import "./chunk-QTBAMHEF.js";
23
+ import "./chunk-NP7DDXHK.js";
24
+ import "./chunk-E5WAYGHT.js";
25
+ import "./chunk-YUOPVPZW.js";
26
26
  import "./chunk-OO5LPTO7.js";
27
27
  export {
28
28
  OPENGENI_SITE_BRIDGE_BOOTSTRAP_GLOBAL,
package/dist/types.d.ts CHANGED
@@ -1,5 +1,5 @@
1
1
  import type { WorkspaceTranscriptionPolicy } from "./transcription.js";
2
- export type { SessionMessageSearchRequest, SessionMessageSearchMatch, SessionMessageSearchResponse, } from "./session-message-search.js";
2
+ export type { SessionMessageSearchRequest, SessionMessageSearchMatch, SessionMessageSearchResponse, SessionMessagePreview, SessionMessagePreviewReference, } from "./session-message-search.js";
3
3
  export type BundledSkillId = "builtin:opengeni-help" | "builtin:opengeni-client" | "builtin:opengeni-visualize" | "builtin:document-parsing" | "builtin:opengeni-skills" | "builtin:opengeni-projects" | "builtin:opengeni-documents" | "builtin:opengeni-spreadsheets" | "builtin:opengeni-presentations" | "builtin:opengeni-sites" | "builtin:opengeni-video-generation";
4
4
  export type CodexRealtimeWebrtcVersion = "v3";
5
5
  export type CodexRealtimeVoice = "juniper" | "maple" | "spruce" | "ember" | "vale" | "breeze" | "arbor" | "sol" | "cove";
@@ -2456,7 +2456,7 @@ export type KnownPermission = (typeof KNOWN_PERMISSIONS)[number];
2456
2456
  * can introduce permissions without breaking older SDK consumers.
2457
2457
  */
2458
2458
  export type Permission = KnownPermission | (string & {});
2459
- export type FirstPartyMcpToolName = "knowledge_search" | "knowledge_prepare_save" | "knowledge_get" | "knowledge_browse" | "knowledge_save" | "knowledge_retain_file" | "knowledge_retain_message" | "knowledge_archive" | "instruction_policy_save" | "instruction_policy_get" | "set_session_title" | "goal_set" | "goal_update" | "goal_progress" | "wait_for_input" | "goal_complete" | "goal_pause" | "goal_resume" | "memory_search" | "memory_save" | "memory_correct" | "preference_registry_summary" | "preference_registry_get" | "task_notes_list" | "task_note_save" | "task_note_archive" | "task_note_replace" | "work_claim_upsert" | "work_claim_release" | "knowledge_propose" | "knowledge_correct" | "task_note_promote_knowledge" | "task_note_promote_instruction_policy" | "task_note_promote_preference" | "instruction_policy_propose" | "preference_propose" | "remember" | "remember_confirm" | "company_profile_propose" | "company_profile_confirm" | "sandboxes_list" | "sandbox_attach" | "sandbox_swap" | "run_on" | "sandbox_provision" | "connected_machine_remove" | "connected_machine_enroll_token" | "project_list" | "project_get" | "project_create" | "project_update" | "project_reorder" | "project_delete" | "session_set_project" | "rig_list" | "rig_get" | "rig_propose_change" | "rig_verify" | "rig_promote" | "sessions_list" | "session_get" | "session_events" | "session_wait" | "command_read" | "command_wait" | "session_create" | "session_send_message" | "session_pause" | "session_resume" | "session_steer" | "session_human_input_respond" | "set_other_session_title" | "interaction_discover" | "browser_open" | "browser_tabs" | "browser_observe" | "browser_act" | "browser_clipboard" | "browser_debug" | "browser_auth" | "interaction_request_human" | "browser_identity" | "browser_publish" | "browser_lifecycle" | "computer_open" | "computer_targets" | "computer_observe" | "computer_clipboard" | "computer_act" | "computer_lifecycle" | "variable_set_list" | "environment_list" | "variable_set_get_variable" | "variable_set_set_variable" | "environment_set_variable" | "capability_catalog_search" | "capability_authorization_request" | "github_connect_link" | "github_repositories_list" | "social_connections_list" | "social_posts_recent" | "social_daily_analysis_context" | "social_search_live" | "social_mentions_live" | "social_thread_fetch" | "social_posts_sync" | "social_post_reply" | "x_accounts_list" | "x_search_live" | "x_mentions_live" | "x_thread_fetch" | "x_posts_sync" | "x_post_reply" | "reddit_accounts_list" | "reddit_search_live" | "reddit_mentions_live" | "reddit_thread_fetch" | "reddit_posts_sync" | "reddit_post_reply" | "scheduled_tasks_list" | "scheduled_tasks_get" | "scheduled_tasks_create" | "scheduled_tasks_update" | "scheduled_tasks_pause" | "scheduled_tasks_resume" | "scheduled_tasks_trigger" | "scheduled_tasks_delete" | "scheduled_task_runs_list" | "slack_bot_list_channels" | "slack_bot_search" | "slack_bot_channel_history" | "slack_bot_thread_replies" | "slack_bot_list_users" | "slack_bot_list_files" | "slack_bot_file_info" | "slack_bot_file_content" | "slack_bot_post_message" | "slack_bot_delete_message" | "fiken_companies_list" | "fiken_contacts_list" | "fiken_contact_create" | "fiken_products_list" | "fiken_invoices_list" | "fiken_invoice_get" | "fiken_invoice_draft_create" | "fiken_bank_accounts_list" | "fiken_purchases_list" | "fiken_sales_list" | "atlassian_sources_list" | "atlassian_search" | "atlassian_get" | "sandbox_file_publish" | "artifacts_list" | "artifacts_get_source" | "artifacts_prepare_upload" | "artifacts_create" | "artifacts_publish" | "artifacts_rollback" | "artifacts_archive" | "artifacts_restore" | "editable_artifact_list" | "editable_artifact_create" | "editable_artifact_import" | "editable_artifact_get" | "editable_artifact_inspect" | "editable_artifact_apply" | "editable_artifact_export" | "editable_artifact_export_status";
2459
+ export type FirstPartyMcpToolName = "knowledge_search" | "knowledge_prepare_save" | "knowledge_get" | "knowledge_browse" | "knowledge_save" | "knowledge_retain_file" | "knowledge_retain_message" | "knowledge_archive" | "instruction_policy_save" | "instruction_policy_get" | "set_session_title" | "goal_set" | "goal_update" | "goal_progress" | "wait_for_input" | "goal_complete" | "goal_pause" | "goal_resume" | "memory_search" | "memory_save" | "memory_correct" | "preference_registry_summary" | "preference_registry_get" | "task_notes_list" | "task_note_save" | "task_note_archive" | "task_note_replace" | "work_claim_upsert" | "work_claim_release" | "knowledge_propose" | "knowledge_correct" | "task_note_promote_knowledge" | "task_note_promote_instruction_policy" | "task_note_promote_preference" | "instruction_policy_propose" | "preference_propose" | "remember" | "remember_confirm" | "company_profile_propose" | "company_profile_confirm" | "sandboxes_list" | "sandbox_attach" | "sandbox_swap" | "run_on" | "sandbox_provision" | "connected_machine_remove" | "connected_machine_enroll_token" | "project_list" | "project_get" | "project_create" | "project_update" | "project_reorder" | "project_delete" | "session_set_project" | "rig_list" | "rig_get" | "rig_propose_change" | "rig_verify" | "rig_promote" | "sessions_list" | "session_get" | "session_events" | "session_wait" | "command_read" | "command_wait" | "session_create" | "session_send_message" | "session_pause" | "session_resume" | "session_steer" | "session_human_input_respond" | "set_other_session_title" | "interaction_discover" | "browser_open" | "browser_tabs" | "browser_observe" | "browser_act" | "browser_read" | "browser_screenshot" | "browser_clipboard" | "browser_debug" | "browser_auth" | "interaction_request_human" | "browser_identity" | "browser_publish" | "browser_lifecycle" | "computer_open" | "computer_targets" | "computer_observe" | "computer_clipboard" | "computer_act" | "computer_lifecycle" | "variable_set_list" | "environment_list" | "variable_set_get_variable" | "variable_set_set_variable" | "environment_set_variable" | "capability_catalog_search" | "capability_authorization_request" | "github_connect_link" | "github_repositories_list" | "social_connections_list" | "social_posts_recent" | "social_daily_analysis_context" | "social_search_live" | "social_mentions_live" | "social_thread_fetch" | "social_posts_sync" | "social_post_reply" | "x_accounts_list" | "x_search_live" | "x_mentions_live" | "x_thread_fetch" | "x_posts_sync" | "x_post_reply" | "reddit_accounts_list" | "reddit_search_live" | "reddit_mentions_live" | "reddit_thread_fetch" | "reddit_posts_sync" | "reddit_post_reply" | "scheduled_tasks_list" | "scheduled_tasks_get" | "scheduled_tasks_create" | "scheduled_tasks_update" | "scheduled_tasks_pause" | "scheduled_tasks_resume" | "scheduled_tasks_trigger" | "scheduled_tasks_delete" | "scheduled_task_runs_list" | "slack_bot_list_channels" | "slack_bot_search" | "slack_bot_channel_history" | "slack_bot_thread_replies" | "slack_bot_list_users" | "slack_bot_list_files" | "slack_bot_file_info" | "slack_bot_file_content" | "slack_bot_post_message" | "slack_bot_delete_message" | "fiken_companies_list" | "fiken_contacts_list" | "fiken_contact_create" | "fiken_products_list" | "fiken_invoices_list" | "fiken_invoice_get" | "fiken_invoice_draft_create" | "fiken_bank_accounts_list" | "fiken_purchases_list" | "fiken_sales_list" | "atlassian_sources_list" | "atlassian_search" | "atlassian_get" | "sandbox_file_publish" | "artifacts_list" | "artifacts_get_source" | "artifacts_prepare_upload" | "artifacts_create" | "artifacts_publish" | "artifacts_rollback" | "artifacts_archive" | "artifacts_restore" | "editable_artifact_list" | "editable_artifact_create" | "editable_artifact_import" | "editable_artifact_get" | "editable_artifact_inspect" | "editable_artifact_apply" | "editable_artifact_export" | "editable_artifact_export_status";
2460
2460
  export type ProductAccessMode = "local" | "configured" | "managed";
2461
2461
  export type ModelCapabilitySupportV1 = "supported" | "unsupported" | "unknown";
2462
2462
  export type ModelCapabilityStateV1 = {
@@ -4638,7 +4638,7 @@ export declare const RETAINED_OUTPUT_MAX_PAGE_BYTES: number;
4638
4638
  export declare const COMPUTER_SCREENSHOT_MAX_BYTES: number;
4639
4639
  export declare const GENERATED_IMAGE_MAX_BYTES: number;
4640
4640
  export declare const GENERATED_VIDEO_MAX_BYTES: number;
4641
- export type RetainedOutputKind = "tool_result" | "assistant_completion" | "internal_update" | "event_media" | "computer_screenshot" | "generated_image" | "generated_video" | "file";
4641
+ export type RetainedOutputKind = "tool_result" | "assistant_completion" | "internal_update" | "event_media" | "computer_screenshot" | "browser_screenshot" | "generated_image" | "generated_video" | "file";
4642
4642
  export type RetainedOutputUnavailableReason = "not_retained" | "pending" | "failed" | "expired" | "deleted" | "missing_storage" | "storage_write_failed" | "quota_exceeded" | "invalid_content" | "oversized" | "unsupported";
4643
4643
  export type RetainedArtifactReference = {
4644
4644
  available: true;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@opengeni/sdk",
3
- "version": "7.1.0",
3
+ "version": "7.1.1",
4
4
  "description": "Framework-agnostic TypeScript SDK for the OpenGeni API: typed client, session lifecycle, SSE event streaming with reconnect + replay-by-sequence, and proxy re-streaming helpers.",
5
5
  "license": "Apache-2.0",
6
6
  "repository": {
@@ -130,7 +130,7 @@
130
130
  },
131
131
  "dependencies": {
132
132
  "@opengeni/connect": "^0.3.0",
133
- "@opengeni/contracts": "^5.1.0"
133
+ "@opengeni/contracts": "^5.1.1"
134
134
  },
135
135
  "engines": {
136
136
  "node": ">=18"
package/src/browser.ts CHANGED
@@ -4,6 +4,8 @@ export type {
4
4
  SessionMessageSearchRequest,
5
5
  SessionMessageSearchMatch,
6
6
  SessionMessageSearchResponse,
7
+ SessionMessagePreview,
8
+ SessionMessagePreviewReference,
7
9
  } from "./session-message-search";
8
10
  export {
9
11
  OpenGeniToolCallError,
package/src/client.ts CHANGED
@@ -2,6 +2,8 @@ import type { ArtifactCatalogListOptions, ArtifactCatalogListResponse } from "./
2
2
  import type {
3
3
  SessionMessageSearchRequest,
4
4
  SessionMessageSearchResponse,
5
+ SessionMessagePreview,
6
+ SessionMessagePreviewReference,
5
7
  } from "./session-message-search";
6
8
  import type {
7
9
  KnowledgeOriginalFileDownload,
@@ -68,6 +70,7 @@ import {
68
70
  import {
69
71
  OpenGeniInteractionClient,
70
72
  decodeComputerFrameMetadataHeader,
73
+ parseBrowserFrameMetadata,
71
74
  type AuthRun,
72
75
  type AuthRunListOptions,
73
76
  type AuthRunListResponse,
@@ -77,8 +80,11 @@ import {
77
80
  type AttachedBrowserDeviceListResponse,
78
81
  type BrowserActionReceipt,
79
82
  type BrowserActionRequest,
83
+ type BrowserScreenshotOptions,
80
84
  type BrowserClipboard,
81
85
  type BrowserDiagnosticBatch,
86
+ type BrowserDomReadRequest,
87
+ type BrowserDomReadResponse,
82
88
  type BrowserDiagnosticsOptions,
83
89
  type BrowserDownload,
84
90
  type BrowserDownloadListResponse,
@@ -89,6 +95,8 @@ import {
89
95
  type BrowserIdentityListResponse,
90
96
  type BrowserIdentityMutationResponse,
91
97
  type BrowserObservation,
98
+ type BrowserFrame,
99
+ type BrowserFrameMetadata,
92
100
  type BrowserOpenTargetRequest,
93
101
  type BrowserSession,
94
102
  type BrowserSessionAttachment,
@@ -97,6 +105,7 @@ import {
97
105
  type BrowserSessionLifecycleRequest,
98
106
  type BrowserSessionListResponse,
99
107
  type BrowserSessionMutationResponse,
108
+ type BrowserTargetState,
100
109
  type BrowserTargetListResponse,
101
110
  type BrowserRevisionListResponse,
102
111
  type ComputerActionReceipt,
@@ -1330,6 +1339,25 @@ export class OpenGeniClient {
1330
1339
  );
1331
1340
  }
1332
1341
 
1342
+ /** Read one selected search result's complete visible text when <=12,000
1343
+ * UTF-16 units. Stale, duplicate or non-message references fail with 404.
1344
+ * The response never includes audit payload fields such as modelContext.
1345
+ */
1346
+ async getSessionMessagePreview(
1347
+ workspaceId: string,
1348
+ sessionId: string,
1349
+ reference: SessionMessagePreviewReference,
1350
+ options: OpenGeniRequestOptions = {},
1351
+ ): Promise<SessionMessagePreview> {
1352
+ return this.requestJson<SessionMessagePreview>(
1353
+ "GET",
1354
+ `/v1/workspaces/${encodeURIComponent(workspaceId)}/sessions/${encodeURIComponent(sessionId)}/events/${encodeURIComponent(reference.eventId)}/message-preview`,
1355
+ undefined,
1356
+ { sequence: String(reference.sequence) },
1357
+ options,
1358
+ );
1359
+ }
1360
+
1333
1361
  async listSessions(
1334
1362
  workspaceId: string,
1335
1363
  options: {
@@ -3926,6 +3954,73 @@ export class OpenGeniClient {
3926
3954
  );
3927
3955
  }
3928
3956
 
3957
+ async captureBrowserTarget(
3958
+ workspaceId: string,
3959
+ browserSessionId: string,
3960
+ targetId: string,
3961
+ options: OpenGeniRequestOptions = {},
3962
+ captureOptions: BrowserScreenshotOptions = {},
3963
+ ): Promise<BrowserFrame> {
3964
+ const query = new URLSearchParams();
3965
+ if (captureOptions.fullPage !== undefined) {
3966
+ if (typeof captureOptions.fullPage !== "boolean") {
3967
+ throw new TypeError("browser screenshot fullPage must be a boolean");
3968
+ }
3969
+ query.set("fullPage", String(captureOptions.fullPage));
3970
+ }
3971
+ if (captureOptions.format !== undefined) {
3972
+ if (captureOptions.format !== "jpeg" && captureOptions.format !== "png") {
3973
+ throw new TypeError("browser screenshot format must be jpeg or png");
3974
+ }
3975
+ query.set("format", captureOptions.format);
3976
+ }
3977
+ if (captureOptions.quality !== undefined) {
3978
+ if (
3979
+ !Number.isSafeInteger(captureOptions.quality) ||
3980
+ captureOptions.quality < 1 ||
3981
+ captureOptions.quality > 100
3982
+ ) {
3983
+ throw new RangeError("browser screenshot quality must be an integer from 1 to 100");
3984
+ }
3985
+ query.set("quality", String(captureOptions.quality));
3986
+ }
3987
+ const suffix = query.size > 0 ? `?${query}` : "";
3988
+ const response = await this.requestResponse(
3989
+ "GET",
3990
+ `/v1/workspaces/${workspaceId}/browser-sessions/${encodeURIComponent(browserSessionId)}/targets/${encodeURIComponent(targetId)}/screenshot${suffix}`,
3991
+ {},
3992
+ options,
3993
+ );
3994
+ const mediaType = response.headers.get("content-type")?.split(";", 1)[0]?.trim().toLowerCase();
3995
+ const header = response.headers.get("x-opengeni-browser-frame");
3996
+ if (
3997
+ (mediaType !== "image/jpeg" && mediaType !== "image/png") ||
3998
+ !header ||
3999
+ header.length > 64 * 1024
4000
+ ) {
4001
+ await cancelResponseBody(response, "browser frame metadata is invalid");
4002
+ throw new OpenGeniApiError(502, "browser frame metadata is invalid");
4003
+ }
4004
+ let metadata: BrowserFrameMetadata;
4005
+ try {
4006
+ metadata = parseBrowserFrameMetadata(
4007
+ JSON.parse(atob(header.replace(/-/gu, "+").replace(/_/gu, "/"))),
4008
+ );
4009
+ } catch {
4010
+ await cancelResponseBody(response, "browser frame metadata is invalid");
4011
+ throw new OpenGeniApiError(502, "browser frame metadata is invalid");
4012
+ }
4013
+ const data = await readBoundedResponseBytes(response, 24 * 1024 * 1024, null);
4014
+ if (
4015
+ metadata.browserSessionId !== browserSessionId ||
4016
+ metadata.targetId !== targetId ||
4017
+ metadata.mediaType !== mediaType
4018
+ ) {
4019
+ throw new OpenGeniApiError(502, "browser frame evidence does not match its request");
4020
+ }
4021
+ return { ...metadata, data };
4022
+ }
4023
+
3929
4024
  async closeBrowserTarget(
3930
4025
  workspaceId: string,
3931
4026
  browserSessionId: string,
@@ -3956,6 +4051,52 @@ export class OpenGeniClient {
3956
4051
  );
3957
4052
  }
3958
4053
 
4054
+ async getBrowserTargetState(
4055
+ workspaceId: string,
4056
+ browserSessionId: string,
4057
+ targetId: string,
4058
+ options: OpenGeniRequestOptions = {},
4059
+ ): Promise<BrowserTargetState> {
4060
+ const state = await this.requestJson<BrowserTargetState>(
4061
+ "GET",
4062
+ `/v1/workspaces/${workspaceId}/browser-sessions/${encodeURIComponent(browserSessionId)}/targets/${encodeURIComponent(targetId)}/state`,
4063
+ undefined,
4064
+ {},
4065
+ options,
4066
+ );
4067
+ if (state.browserSessionId !== browserSessionId || state.targetId !== targetId) {
4068
+ throw new OpenGeniApiError(502, "browser target state belongs to another binding");
4069
+ }
4070
+ return state;
4071
+ }
4072
+
4073
+ async readBrowserDom(
4074
+ workspaceId: string,
4075
+ browserSessionId: string,
4076
+ targetId: string,
4077
+ request: BrowserDomReadRequest,
4078
+ options: OpenGeniRequestOptions = {},
4079
+ ): Promise<BrowserDomReadResponse> {
4080
+ const result = await this.requestJson<BrowserDomReadResponse>(
4081
+ "POST",
4082
+ `/v1/workspaces/${workspaceId}/browser-sessions/${encodeURIComponent(browserSessionId)}/targets/${encodeURIComponent(targetId)}/dom-read`,
4083
+ request,
4084
+ {},
4085
+ options,
4086
+ );
4087
+ if (
4088
+ result.browserSessionId !== browserSessionId ||
4089
+ result.targetId !== targetId ||
4090
+ result.kind !== request.kind ||
4091
+ result.targetGeneration !== request.expectedTargetGeneration ||
4092
+ result.documentGeneration !== request.expectedDocumentGeneration ||
4093
+ result.frameId !== request.expectedFrameId
4094
+ ) {
4095
+ throw new OpenGeniApiError(502, "browser DOM read belongs to another binding");
4096
+ }
4097
+ return result;
4098
+ }
4099
+
3959
4100
  async actInBrowser(
3960
4101
  workspaceId: string,
3961
4102
  browserSessionId: string,
@@ -6247,9 +6388,12 @@ export class OpenGeniClient {
6247
6388
  ): Promise<RetainedScreenshotDownload> {
6248
6389
  const metadata = await this.getSessionRetainedArtifact(workspaceId, sessionId, artifactId);
6249
6390
  if (!metadata.available) return { metadata, bytes: null };
6391
+ const supportedScreenshot =
6392
+ (metadata.kind === "computer_screenshot" && metadata.contentType === "image/png") ||
6393
+ (metadata.kind === "browser_screenshot" &&
6394
+ ["image/png", "image/jpeg", "image/webp"].includes(metadata.contentType));
6250
6395
  if (
6251
- metadata.kind !== "computer_screenshot" ||
6252
- metadata.contentType !== "image/png" ||
6396
+ !supportedScreenshot ||
6253
6397
  !metadata.dimensions ||
6254
6398
  metadata.originalBytes <= 0 ||
6255
6399
  metadata.originalBytes > COMPUTER_SCREENSHOT_MAX_BYTES
package/src/index.ts CHANGED
@@ -8,6 +8,8 @@ export type {
8
8
  SessionMessageSearchRequest,
9
9
  SessionMessageSearchMatch,
10
10
  SessionMessageSearchResponse,
11
+ SessionMessagePreview,
12
+ SessionMessagePreviewReference,
11
13
  } from "./session-message-search";
12
14
  export { OpenGeniEmbeddingClient as OpenGeniClient } from "./embedding-client";
13
15
  export { pluginMcpUnavailableReason } from "@opengeni/contracts/plugin-discovery";
@@ -1,4 +1,15 @@
1
1
  import type { OpenGeniRequestOptions } from "./client";
2
+ import type {
3
+ BrowserDomReadRequest,
4
+ BrowserDomReadResponse,
5
+ BrowserTargetState,
6
+ } from "@opengeni/contracts";
7
+ export type {
8
+ BrowserDomReadRequest,
9
+ BrowserDomReadResponse,
10
+ BrowserDomSafeAttribute,
11
+ BrowserTargetState,
12
+ } from "@opengeni/contracts";
2
13
  import { OpenGeniApiError } from "./errors";
3
14
  import type { RetainedArtifactReference } from "./types";
4
15
 
@@ -802,6 +813,17 @@ export type BrowserObservation = {
802
813
  frameId: string | null;
803
814
  semantic: InteractionSemanticSnapshot | InteractionSemanticDiff | null;
804
815
  screenshot: RetainedArtifactReference | null;
816
+ viewport?:
817
+ | {
818
+ width: number;
819
+ height: number;
820
+ visualWidth: number;
821
+ visualHeight: number;
822
+ deviceScaleFactor: number;
823
+ maxTouchPoints: number;
824
+ }
825
+ | null
826
+ | undefined;
805
827
  focusedRef: string | null;
806
828
  changedRegions: InteractionRect[];
807
829
  diagnostics: InteractionDiagnosticSummary;
@@ -863,6 +885,15 @@ export type BrowserPermissionSetting = "granted" | "denied" | "prompt";
863
885
 
864
886
  export type BrowserAction =
865
887
  | { type: "navigate"; url: string }
888
+ | { type: "history"; direction: "back" | "forward" }
889
+ | { type: "activate" }
890
+ | {
891
+ type: "viewport";
892
+ width: number;
893
+ height: number;
894
+ mobile: boolean;
895
+ deviceScaleFactor?: number | undefined;
896
+ }
866
897
  | {
867
898
  type: "click";
868
899
  locator: BrowserLocator;
@@ -1475,6 +1506,26 @@ export interface InteractionTransport {
1475
1506
  targetId: string,
1476
1507
  options?: OpenGeniRequestOptions,
1477
1508
  ): Promise<BrowserObservation>;
1509
+ getBrowserTargetState(
1510
+ workspaceId: string,
1511
+ browserSessionId: string,
1512
+ targetId: string,
1513
+ options?: OpenGeniRequestOptions,
1514
+ ): Promise<BrowserTargetState>;
1515
+ readBrowserDom(
1516
+ workspaceId: string,
1517
+ browserSessionId: string,
1518
+ targetId: string,
1519
+ request: BrowserDomReadRequest,
1520
+ options?: OpenGeniRequestOptions,
1521
+ ): Promise<BrowserDomReadResponse>;
1522
+ captureBrowserTarget(
1523
+ workspaceId: string,
1524
+ browserSessionId: string,
1525
+ targetId: string,
1526
+ options?: OpenGeniRequestOptions,
1527
+ captureOptions?: BrowserScreenshotOptions,
1528
+ ): Promise<BrowserFrame>;
1478
1529
  actInBrowser(
1479
1530
  workspaceId: string,
1480
1531
  browserSessionId: string,
@@ -2121,6 +2172,49 @@ export class BrowserSessionResource {
2121
2172
  return await this.transport.observeBrowserTarget(this.workspaceId, this.id, targetId, options);
2122
2173
  }
2123
2174
 
2175
+ async targetState(
2176
+ targetId: string,
2177
+ options: OpenGeniRequestOptions = {},
2178
+ ): Promise<BrowserTargetState> {
2179
+ return await this.transport.getBrowserTargetState(this.workspaceId, this.id, targetId, options);
2180
+ }
2181
+
2182
+ async readDom(
2183
+ targetId: string,
2184
+ request: BrowserDomReadRequest,
2185
+ options: OpenGeniRequestOptions = {},
2186
+ ): Promise<BrowserDomReadResponse> {
2187
+ return await this.transport.readBrowserDom(
2188
+ this.workspaceId,
2189
+ this.id,
2190
+ targetId,
2191
+ request,
2192
+ options,
2193
+ );
2194
+ }
2195
+
2196
+ async capture(
2197
+ targetId: string,
2198
+ options: OpenGeniRequestOptions = {},
2199
+ captureOptions: BrowserScreenshotOptions = {},
2200
+ ): Promise<BrowserFrame> {
2201
+ return await this.transport.captureBrowserTarget(
2202
+ this.workspaceId,
2203
+ this.id,
2204
+ targetId,
2205
+ options,
2206
+ captureOptions,
2207
+ );
2208
+ }
2209
+
2210
+ async screenshot(
2211
+ targetId: string,
2212
+ captureOptions: BrowserScreenshotOptions = {},
2213
+ options: OpenGeniRequestOptions = {},
2214
+ ): Promise<BrowserFrame> {
2215
+ return await this.capture(targetId, options, captureOptions);
2216
+ }
2217
+
2124
2218
  async act(
2125
2219
  request: BrowserActionRequest,
2126
2220
  options: OpenGeniRequestOptions = {},
@@ -2505,6 +2599,14 @@ export type BrowserFrameMetadata = {
2505
2599
 
2506
2600
  export type BrowserFrame = BrowserFrameMetadata & { data: Uint8Array };
2507
2601
 
2602
+ export type BrowserScreenshotOptions = {
2603
+ /** Capture the document content instead of the visible viewport. */
2604
+ fullPage?: boolean;
2605
+ format?: "jpeg" | "png";
2606
+ /** JPEG quality, from 1 to 100. Ignored for PNG. */
2607
+ quality?: number;
2608
+ };
2609
+
2508
2610
  export type ComputerFrameMetadata = {
2509
2611
  frameId: string;
2510
2612
  computerSessionId: string;
package/src/knowledge.ts CHANGED
@@ -14,6 +14,7 @@ export type {
14
14
  KnowledgeOriginalFileDownload,
15
15
  KnowledgeEntryKind,
16
16
  KnowledgeEntryScope,
17
+ KnowledgeIndexStatus,
17
18
  KnowledgeEntryContent,
18
19
  KnowledgeEntrySource,
19
20
  KnowledgeEntryEvidence,
@@ -37,3 +37,11 @@ export type SessionMessageSearchResponse = {
37
37
  /** True only on exhaustion. Concurrent mutations require a fresh traversal. */
38
38
  countIsExact: boolean;
39
39
  };
40
+
41
+ /** Exact selected event reference from a search match; no payload fields are read. */
42
+ export type SessionMessagePreviewReference = { eventId: string; sequence: number };
43
+
44
+ /** A complete visible message, or an explicit over-12,000-UTF-16-unit result. */
45
+ export type SessionMessagePreview =
46
+ | { status: "available"; text: string }
47
+ | { status: "unavailable" };