@opengeni/react 7.1.0 → 7.1.2-canary.36079201993001

package/src/hooks/use-browser-session.ts

@@ -23,6 +23,9 @@ export type UseBrowserSessionOptions = EmbeddedBrowserInteractionClientOverride
   browserSessionId: string | null;
   enabled?: boolean | undefined;
   pollIntervalMs?: number | undefined;
+  /** Keep session/tab inventory current without repeatedly fetching the full
+   * accessibility tree while a live frame is available or the viewer is hidden. */
+  semanticObservationEnabled?: boolean | undefined;
 };
 
 export type UseBrowserSessionResult = {
@@ -58,6 +61,13 @@ export function useBrowserSession(options: UseBrowserSessionOptions): UseBrowser
   const { client, workspaceId } = useEmbeddedBrowserInteraction(options);
   const browserSessionId = options.browserSessionId;
   const enabled = (options.enabled ?? true) && browserSessionId !== null;
+  const semanticObservationEnabled = options.semanticObservationEnabled ?? true;
+  const semanticObservationEnabledRef = useRef(semanticObservationEnabled);
+  semanticObservationEnabledRef.current = semanticObservationEnabled;
+  const previousSemanticObservationRef = useRef({
+    browserSessionId,
+    enabled: semanticObservationEnabled,
+  });
   const pageLive = usePageLiveActivity();
   const pollIntervalMs = Math.max(750, options.pollIntervalMs ?? 2_000);
   const [state, setState] = useState<{
@@ -75,10 +85,16 @@ export function useBrowserSession(options: UseBrowserSessionOptions): UseBrowser
   const refreshBlocked = isNonRetryableInteractionError(visible.error);
   const selectedTargetIdRef = useRef<string | null>(visible.selectedTargetId);
   selectedTargetIdRef.current = visible.selectedTargetId;
+  const selectedTargetRef = useRef<BrowserTarget | null>(
+    visible.targets.find((target) => target.id === visible.selectedTargetId) ?? null,
+  );
+  selectedTargetRef.current =
+    visible.targets.find((target) => target.id === visible.selectedTargetId) ?? null;
   const observationRef = useRef<{
     browserSessionId: string | null;
     observation: BrowserObservation | null;
   }>({ browserSessionId, observation: visible.observation });
+  const lastObservationAtRef = useRef(0);
   if (observationRef.current.browserSessionId !== browserSessionId) {
     observationRef.current = {
       browserSessionId,
@@ -124,7 +140,15 @@ export function useBrowserSession(options: UseBrowserSessionOptions): UseBrowser
       if (!mountedRef.current || requestRef.current.id !== id) return;
       const selected = chooseTarget(targetResponse.targets, selectedTargetIdRef.current);
       selectedTargetIdRef.current = selected?.id ?? null;
-      observationRef.current = { browserSessionId, observation: null };
+      const previousObservation =
+        observationRef.current.browserSessionId === browserSessionId
+          ? observationRef.current.observation
+          : null;
+      const retainedObservation =
+        selected && previousObservation && sameObservationTarget(previousObservation, selected)
+          ? previousObservation
+          : null;
+      observationRef.current = { browserSessionId, observation: retainedObservation };
       setState((current) =>
         current.browserSessionId === browserSessionId
           ? {
@@ -132,13 +156,13 @@ export function useBrowserSession(options: UseBrowserSessionOptions): UseBrowser
               session,
               targets: targetResponse.targets,
               selectedTargetId: selected?.id ?? null,
-              observation: null,
+              observation: retainedObservation,
               loading: false,
               error: null,
             }
           : current,
       );
-      if (!selected) return;
+      if (!selected || !semanticObservationEnabledRef.current) return;
       try {
         const observation = await client.observeBrowserTarget(
           workspaceId,
@@ -148,6 +172,7 @@ export function useBrowserSession(options: UseBrowserSessionOptions): UseBrowser
         );
         if (!mountedRef.current || requestRef.current.id !== id) return;
         observationRef.current = { browserSessionId, observation };
+        lastObservationAtRef.current = Date.now();
         setState((current) =>
           current.browserSessionId === browserSessionId &&
           current.selectedTargetId === observation.target.id
@@ -180,6 +205,37 @@ export function useBrowserSession(options: UseBrowserSessionOptions): UseBrowser
     }
   }, [browserSessionId, client, enabled, workspaceId]);
 
+  const refreshSemantic = useCallback(async (): Promise<void> => {
+    if (!enabled || !browserSessionId) return;
+    const target = selectedTargetRef.current;
+    if (!target) return;
+    const id = requestRef.current.id + 1;
+    requestRef.current.controller?.abort();
+    const controller = new AbortController();
+    requestRef.current = { id, controller };
+    try {
+      const observation = await client.observeBrowserTarget(
+        workspaceId,
+        browserSessionId,
+        target.id,
+        { signal: controller.signal },
+      );
+      if (!mountedRef.current || requestRef.current.id !== id) return;
+      if (!sameObservationTarget(observation, selectedTargetRef.current)) return;
+      observationRef.current = { browserSessionId, observation };
+      lastObservationAtRef.current = Date.now();
+      setState((current) =>
+        current.browserSessionId === browserSessionId && current.selectedTargetId === target.id
+          ? { ...current, observation }
+          : current,
+      );
+    } catch {
+      // A failed semantic fallback cannot disable an otherwise live browser.
+    } finally {
+      if (requestRef.current.id === id) requestRef.current = { id, controller: null };
+    }
+  }, [browserSessionId, client, enabled, workspaceId]);
+
   useEffect(() => {
     mountedRef.current = true;
     if (!enabled) {
@@ -199,6 +255,37 @@ export function useBrowserSession(options: UseBrowserSessionOptions): UseBrowser
     };
   }, [browserSessionId, enabled, invalidateRefresh, refresh]);
 
+  useEffect(() => {
+    const previous = previousSemanticObservationRef.current;
+    previousSemanticObservationRef.current = {
+      browserSessionId,
+      enabled: semanticObservationEnabled,
+    };
+    if (
+      enabled &&
+      semanticObservationEnabled &&
+      previous.browserSessionId === browserSessionId &&
+      !previous.enabled
+    ) {
+      // The media stream disappeared or the user returned to the Browser tab.
+      // A just-returned action/tab observation is already a fresh fallback.
+      // Otherwise drop the old tree/fence until a new read settles.
+      const currentObservation = observationRef.current.observation;
+      if (
+        currentObservation &&
+        sameObservationTarget(currentObservation, selectedTargetRef.current) &&
+        Date.now() - lastObservationAtRef.current < pollIntervalMs
+      ) {
+        return;
+      }
+      observationRef.current = { browserSessionId, observation: null };
+      setState((current) =>
+        current.browserSessionId === browserSessionId ? { ...current, observation: null } : current,
+      );
+      void refreshSemantic();
+    }
+  }, [browserSessionId, enabled, pollIntervalMs, refreshSemantic, semanticObservationEnabled]);
+
   useEffect(() => {
     if (!enabled || !pageLive || visible.mutating || refreshBlocked) return;
     const timer = setInterval(() => {
@@ -301,6 +388,7 @@ export function useBrowserSession(options: UseBrowserSessionOptions): UseBrowser
         }
         selectedTargetIdRef.current = target.id;
         observationRef.current = { browserSessionId, observation };
+        lastObservationAtRef.current = Date.now();
         setState((current) =>
           current.browserSessionId === browserSessionId
             ? {
@@ -332,6 +420,7 @@ export function useBrowserSession(options: UseBrowserSessionOptions): UseBrowser
         const target = observation.target;
         selectedTargetIdRef.current = target.id;
         observationRef.current = { browserSessionId, observation };
+        lastObservationAtRef.current = Date.now();
         setState((current) =>
           current.browserSessionId === browserSessionId
             ? {
@@ -365,6 +454,7 @@ export function useBrowserSession(options: UseBrowserSessionOptions): UseBrowser
           : null;
         selectedTargetIdRef.current = selected?.id ?? null;
         observationRef.current = { browserSessionId, observation };
+        if (observation) lastObservationAtRef.current = Date.now();
         setState((current) =>
           current.browserSessionId === browserSessionId
             ? {
@@ -424,6 +514,7 @@ export function useBrowserSession(options: UseBrowserSessionOptions): UseBrowser
             browserSessionId,
             observation: receipt.observation,
           };
+          lastObservationAtRef.current = Date.now();
           setState((current) =>
             current.browserSessionId === browserSessionId
               ? {
@@ -540,6 +631,19 @@ function replaceTarget(targets: readonly BrowserTarget[], target: BrowserTarget)
   return next.sort((left, right) => left.createdAt.localeCompare(right.createdAt));
 }
 
+function sameObservationTarget(
+  observation: BrowserObservation,
+  target: BrowserTarget | null,
+): boolean {
+  return (
+    target !== null &&
+    observation.target.id === target.id &&
+    observation.target.controllerGeneration === target.controllerGeneration &&
+    observation.target.targetGeneration === target.targetGeneration &&
+    observation.target.documentGeneration === target.documentGeneration
+  );
+}
+
 function isMissingBrowserTarget(error: unknown): boolean {
   return error instanceof OpenGeniApiError && error.code === "target_not_found";
 }

package/src/hooks/use-voice-input.ts

@@ -163,6 +163,7 @@ export function useVoiceInput({
   const automaticRetryAttemptRef = useRef(0);
   const ownerHeartbeatTimerRef = useRef<ReturnType<typeof setInterval> | null>(null);
   const controllerRef = useRef<AbortController | null>(null);
+  const disabledRef = useRef(disabled);
   const valueRef = useRef(value);
   const storeRef = useRef<VoiceRecordingStore | null>(null);
   const storePromiseRef = useRef<Promise<VoiceRecordingStore> | null>(null);
@@ -176,6 +177,7 @@ export function useVoiceInput({
   const statusRef = useRef(status);
   valueRef.current = value;
   statusRef.current = status;
+  disabledRef.current = disabled;
   workspaceIdRef.current = workspaceId;
 
   const ensureOwnerId = useCallback(async (): Promise<string> => {
@@ -468,6 +470,14 @@ export function useVoiceInput({
           return;
         }
 
+        // Session creation can disable the composer while a recording is
+        // transcribing. Keep the durable transcript for explicit insertion
+        // later; never append text after Send captured its visible snapshot.
+        if (disabledRef.current) {
+          setStatus("transcript-ready");
+          return;
+        }
+
         const next = appendFinalTranscript(valueRef.current, response.text);
         if (next !== valueRef.current) {
           valueRef.current = next;
@@ -944,6 +954,7 @@ export function useVoiceInput({
   const insertSavedTranscript = useCallback(async (): Promise<void> => {
     const manifest = manifestRef.current;
     if (
+      disabledRef.current ||
       !manifest ||
       manifest.workspaceId !== workspaceId ||
       manifest.finalizationState !== "transcript-ready" ||
@@ -955,7 +966,7 @@ export function useVoiceInput({
     controllerRef.current?.abort();
     controllerRef.current = null;
     const store = await ensureStore();
-    if (generation !== generationRef.current) return;
+    if (generation !== generationRef.current || disabledRef.current) return;
     const next = appendFinalTranscript(valueRef.current, manifest.transcriptText);
     if (next !== valueRef.current) {
       valueRef.current = next;

package/src/timeline/compaction-copy.ts

@@ -15,6 +15,8 @@ export function compactionSkipSubtitle(
       return "No active history to compact";
     case "replacement_not_smaller":
       return "Checkpoint would not reduce memory size";
+    case "replacement_exceeds_model_budget":
+      return "Checkpoint exceeds this model's context budget. Chat history is unchanged.";
     case "replacement_unchanged":
       return "Checkpoint made no progress";
     case "summarization_failed":

package/src/timeline/parsers.ts

@@ -566,8 +566,8 @@ export function retainedScreenshotMetadata(out: unknown): RetainedArtifactMetada
   if (!value.available) {
     return typeof value.reason === "string" ? (value as unknown as RetainedArtifactMetadata) : null;
   }
-  return value.kind === "computer_screenshot" &&
-    value.contentType === "image/png" &&
+  return (value.kind === "computer_screenshot" || value.kind === "browser_screenshot") &&
+    ["image/png", "image/jpeg", "image/webp"].includes(String(value.contentType)) &&
     typeof value.originalBytes === "number" &&
     typeof value.sha256 === "string" &&
     value.dimensions !== null &&

package/src/timeline/projection.ts

@@ -1016,6 +1016,7 @@ export function buildTimeline(
       case "tool.auth_needed":
       case "credential.auth_needed": {
         const capability = capabilityAuthorizationRequest(payload.capability);
+        const setupRequest = customMcpSetupRequest(payload.setupRequest);
         if (
           event.type === "tool.auth_needed" &&
           !stringValue(payload.toolName) &&
@@ -1045,11 +1046,12 @@ export function buildTimeline(
             payload.connectionSubjectScope === "workspace"
               ? payload.connectionSubjectScope
               : null,
-          source: capability
-            ? "capability"
-            : event.type === "tool.auth_needed"
-              ? "tool"
-              : "credential",
+          source:
+            capability || setupRequest
+              ? "capability"
+              : event.type === "tool.auth_needed"
+                ? "tool"
+                : "credential",
           providerDomain: stringValue(payload.providerDomain),
           connectionId: typeof payload.connectionId === "string" ? payload.connectionId : null,
           authoritySource: payload.authoritySource === "host" ? "host" : null,
@@ -1064,6 +1066,7 @@ export function buildTimeline(
           authorizationUrl:
             typeof payload.authorizationUrl === "string" ? payload.authorizationUrl : null,
           capability,
+          setupRequest,
           occurredAt: event.occurredAt,
         });
         break;
@@ -2623,6 +2626,29 @@ function capabilityAuthorizationRequest(
   };
 }
 
+function customMcpSetupRequest(value: unknown): NonNullable<AuthNeededItem["setupRequest"]> | null {
+  if (!value || typeof value !== "object" || Array.isArray(value)) return null;
+  const input = value as Record<string, unknown>;
+  if (
+    input.kind !== "mcp" ||
+    typeof input.name !== "string" ||
+    typeof input.rationale !== "string" ||
+    typeof input.endpointUrl !== "string"
+  )
+    return null;
+  try {
+    if (new URL(input.endpointUrl).protocol !== "https:") return null;
+  } catch {
+    return null;
+  }
+  return {
+    kind: "mcp",
+    name: input.name,
+    endpointUrl: input.endpointUrl,
+    rationale: input.rationale,
+  };
+}
+
 function stringList(value: unknown): string[] {
   return Array.isArray(value)
     ? value.filter((entry): entry is string => typeof entry === "string" && entry.trim().length > 0)

package/src/timeline/tool-renderers.tsx

@@ -772,21 +772,73 @@ function ComputerCallRenderer({ item, loadRetainedScreenshot }: ToolRendererProp
   );
 }
 
-function RetainedSessionImageDisclosure({
-  artifact,
-  load,
-  title,
-  caption,
-  noun,
-  icon,
-  lightboxLabel,
-  batched,
-  failed,
-  cancelled,
-  filename,
-  defaultOpen,
-  children,
-}: {
+/** Browser tool images share authenticated screenshot loading, with their own media kind. */
+function BrowserScreenshotRenderer(props: ToolRendererProps) {
+  const { item, loadRetainedScreenshot } = props;
+  const leaf = mcpToolLeaf(item.name);
+  const title =
+    leaf === "browser_observe"
+      ? "Observed browser"
+      : leaf === "browser_act"
+        ? "Used browser"
+        : "Browser screenshot";
+  const retained = retainedScreenshotMetadata(item.output);
+  const isFailed = item.status === "failed";
+  const isCancelled = item.status === "cancelled";
+
+  if (item.status === "running" && leaf === "browser_screenshot") {
+    return (
+      <ActivityDisclosure
+        icon={<CameraIcon className={ICON_SIZE} />}
+        iconTone="running"
+        title={title}
+        running
+        media={<MediaSkeleton />}
+      >
+        <BodyNote>capturing frame…</BodyNote>
+      </ActivityDisclosure>
+    );
+  }
+
+  if (!retained) return <GenericRenderer {...props} />;
+  if (!retained.available) {
+    const state =
+      retained.reason === "expired" || retained.reason === "deleted"
+        ? retained.reason
+        : "unavailable";
+    return (
+      <ActivityDisclosure
+        icon={<CameraOffIcon className={ICON_SIZE} />}
+        iconTone={isFailed ? "failed" : "muted"}
+        title={`${title} · ${state}`}
+        failed={isFailed}
+        cancelled={isCancelled}
+        preview={`screenshot ${state}`}
+        media={<MediaEmpty />}
+      >
+        <BodyNote tone={isFailed ? "error" : undefined}>
+          Screenshot {state}: {retained.reason.replaceAll("_", " ")}.
+        </BodyNote>
+      </ActivityDisclosure>
+    );
+  }
+  return (
+    <RetainedSessionImageDisclosure
+      artifact={retained}
+      load={loadRetainedScreenshot}
+      title={title}
+      caption={title}
+      noun="screenshot"
+      icon={<CameraIcon className={ICON_SIZE} />}
+      lightboxLabel="Browser screenshot"
+      batched={null}
+      failed={isFailed}
+      cancelled={isCancelled}
+    />
+  );
+}
+
+type RetainedSessionImageDisclosureProps = {
   artifact: RetainedArtifactReference;
   load: ToolRendererProps["loadRetainedArtifact"];
   title: string;
@@ -800,7 +852,41 @@ function RetainedSessionImageDisclosure({
   filename?: string;
   defaultOpen?: boolean;
   children?: ReactNode;
-}) {
+};
+
+function RetainedSessionImageDisclosure(props: RetainedSessionImageDisclosureProps) {
+  const compact = useContext(CompactActivityContext);
+  // The rolling progress label has no screenshot loader by design. Do not run
+  // the full image hook there: it would report the missing loader as an error.
+  if (compact) {
+    return (
+      <ActivityDisclosure
+        icon={props.icon}
+        title={props.title}
+        compactPreview={null}
+        failed={props.failed}
+        cancelled={props.cancelled}
+      />
+    );
+  }
+  return <LoadedRetainedSessionImageDisclosure {...props} />;
+}
+
+function LoadedRetainedSessionImageDisclosure({
+  artifact,
+  load,
+  title,
+  caption,
+  noun,
+  icon,
+  lightboxLabel,
+  batched,
+  failed,
+  cancelled,
+  filename,
+  defaultOpen,
+  children,
+}: RetainedSessionImageDisclosureProps) {
   const state = useRetainedImageObjectUrl(artifact, load);
   const downloadFilename = filename ?? retainedImageFilename(artifact);
 
@@ -2401,6 +2487,9 @@ const BASE_ENTRIES: ToolRegistryEntry[] = [
   { match: "name", name: "apply_patch_call", render: ApplyPatchRenderer },
   { match: "name", name: "apply_patch", render: ApplyPatchRenderer },
   { match: "name", name: "computer_call", render: ComputerCallRenderer },
+  { match: "name", name: "browser_screenshot", render: BrowserScreenshotRenderer },
+  { match: "name", name: "browser_observe", render: BrowserScreenshotRenderer },
+  { match: "name", name: "browser_act", render: BrowserScreenshotRenderer },
   // Function-mode computer tools (codex / chat-wire transports).
   { match: "name", name: "computer_screenshot", render: ComputerCallRenderer },
   { match: "name", name: "computer_click", render: ComputerCallRenderer },

package/src/timeline/turn-summary.tsx

@@ -21,9 +21,11 @@ import {
   applyPatchOpsFromToolItem,
   isApplyPatch,
   mediaPreviewFact,
+  retainedScreenshotMetadata,
   screenshotDataUrl,
 } from "./parsers";
 import { rawTypeOf } from "./registry";
+import { mcpToolLeaf } from "./tool-display-name";
 import type { ActivityItem, ToolCallItem, TurnOutcome } from "./types";
 export type { TurnOutcome } from "./types";
 
@@ -626,11 +628,17 @@ const BUILT_IN_TURN_SUMMARY_FACETS: readonly TurnSummaryFacet[] = Object.freeze(
     summarize: ({ toolCalls }) => {
       let screenshots = 0;
       for (const item of toolCalls) {
+        const leaf = mcpToolLeaf(item.name);
         if (
           (rawTypeOf(item) === "computer_call" ||
             item.name === "computer_call" ||
-            item.name === "computer_screenshot") &&
-          (screenshotDataUrl(item.output) !== null || mediaPreviewFact(item.output) !== null)
+            item.name === "computer_screenshot" ||
+            leaf === "browser_screenshot" ||
+            leaf === "browser_observe" ||
+            leaf === "browser_act") &&
+          (retainedScreenshotMetadata(item.output) !== null ||
+            screenshotDataUrl(item.output) !== null ||
+            mediaPreviewFact(item.output) !== null)
         ) {
           screenshots += 1;
         }

package/src/timeline/types.ts

@@ -475,6 +475,8 @@ export type AuthNeededItem = {
   authorizationUrl: string | null;
   /** Agent-selected catalog recommendation. The host still owns authorization. */
   capability?: NonNullable<ToolAuthNeededPayload["capability"]> | null | undefined;
+  /** Human-reviewed custom integration proposal. Never treated as catalog authority. */
+  setupRequest?: ToolAuthNeededPayload["setupRequest"] | null | undefined;
   occurredAt: string;
 };
 

package/styles/compiled.css

@@ -809,6 +809,9 @@
 :where(.og-root).og-session-capability-setup [role=alert],:where(.og-root) .og-session-capability-setup [role=alert] {
   color: var(--og-color-danger,#b42318);
 }
+:where(.og-root).og-session-capability-setup .og-session-capability-notice,:where(.og-root) .og-session-capability-setup .og-session-capability-notice {
+  color: var(--og-color-fg-muted);
+}
 :where(.og-root).og-session-capability-shell[data-state="complete"],:where(.og-root) .og-session-capability-shell[data-state="complete"] {
   border-color: var(--og-color-status-success,#16804a);
   @supports (color: color-mix(in lab, red, red)) {
@@ -857,6 +860,9 @@
   color: var(--og-color-fg-muted);
   padding-block: 8px;
 }
+:where(.og-root).og-session-capability-setup .og-session-capability-stop,:where(.og-root) .og-session-capability-setup .og-session-capability-stop {
+  justify-self: end;
+}
 @media (pointer:coarse) {
   :where(.og-root).og-session-capability-actions button,:where(.og-root) .og-session-capability-actions button,:where(.og-root).og-session-capability-setup button,:where(.og-root) .og-session-capability-setup button {
     min-height: 44px;

package/styles/session-capability.css

@@ -28,6 +28,7 @@
 .og-session-capability-setup button { font:inherit; padding:9px 12px; border-radius:7px; border:1px solid var(--og-color-border); background:var(--og-color-surface-2); color:inherit; cursor:pointer; }
 .og-session-capability-setup button:disabled { opacity:.55; cursor:not-allowed; }
 .og-session-capability-setup [role=alert] { color:var(--og-color-danger,#b42318); }
+.og-session-capability-setup .og-session-capability-notice { color:var(--og-color-fg-muted); }
 .og-session-capability-shell[data-state="complete"] { border-color:color-mix(in srgb,var(--og-color-status-success,#16804a) 35%,var(--og-color-border)); }
 .og-session-capability-shell[data-state="complete"] .og-session-capability-header > svg,
 .og-session-capability-shell[data-state="complete"] .og-session-capability-copy { color:var(--og-color-status-success,#16804a); }
@@ -39,4 +40,5 @@
 .og-session-capability-setup .og-session-capability-primary { justify-self:end; min-height:40px; padding:10px 16px; border:0; background:var(--og-color-accent-deep,#006cdb); color:var(--og-color-accent-fg,#fff); font-weight:500; }
 .og-session-capability-setup .og-session-capability-primary:hover:not(:disabled) { filter:brightness(.94); }
 .og-session-capability-progress { color:var(--og-color-fg-muted); padding-block:8px; }
+.og-session-capability-setup .og-session-capability-stop { justify-self:end; }
 @media (pointer:coarse) { .og-session-capability-actions button,.og-session-capability-setup button { min-height:44px; } }