@opengeni/react 7.1.0 → 7.1.2

package/src/components/session-mcp-capability-card.tsx

@@ -1,5 +1,6 @@
 import {
   authorizeConnectAttempt,
+  ConnectPopupClosedError,
   ConnectController,
   createBrowserConnectNavigation,
   reserveBrowserConnectNavigation,
@@ -77,25 +78,32 @@ function ScopedCard({
   const [complete, setComplete] = useState(false);
   const [connected, setConnected] = useState(false);
   const [waiting, setWaiting] = useState(false);
+  const [reconciling, setReconciling] = useState(false);
   const [busy, setBusy] = useState(false);
   const [error, setError] = useState<string | null>(null);
+  const [notice, setNotice] = useState<string | null>(null);
+  const [retryFresh, setRetryFresh] = useState(false);
   const [ownership, setOwnership] = useState<ConnectOwnership>("workspace");
   const lifetime = useRef<AbortController | null>(null);
-  const operation = useRef(false);
+  const authorization = useRef<AbortController | null>(null);
+  const operation = useRef<AbortController | null>(null);
+  const reservedPopup = useRef<(() => void) | null>(null);
   const startKey = useRef(crypto.randomUUID());
   const advanceKey = useRef(crypto.randomUUID());
+  const startInput = useRef<string | null>(null);
   const storageKey = sessionId
     ? `opengeni:session-connect:${workspaceId}:${sessionId}:${capabilityId}`
     : `opengeni:workspace-connect:${workspaceId}:${capabilityId}`;
   const current = () => !!lifetime.current && !lifetime.current.signal.aborted;
 
-  async function load() {
+  async function load(active: () => boolean = current) {
     const invocation = lifetime.current;
     const [catalog, session] = await Promise.all([
       client.listCapabilities(workspaceId),
       sessionId ? client.getSession(workspaceId, sessionId) : Promise.resolve(null),
     ]);
-    if (!invocation || invocation.signal.aborted || lifetime.current !== invocation) return null;
+    if (!invocation || invocation.signal.aborted || lifetime.current !== invocation || !active())
+      return null;
     const resolved = catalog.items.find((entry) => entry.id === capabilityId);
     if (!resolved || resolved.kind !== "mcp" || resolved.authKind !== "oauth2")
       throw new Error(
@@ -112,11 +120,11 @@ function ScopedCard({
         resolved.connectionRef.subjectScope === "subject"
           ? await client.listOwnConnectionAccounts(workspaceId)
           : await client.listConnections(workspaceId);
-      if (invocation.signal.aborted || lifetime.current !== invocation) return null;
+      if (invocation.signal.aborted || lifetime.current !== invocation || !active()) return null;
       const matches = matchingActiveMcpConnections(resolved, connections);
       accountReady = matches.length === 1;
     }
-    if (invocation.signal.aborted || lifetime.current !== invocation) return null;
+    if (invocation.signal.aborted || lifetime.current !== invocation || !active()) return null;
     const selected =
       session?.toolPolicy.mode === "workspace_default"
         ? session.effectiveToolPolicy?.selectedIds
@@ -185,23 +193,32 @@ function ScopedCard({
     };
   }, [client, item?.logoAssetPath]);
 
-  async function run(action: () => Promise<void>) {
+  async function run(action: (active: () => boolean) => Promise<void>) {
     if (operation.current || !current()) return;
-    operation.current = true;
+    const pending = new AbortController();
+    operation.current = pending;
+    const active = () => current() && operation.current === pending && !pending.signal.aborted;
     setBusy(true);
     setError(null);
+    setNotice(null);
+    setRetryFresh(false);
     try {
-      await action();
+      await action(active);
     } catch (failure) {
-      if (current())
-        setError(
-          failure instanceof Error
-            ? failure.message
-            : "Connection setup could not finish. Try again.",
-        );
+      if (active()) {
+        if (failure instanceof ConnectPopupClosedError) setNotice(failure.message);
+        else
+          setError(
+            failure instanceof Error
+              ? failure.message
+              : "Connection setup could not finish. Try again.",
+          );
+      }
     } finally {
-      operation.current = false;
-      if (current()) setBusy(false);
+      if (operation.current === pending) {
+        operation.current = null;
+        if (current()) setBusy(false);
+      }
     }
   }
 
@@ -215,15 +232,15 @@ function ScopedCard({
     }
   }
 
-  async function reconcile(attempt: ConnectAttempt) {
-    if (!current()) return;
+  async function reconcile(attempt: ConnectAttempt, active: () => boolean = current) {
+    if (!active()) return;
     if (attempt.state !== "complete" || !attempt.account || !attempt.credentialsCommitted) return;
-    const resolved = await load();
-    if (!resolved || !current()) return;
+    const resolved = await load(active);
+    if (!resolved || !active()) return;
     const connection = (await client.listConnections(workspaceId)).find(
       (entry) => entry.id === attempt.account!.id,
     );
-    if (!current()) return;
+    if (!active()) return;
     if (
       !connection ||
       connection.status !== "active" ||
@@ -255,11 +272,12 @@ function ScopedCard({
               connectionId: connection.id,
             },
     });
-    if (!current()) return;
-    const enabled = await load();
-    if (enabled && current()) {
-      await finishConnection(enabled);
+    if (!active()) return;
+    const enabled = await load(active);
+    if (enabled && active()) {
+      await finishConnection(enabled, active);
     }
+    if (!active()) return;
     try {
       sessionStorage.removeItem(storageKey);
     } catch {
@@ -269,14 +287,14 @@ function ScopedCard({
 
   async function open() {
     setExpanded(true);
-    await run(async () => {
-      await load();
-      if (!current()) return;
-      await recoverSavedAttempt();
+    await run(async (active) => {
+      await load(active);
+      if (!active()) return;
+      await recoverSavedAttempt(active);
     });
   }
 
-  async function recoverSavedAttempt() {
+  async function recoverSavedAttempt(active: () => boolean = current) {
     let saved: string | null = null;
     try {
       saved = sessionStorage.getItem(storageKey);
@@ -285,20 +303,31 @@ function ScopedCard({
     }
     if (saved) {
       const attempt = await controller.recover(saved);
-      await reconcile(attempt);
+      if (!active()) return;
+      if (["failed", "expired", "cancelled"].includes(attempt.state)) setRetryFresh(true);
+      if (attempt.state === "complete") setReconciling(true);
+      try {
+        await reconcile(attempt, active);
+      } finally {
+        if (active()) setReconciling(false);
+      }
     }
   }
 
-  async function begin() {
-    await run(async () => {
-      if (view.attempt && ["failed", "expired", "cancelled"].includes(view.attempt.state)) {
+  async function begin(fresh = false) {
+    await run(async (active) => {
+      if (
+        fresh ||
+        (view.attempt && ["failed", "expired", "cancelled"].includes(view.attempt.state))
+      ) {
         startKey.current = crypto.randomUUID();
         advanceKey.current = crypto.randomUUID();
       }
       const reserved = reserveBrowserConnectNavigation(window);
+      reservedPopup.current = reserved.close;
       try {
-        const resolved = await load();
-        if (!resolved || !current()) return;
+        const resolved = await load(active);
+        if (!resolved || !active()) return;
         if (
           resolved.connectionRef &&
           ownership !==
@@ -322,13 +351,22 @@ function ScopedCard({
               entry.kind === "oauth2" &&
               entry.metadata.mcpUrl === mcpUrl,
           );
-          if (!current()) return;
+          if (!active()) return;
           if (connections.length > 1)
             throw new Error(
               "More than one personal account matches. Choose the account in connection settings before reconnecting.",
             );
           reconnectAccountId = connections[0]?.id;
         }
+        // A closed request can still have reached the server. Keep its key for
+        // an exact replay, but never reuse it when ownership or another input
+        // changes while the user reopens setup.
+        const input = JSON.stringify({ ownership, returnUrl, reconnectAccountId, mcpUrl });
+        if (startInput.current !== null && startInput.current !== input) {
+          startKey.current = crypto.randomUUID();
+          advanceKey.current = crypto.randomUUID();
+        }
+        startInput.current = input;
         let attempt = await controller.begin({
           providerId: "mcp-oauth",
           ownership,
@@ -336,49 +374,79 @@ function ScopedCard({
           idempotencyKey: startKey.current,
           ...(reconnectAccountId ? { reconnectAccountId } : {}),
         });
-        if (!current()) return;
+        if (!active()) return;
         remember(attempt.id);
         if (attempt.state === "credential_input")
           attempt = await controller.advance(
             { type: "credentials", values: { mcpUrl } },
             advanceKey.current,
           );
+        if (!active()) return;
         if (attempt.nextAction.type === "authorize")
-          await performAuthorization(attempt, reserved.navigation);
-        else await reconcile(attempt);
+          await performAuthorization(attempt, reserved.navigation, active);
+        else await reconcile(attempt, active);
       } finally {
+        if (reservedPopup.current === reserved.close) reservedPopup.current = null;
         reserved.close();
       }
     });
   }
 
-  async function performAuthorization(attempt: ConnectAttempt, navigation: ConnectNavigation) {
+  async function performAuthorization(
+    attempt: ConnectAttempt,
+    navigation: ConnectNavigation,
+    active: () => boolean,
+  ) {
+    if (!active()) return;
+    const pending = new AbortController();
+    authorization.current = pending;
+    const abortOnUnmount = () => pending.abort(lifetime.current?.signal.reason);
+    lifetime.current!.signal.addEventListener("abort", abortOnUnmount, { once: true });
     setWaiting(true);
     try {
       const result = await authorizeConnectAttempt(controller.transport, attempt, navigation, {
         mode: "popup",
-        signal: lifetime.current!.signal,
+        signal: pending.signal,
       });
-      if (!current() || !result) return;
-      await controller.refresh();
-      await reconcile(result);
-      if (result.state !== "complete")
+      if (!active() || !result) return;
+      setWaiting(false);
+      setReconciling(true);
+      await reconcile(result, active);
+      if (!active()) return;
+      if (result.state === "cancelled") {
+        setNotice("Sign-in was cancelled. You can try connecting again.");
+        setRetryFresh(true);
+        return;
+      }
+      if (result.state !== "complete") {
+        if (["failed", "expired"].includes(result.state)) setRetryFresh(true);
         throw new Error("Sign-in did not finish. You can try connecting again.");
+      }
     } finally {
-      if (current()) setWaiting(false);
+      lifetime.current?.signal.removeEventListener("abort", abortOnUnmount);
+      if (authorization.current === pending) authorization.current = null;
+      if (active()) {
+        setReconciling(false);
+        setWaiting(false);
+      }
     }
   }
 
   function authorize(attempt: ConnectAttempt) {
     // run invokes action synchronously, retaining the browser click gesture.
-    return run(() => performAuthorization(attempt, createBrowserConnectNavigation(window)));
+    return run((active) =>
+      performAuthorization(attempt, createBrowserConnectNavigation(window), active),
+    );
   }
 
-  async function finishConnection(capability: CapabilityCatalogItem) {
+  async function finishConnection(
+    capability: CapabilityCatalogItem,
+    active: () => boolean = current,
+  ) {
     if (!sessionId) {
       // Connection management does not select session tools.
       await onConfigured?.();
-      if (!current()) return;
+      if (!active()) return;
       setComplete(true);
       setExpanded(false);
       onClose?.();
@@ -388,31 +456,47 @@ function ScopedCard({
       const selected = (await client.listConnections(workspaceId)).find(
         (entry) => entry.id === capability.connectionRef?.connectionId,
       );
-      if (!current()) return;
+      if (!active()) return;
       if (!selected || selected.status !== "active")
         throw new Error("This account needs reconnection before it can be used here.");
     }
     if (capability.connectionRef?.subjectScope === "subject") {
       const accounts = await client.listOwnConnectionAccounts(workspaceId);
-      if (!current()) return;
+      if (!active()) return;
       if (matchingActiveMcpConnections(capability, accounts).length === 0)
         throw new Error("Your account needs reconnection before it can be used here.");
     }
-    if (!current()) return;
-    await attachSessionCapability(client, workspaceId, sessionId, capability, current);
-    if (!current()) return;
+    if (!active()) return;
+    await attachSessionCapability(client, workspaceId, sessionId, capability, active);
+    if (!active()) return;
     await onConfigured?.();
-    if (!current()) return;
+    if (!active()) return;
     setComplete(true);
     setExpanded(false);
   }
 
   async function useHere() {
-    await run(async () => {
-      if (item) await finishConnection(item);
+    await run(async (active) => {
+      if (item) await finishConnection(item, active);
     });
   }
 
+  function close() {
+    const pending = operation.current;
+    if (pending) {
+      operation.current = null;
+      pending.abort(new ConnectPopupClosedError());
+      authorization.current?.abort(new ConnectPopupClosedError());
+      authorization.current = null;
+      reservedPopup.current?.();
+      setBusy(false);
+      setWaiting(false);
+      setReconciling(false);
+    }
+    setExpanded(false);
+    onClose?.();
+  }
+
   return (
     <SessionCapabilityFrame
       name={item?.name ?? name}
@@ -436,15 +520,16 @@ function ScopedCard({
           : "Review access before signing in. You'll return here after authorization."
       }
       onOpen={() => void open()}
-      onClose={() => {
-        setExpanded(false);
-        onClose?.();
-      }}
-      busy={busy || view.busy}
+      onClose={close}
       dialogOnly={dialogOnly}
     >
       <div className="og-session-capability-setup">
         {error ? <p role="alert">{error}</p> : null}
+        {notice ? (
+          <p role="status" className="og-session-capability-notice">
+            {notice}
+          </p>
+        ) : null}
         {!item ? (
           <>
             <p role="status">
@@ -458,11 +543,24 @@ function ScopedCard({
           <>
             <p>{item.description || rationale}</p>
             {busy ? (
-              <p role="status" className="og-session-capability-progress">
-                {waiting
-                  ? `Finish signing in with ${item.name} in the opened window. This will close automatically when you’re connected.`
-                  : "Preparing your connection…"}
-              </p>
+              <>
+                <p role="status" className="og-session-capability-progress">
+                  {waiting
+                    ? `Finish signing in with ${item.name} in the opened window. This will close automatically when you’re connected.`
+                    : reconciling
+                      ? "Finishing your connection…"
+                      : "Preparing your connection…"}
+                </p>
+                {waiting ? (
+                  <button
+                    type="button"
+                    className="og-session-capability-stop"
+                    onClick={() => authorization.current?.abort(new ConnectPopupClosedError())}
+                  >
+                    Stop waiting
+                  </button>
+                ) : null}
+              </>
             ) : !connected ? (
               <>
                 {!item.connectionRef ? (
@@ -498,11 +596,17 @@ function ScopedCard({
                 <button
                   className="og-session-capability-primary"
                   onClick={() => {
-                    if (view.attempt?.nextAction.type === "authorize") void authorize(view.attempt);
+                    if (notice || retryFresh) void begin(true);
+                    else if (
+                      view.attempt?.nextAction.type === "authorize" &&
+                      view.attempt.ownership === ownership
+                    )
+                      void authorize(view.attempt);
                     else void begin();
                   }}
                 >
-                  {error && view.attempt?.nextAction.type === "authorize"
+                  {retryFresh ||
+                  ((error || notice) && view.attempt?.nextAction.type === "authorize")
                     ? "Try signing in again"
                     : `Continue to ${item.name}`}
                 </button>

package/src/hooks/use-browser-session.ts

@@ -23,6 +23,9 @@ export type UseBrowserSessionOptions = EmbeddedBrowserInteractionClientOverride
   browserSessionId: string | null;
   enabled?: boolean | undefined;
   pollIntervalMs?: number | undefined;
+  /** Keep session/tab inventory current without repeatedly fetching the full
+   * accessibility tree while a live frame is available or the viewer is hidden. */
+  semanticObservationEnabled?: boolean | undefined;
 };
 
 export type UseBrowserSessionResult = {
@@ -58,6 +61,13 @@ export function useBrowserSession(options: UseBrowserSessionOptions): UseBrowser
   const { client, workspaceId } = useEmbeddedBrowserInteraction(options);
   const browserSessionId = options.browserSessionId;
   const enabled = (options.enabled ?? true) && browserSessionId !== null;
+  const semanticObservationEnabled = options.semanticObservationEnabled ?? true;
+  const semanticObservationEnabledRef = useRef(semanticObservationEnabled);
+  semanticObservationEnabledRef.current = semanticObservationEnabled;
+  const previousSemanticObservationRef = useRef({
+    browserSessionId,
+    enabled: semanticObservationEnabled,
+  });
   const pageLive = usePageLiveActivity();
   const pollIntervalMs = Math.max(750, options.pollIntervalMs ?? 2_000);
   const [state, setState] = useState<{
@@ -75,10 +85,16 @@ export function useBrowserSession(options: UseBrowserSessionOptions): UseBrowser
   const refreshBlocked = isNonRetryableInteractionError(visible.error);
   const selectedTargetIdRef = useRef<string | null>(visible.selectedTargetId);
   selectedTargetIdRef.current = visible.selectedTargetId;
+  const selectedTargetRef = useRef<BrowserTarget | null>(
+    visible.targets.find((target) => target.id === visible.selectedTargetId) ?? null,
+  );
+  selectedTargetRef.current =
+    visible.targets.find((target) => target.id === visible.selectedTargetId) ?? null;
   const observationRef = useRef<{
     browserSessionId: string | null;
     observation: BrowserObservation | null;
   }>({ browserSessionId, observation: visible.observation });
+  const lastObservationAtRef = useRef(0);
   if (observationRef.current.browserSessionId !== browserSessionId) {
     observationRef.current = {
       browserSessionId,
@@ -124,7 +140,15 @@ export function useBrowserSession(options: UseBrowserSessionOptions): UseBrowser
       if (!mountedRef.current || requestRef.current.id !== id) return;
       const selected = chooseTarget(targetResponse.targets, selectedTargetIdRef.current);
       selectedTargetIdRef.current = selected?.id ?? null;
-      observationRef.current = { browserSessionId, observation: null };
+      const previousObservation =
+        observationRef.current.browserSessionId === browserSessionId
+          ? observationRef.current.observation
+          : null;
+      const retainedObservation =
+        selected && previousObservation && sameObservationTarget(previousObservation, selected)
+          ? previousObservation
+          : null;
+      observationRef.current = { browserSessionId, observation: retainedObservation };
       setState((current) =>
         current.browserSessionId === browserSessionId
           ? {
@@ -132,13 +156,13 @@ export function useBrowserSession(options: UseBrowserSessionOptions): UseBrowser
               session,
               targets: targetResponse.targets,
               selectedTargetId: selected?.id ?? null,
-              observation: null,
+              observation: retainedObservation,
               loading: false,
               error: null,
             }
           : current,
       );
-      if (!selected) return;
+      if (!selected || !semanticObservationEnabledRef.current) return;
       try {
         const observation = await client.observeBrowserTarget(
           workspaceId,
@@ -148,6 +172,7 @@ export function useBrowserSession(options: UseBrowserSessionOptions): UseBrowser
         );
         if (!mountedRef.current || requestRef.current.id !== id) return;
         observationRef.current = { browserSessionId, observation };
+        lastObservationAtRef.current = Date.now();
         setState((current) =>
           current.browserSessionId === browserSessionId &&
           current.selectedTargetId === observation.target.id
@@ -180,6 +205,37 @@ export function useBrowserSession(options: UseBrowserSessionOptions): UseBrowser
     }
   }, [browserSessionId, client, enabled, workspaceId]);
 
+  const refreshSemantic = useCallback(async (): Promise<void> => {
+    if (!enabled || !browserSessionId) return;
+    const target = selectedTargetRef.current;
+    if (!target) return;
+    const id = requestRef.current.id + 1;
+    requestRef.current.controller?.abort();
+    const controller = new AbortController();
+    requestRef.current = { id, controller };
+    try {
+      const observation = await client.observeBrowserTarget(
+        workspaceId,
+        browserSessionId,
+        target.id,
+        { signal: controller.signal },
+      );
+      if (!mountedRef.current || requestRef.current.id !== id) return;
+      if (!sameObservationTarget(observation, selectedTargetRef.current)) return;
+      observationRef.current = { browserSessionId, observation };
+      lastObservationAtRef.current = Date.now();
+      setState((current) =>
+        current.browserSessionId === browserSessionId && current.selectedTargetId === target.id
+          ? { ...current, observation }
+          : current,
+      );
+    } catch {
+      // A failed semantic fallback cannot disable an otherwise live browser.
+    } finally {
+      if (requestRef.current.id === id) requestRef.current = { id, controller: null };
+    }
+  }, [browserSessionId, client, enabled, workspaceId]);
+
   useEffect(() => {
     mountedRef.current = true;
     if (!enabled) {
@@ -199,6 +255,37 @@ export function useBrowserSession(options: UseBrowserSessionOptions): UseBrowser
     };
   }, [browserSessionId, enabled, invalidateRefresh, refresh]);
 
+  useEffect(() => {
+    const previous = previousSemanticObservationRef.current;
+    previousSemanticObservationRef.current = {
+      browserSessionId,
+      enabled: semanticObservationEnabled,
+    };
+    if (
+      enabled &&
+      semanticObservationEnabled &&
+      previous.browserSessionId === browserSessionId &&
+      !previous.enabled
+    ) {
+      // The media stream disappeared or the user returned to the Browser tab.
+      // A just-returned action/tab observation is already a fresh fallback.
+      // Otherwise drop the old tree/fence until a new read settles.
+      const currentObservation = observationRef.current.observation;
+      if (
+        currentObservation &&
+        sameObservationTarget(currentObservation, selectedTargetRef.current) &&
+        Date.now() - lastObservationAtRef.current < pollIntervalMs
+      ) {
+        return;
+      }
+      observationRef.current = { browserSessionId, observation: null };
+      setState((current) =>
+        current.browserSessionId === browserSessionId ? { ...current, observation: null } : current,
+      );
+      void refreshSemantic();
+    }
+  }, [browserSessionId, enabled, pollIntervalMs, refreshSemantic, semanticObservationEnabled]);
+
   useEffect(() => {
     if (!enabled || !pageLive || visible.mutating || refreshBlocked) return;
     const timer = setInterval(() => {
@@ -301,6 +388,7 @@ export function useBrowserSession(options: UseBrowserSessionOptions): UseBrowser
         }
         selectedTargetIdRef.current = target.id;
         observationRef.current = { browserSessionId, observation };
+        lastObservationAtRef.current = Date.now();
         setState((current) =>
           current.browserSessionId === browserSessionId
             ? {
@@ -332,6 +420,7 @@ export function useBrowserSession(options: UseBrowserSessionOptions): UseBrowser
         const target = observation.target;
         selectedTargetIdRef.current = target.id;
         observationRef.current = { browserSessionId, observation };
+        lastObservationAtRef.current = Date.now();
         setState((current) =>
           current.browserSessionId === browserSessionId
             ? {
@@ -365,6 +454,7 @@ export function useBrowserSession(options: UseBrowserSessionOptions): UseBrowser
           : null;
         selectedTargetIdRef.current = selected?.id ?? null;
         observationRef.current = { browserSessionId, observation };
+        if (observation) lastObservationAtRef.current = Date.now();
         setState((current) =>
           current.browserSessionId === browserSessionId
             ? {
@@ -424,6 +514,7 @@ export function useBrowserSession(options: UseBrowserSessionOptions): UseBrowser
             browserSessionId,
             observation: receipt.observation,
           };
+          lastObservationAtRef.current = Date.now();
           setState((current) =>
             current.browserSessionId === browserSessionId
               ? {
@@ -540,6 +631,19 @@ function replaceTarget(targets: readonly BrowserTarget[], target: BrowserTarget)
   return next.sort((left, right) => left.createdAt.localeCompare(right.createdAt));
 }
 
+function sameObservationTarget(
+  observation: BrowserObservation,
+  target: BrowserTarget | null,
+): boolean {
+  return (
+    target !== null &&
+    observation.target.id === target.id &&
+    observation.target.controllerGeneration === target.controllerGeneration &&
+    observation.target.targetGeneration === target.targetGeneration &&
+    observation.target.documentGeneration === target.documentGeneration
+  );
+}
+
 function isMissingBrowserTarget(error: unknown): boolean {
   return error instanceof OpenGeniApiError && error.code === "target_not_found";
 }

package/src/timeline/parsers.ts

@@ -566,8 +566,8 @@ export function retainedScreenshotMetadata(out: unknown): RetainedArtifactMetada
   if (!value.available) {
     return typeof value.reason === "string" ? (value as unknown as RetainedArtifactMetadata) : null;
   }
-  return value.kind === "computer_screenshot" &&
-    value.contentType === "image/png" &&
+  return (value.kind === "computer_screenshot" || value.kind === "browser_screenshot") &&
+    ["image/png", "image/jpeg", "image/webp"].includes(String(value.contentType)) &&
     typeof value.originalBytes === "number" &&
     typeof value.sha256 === "string" &&
     value.dimensions !== null &&