@opengeni/react 7.6.1 → 7.7.0-canary.37090756480001

package/dist/{chunk-6NEZPF53.js → chunk-YN7I3STW.js}

@@ -1,10 +1,11 @@
 import {
   InteractionInterventionBanner,
+  isInteractionControlUnavailable,
   isNonRetryableInteractionError,
   isSourcePlacementChangedError,
   useInteractionInterventions,
   useInteractionInvalidation
-} from "./chunk-YSMMDOOE.js";
+} from "./chunk-GISEVGS6.js";
 import {
   copyTextToClipboard
 } from "./chunk-JVYQRRC4.js";
@@ -22,7 +23,7 @@ import {
 } from "./chunk-5J4FOW6Q.js";
 import {
   usePageLiveActivity
-} from "./chunk-KZ73RL76.js";
+} from "./chunk-YQEAA6LT.js";
 import {
   useEmbeddedComputerInteraction
 } from "./chunk-NEXLLTP4.js";
@@ -452,6 +453,7 @@ function useComputerSession(options) {
     requestRef.current.controller?.abort();
     const controller = new AbortController();
     requestRef.current = { id, controller };
+    let discoveryCompleted = false;
     try {
       const [session, targetResponse] = await Promise.all([
         client.getComputerSession(workspaceId, computerSessionId, {
@@ -461,6 +463,7 @@ function useComputerSession(options) {
           signal: controller.signal
         })
       ]);
+      discoveryCompleted = true;
       if (!mountedRef.current || requestRef.current.id !== id) return;
       const targets = sortComputerTargets(targetResponse.targets);
       const selected = chooseTarget(targets, selectedTargetIdRef.current, session.platform);
@@ -477,7 +480,8 @@ function useComputerSession(options) {
           selectedTargetId: selected?.id ?? null,
           observation: retainedObservation,
           loading: false,
-          error: null
+          error: null,
+          controlError: null
         } : current
       );
       const observation = selected ? await client.observeComputerTarget(workspaceId, computerSessionId, selected.id, {
@@ -500,11 +504,16 @@ function useComputerSession(options) {
       );
     } catch (cause) {
       if (controller.signal.aborted || !mountedRef.current || requestRef.current.id !== id) return;
+      const error = cause instanceof Error ? cause : new Error(String(cause));
       setState(
         (current) => current.computerSessionId === computerSessionId ? {
           ...current,
           loading: false,
-          error: cause instanceof Error ? cause : new Error(String(cause))
+          error,
+          controlError: isInteractionControlUnavailable(
+            error,
+            discoveryCompleted ? "observation" : "control"
+          ) ? error : current.controlError
         } : current
       );
     } finally {
@@ -561,11 +570,19 @@ function useComputerSession(options) {
         (current) => current.computerSessionId === scopeComputerSessionId ? { ...current, mutating: true, error: null } : current
       );
       try {
-        return await operation();
+        const result = await operation();
+        setState(
+          (current) => current.computerSessionId === scopeComputerSessionId ? { ...current, controlError: null } : current
+        );
+        return result;
       } catch (cause) {
         const error = cause instanceof Error ? cause : new Error(String(cause));
         setState(
-          (current) => current.computerSessionId === scopeComputerSessionId ? { ...current, error } : current
+          (current) => current.computerSessionId === scopeComputerSessionId ? {
+            ...current,
+            error,
+            controlError: isInteractionControlUnavailable(error) ? error : current.controlError
+          } : current
         );
         throw error;
       } finally {
@@ -612,7 +629,12 @@ function useComputerSession(options) {
       } catch (cause) {
         const error = cause instanceof Error ? cause : new Error(String(cause));
         setState(
-          (current) => current.computerSessionId === computerSessionId && current.selectedTargetId === target.id ? { ...current, loading: false, error } : current
+          (current) => current.computerSessionId === computerSessionId && current.selectedTargetId === target.id ? {
+            ...current,
+            loading: false,
+            error,
+            controlError: isInteractionControlUnavailable(error, "observation") ? error : current.controlError
+          } : current
         );
         throw error;
       }
@@ -684,7 +706,19 @@ function useComputerSession(options) {
   );
   const readClipboard = useCallback2(async () => {
     if (!computerSessionId) throw new Error("No desktop is selected.");
-    return await client.readComputerClipboard(workspaceId, computerSessionId);
+    try {
+      return await client.readComputerClipboard(workspaceId, computerSessionId);
+    } catch (cause) {
+      const error = cause instanceof Error ? cause : new Error(String(cause));
+      setState(
+        (current) => current.computerSessionId === computerSessionId ? {
+          ...current,
+          error,
+          controlError: isInteractionControlUnavailable(error) ? error : current.controlError
+        } : current
+      );
+      throw error;
+    }
   }, [client, computerSessionId, workspaceId]);
   return {
     session: visible.session,
@@ -694,6 +728,7 @@ function useComputerSession(options) {
     loading: visible.loading,
     mutating: visible.mutating,
     error: visible.error,
+    controlError: visible.controlError,
     refresh,
     readClipboard,
     selectTarget,
@@ -710,7 +745,8 @@ function emptyState(computerSessionId, loading) {
     observation: null,
     loading,
     mutating: false,
-    error: null
+    error: null,
+    controlError: null
   };
 }
 function chooseTarget(targets, preferredId, platform) {
@@ -1084,7 +1120,9 @@ function ComputerViewer({
   const displayedFrame = frames.frame && frames.frame.computerSessionId === selection?.sessionId && frames.frame.targetId === computer.selectedTarget?.id && frames.frame.targetGeneration === computer.selectedTarget?.targetGeneration ? frames.frame : null;
   const rfbStream = frames.attachment?.stream.kind === "direct_rfb" ? frames.attachment.stream : null;
   const machineLocked = selectedRegistrySession?.failureCode === "machine_locked";
-  const rfbInputEnabled = !machineLocked && computer.session?.capabilities?.pointerInput === true && computer.session?.capabilities?.keyboardInput === true;
+  const controlUnavailable = computer.controlError !== null;
+  const connectionState = controlUnavailable ? "error" : frames.state;
+  const rfbInputEnabled = !machineLocked && !controlUnavailable && computer.session?.capabilities?.pointerInput === true && computer.session?.capabilities?.keyboardInput === true;
   const rfbCapability = useMemo2(() => {
     if (!rfbStream || !frames.attachment) return null;
     const bounds = computer.selectedTarget?.bounds;
@@ -1192,6 +1230,7 @@ function ComputerViewer({
   ]);
   const perform = useCallback4(
     async (action, frame) => {
+      if (computer.controlError) throw computer.controlError;
       let receipt;
       if (action.type === "pointer") {
         if (!frame) throw new Error("Desktop view is not ready for pointer input.");
@@ -1203,8 +1242,12 @@ function ComputerViewer({
         throw new Error(receipt.error?.message ?? "Desktop input did not complete.");
       }
     },
-    [act, actFromFrame]
+    [act, actFromFrame, computer.controlError]
   );
+  const reconnect = () => {
+    void refreshComputer();
+    frames.reconnect();
+  };
   const copyFromRfb = useCallback4(
     (event) => {
       if (!rfbInputEnabled || !rfbStream || computer.session?.capabilities?.clipboard !== true)
@@ -1298,7 +1341,7 @@ function ComputerViewer({
             },
             onCreate: hideGenericCreate ? void 0 : createComputer,
             onRefresh: () => {
-              if (frames.state === "error" && !generationLossSelected && !generationLossFrames) {
+              if (connectionState === "error" && !generationLossSelected && !generationLossFrames) {
                 frames.reconnect();
               }
               void Promise.all([refreshRegistry(), refreshComputer()]);
@@ -1356,7 +1399,19 @@ function ComputerViewer({
                   className: "h-full"
                 }
               ),
-              !rfbInputEnabled ? /* @__PURE__ */ jsx("div", { className: "pointer-events-none absolute bottom-3 left-1/2 -translate-x-1/2 rounded-full border border-white/10 bg-black/70 px-3 py-1 text-[11px] text-white/80 backdrop-blur", children: "View only \xB7 direct input unavailable" }) : null
+              !rfbInputEnabled ? /* @__PURE__ */ jsx("div", { className: "pointer-events-none absolute bottom-3 left-1/2 -translate-x-1/2 rounded-full border border-white/10 bg-black/70 px-3 py-1 text-[11px] text-white/80 backdrop-blur", children: controlUnavailable ? /* @__PURE__ */ jsxs(Fragment, { children: [
+                "Desktop controls unavailable",
+                " ",
+                /* @__PURE__ */ jsx(
+                  "button",
+                  {
+                    type: "button",
+                    onClick: reconnect,
+                    className: "pointer-events-auto ml-2 font-medium underline",
+                    children: "Reconnect"
+                  }
+                )
+              ] }) : "View only \xB7 direct input unavailable" }) : null
             ] }) : /* @__PURE__ */ jsx(
               ComputerViewport,
               {
@@ -1364,16 +1419,18 @@ function ComputerViewer({
                 observation: computer.observation,
                 target: computer.selectedTarget,
                 machineLocked,
-                connectionState: frames.state,
-                connectionError: frames.error ?? computer.error,
+                controlUnavailable,
+                connectionState,
+                connectionError: computer.controlError ?? frames.error ?? computer.error,
                 mutating: computer.mutating,
                 backgroundActions: computer.session?.capabilities?.backgroundActions === true,
+                backgroundInput: computer.session?.capabilities?.backgroundInput === true,
                 clipboardEnabled: computer.session?.capabilities?.clipboard === true,
                 pointerInput: computer.session?.capabilities?.pointerInput === true,
                 keyboardInput: computer.session?.capabilities?.keyboardInput === true,
                 onAction: perform,
                 onReadClipboard: computer.readClipboard,
-                onReconnect: generationLossSelected || generationLossFrames ? () => void 0 : frames.reconnect,
+                onReconnect: generationLossSelected || generationLossFrames ? () => void 0 : reconnect,
                 onError: (cause) => notifyError(cause, "Desktop input failed.")
               },
               JSON.stringify([
@@ -1387,7 +1444,8 @@ function ComputerViewer({
               ComputerSemanticPanel,
               {
                 observation: computer.observation,
-                mutating: computer.mutating,
+                mutating: computer.mutating || controlUnavailable,
+                controlUnavailable,
                 onAction: (action) => void perform(action, null).catch(
                   (cause) => notifyError(cause, "Desktop action failed.")
                 )
@@ -1399,10 +1457,11 @@ function ComputerViewer({
             {
               session: computer.session,
               target: computer.selectedTarget,
-              connectionState: frames.state,
+              connectionState,
+              controlUnavailable,
               refreshing: registry.refreshing,
               showControls,
-              controlCount: semanticNodes(computer.observation).length,
+              controlCount: semanticControls(computer.observation).length,
               onToggleControls: () => setShowControls((current) => !current)
             }
           )
@@ -1627,6 +1686,7 @@ function ComputerLifecyclePanel(props) {
 function ComputerViewport(props) {
   const canvasRef = useRef4(null);
   const inputRef = useRef4(null);
+  const composingRef = useRef4(false);
   const pointerStartRef = useRef4(null);
   const clickTimerRef = useRef4(null);
   const lastClickRef = useRef4(null);
@@ -1647,7 +1707,7 @@ function ComputerViewport(props) {
   readClipboardRef.current = props.onReadClipboard;
   errorRef.current = props.onError;
   const streamFailed = props.connectionState === "error";
-  const rawInputEnabled = !streamFailed && !props.machineLocked && (!props.backgroundActions || props.target?.kind === "screen" || props.target?.focused === true);
+  const rawInputEnabled = !streamFailed && !props.controlUnavailable && !props.machineLocked && (props.backgroundInput || !props.backgroundActions || props.target?.kind === "screen" || props.target?.focused === true);
   const clearBufferedInput = useCallback4(() => {
     if (clickTimerRef.current) clearTimeout(clickTimerRef.current);
     if (wheelRef.current?.timer) clearTimeout(wheelRef.current.timer);
@@ -1657,10 +1717,25 @@ function ComputerViewport(props) {
     pendingTextRef.current = null;
     pointerStartRef.current = null;
     lastClickRef.current = null;
+    composingRef.current = false;
     if (inputRef.current) inputRef.current.value = "";
   }, []);
   const pointerInputEnabled = rawInputEnabled && props.pointerInput;
   const keyboardInputEnabled = rawInputEnabled && props.keyboardInput;
+  useLayoutEffect(() => {
+    const canvas = canvasRef.current;
+    const viewport = canvas?.parentElement;
+    if (!canvas || !viewport) return;
+    const fit = () => fitComputerCanvas(canvas);
+    fit();
+    if (typeof ResizeObserver === "undefined") {
+      window.addEventListener("resize", fit);
+      return () => window.removeEventListener("resize", fit);
+    }
+    const observer = new ResizeObserver(fit);
+    observer.observe(viewport);
+    return () => observer.disconnect();
+  }, []);
   const paintQueuedFrames = useCallback4(() => {
     if (decodingFrameRef.current) return;
     decodingFrameRef.current = true;
@@ -1925,6 +2000,9 @@ function ComputerViewport(props) {
   };
   const keyDown = (event) => {
     if (!keyboardInputEnabled) return;
+    if (composingRef.current || event.nativeEvent.isComposing || event.nativeEvent.keyCode === 229) {
+      return;
+    }
     flushPendingWheel();
     const command = event.metaKey || event.ctrlKey;
     if (props.clipboardEnabled && command && !event.altKey && ["c", "v"].includes(event.key.toLowerCase())) {
@@ -1970,7 +2048,8 @@ function ComputerViewport(props) {
       await dispatch({ type: "clipboard", operation: "paste" }, null);
     });
   };
-  const input = (value) => {
+  const input = (value, nativeComposing = false) => {
+    if (composingRef.current || nativeComposing) return;
     if (!value || !keyboardInputEnabled) return;
     flushPendingWheel();
     flushPendingClick();
@@ -1987,6 +2066,10 @@ function ComputerViewport(props) {
     }
     if (inputRef.current) inputRef.current.value = "";
   };
+  const compositionEnd = (event) => {
+    composingRef.current = false;
+    input(event.currentTarget.value || event.data);
+  };
   const showCanvas = props.frame !== null && paintedFrame !== null && sameComputerTarget(props.frame, paintedFrame) && !props.machineLocked && !streamFailed;
   return /* @__PURE__ */ jsxs("div", { className: "relative min-h-0 flex-1 overflow-hidden bg-black", children: [
     /* @__PURE__ */ jsx(
@@ -2013,7 +2096,11 @@ function ComputerViewport(props) {
       {
         ref: inputRef,
         defaultValue: "",
-        onInput: (event) => input(event.currentTarget.value),
+        onInput: (event) => input(event.currentTarget.value, event.nativeEvent.isComposing),
+        onCompositionStart: () => {
+          composingRef.current = true;
+        },
+        onCompositionEnd: compositionEnd,
         onKeyDown: keyDown,
         onCopy: copy,
         onPaste: paste,
@@ -2030,6 +2117,7 @@ function ComputerViewport(props) {
       {
         observation: props.observation,
         machineLocked: props.machineLocked,
+        controlUnavailable: props.controlUnavailable,
         connectionState: props.connectionState,
         error: props.connectionError,
         onAction: (action) => enqueue(action, null),
@@ -2069,14 +2157,14 @@ function ComputerViewportFallback(props) {
     ] }) });
   }
   const controlFailure = interactionControlFailureFromError(props.error);
-  const interactive = semanticNodes(props.observation).filter((node) => semanticAction(node) !== null).slice(0, 10);
+  const interactive = semanticControls(props.observation).filter((node) => semanticAction(node) !== null).slice(0, 10);
   return /* @__PURE__ */ jsx("div", { className: "absolute inset-0 grid place-items-center bg-og-bg p-6", children: /* @__PURE__ */ jsxs("div", { className: "w-full max-w-md rounded-og-lg border border-og-border bg-og-surface-1 p-4 shadow-lg", children: [
     /* @__PURE__ */ jsxs("div", { className: "flex items-center gap-2", children: [
       props.error ? /* @__PURE__ */ jsx(CircleAlertIcon, { className: "size-4 text-og-danger" }) : /* @__PURE__ */ jsx(LoaderCircleIcon, { className: "size-4 animate-spin text-og-fg-muted" }),
-      /* @__PURE__ */ jsx("p", { className: "text-og-menu font-medium text-og-fg", children: isAttachedChromeGenerationLossError(props.error) ? "Chrome reconnected\u2014open a fresh browser/desktop." : props.error ? "Live view disconnected" : computerConnectionLabel(props.connectionState) })
+      /* @__PURE__ */ jsx("p", { className: "text-og-menu font-medium text-og-fg", children: isAttachedChromeGenerationLossError(props.error) ? "Chrome reconnected\u2014open a fresh browser/desktop." : props.controlUnavailable ? "Desktop controls unavailable" : props.error ? "Live view disconnected" : computerConnectionLabel(props.connectionState) })
     ] }),
     props.error ? /* @__PURE__ */ jsx("p", { className: "mt-2 text-og-control leading-5 text-og-fg-muted", children: isAttachedChromeGenerationLossError(props.error) ? "Chrome reconnected\u2014open a fresh browser/desktop. Use Browser \u2192 New browser \u2192 Connected Chrome." : controlFailure?.message ?? props.error.message }) : null,
-    interactive.length > 0 ? /* @__PURE__ */ jsxs("div", { className: "mt-3 border-t border-og-border pt-3", children: [
+    interactive.length > 0 && !props.controlUnavailable ? /* @__PURE__ */ jsxs("div", { className: "mt-3 border-t border-og-border pt-3", children: [
       /* @__PURE__ */ jsx("p", { className: "mb-2 text-og-xs text-og-fg-subtle", children: "App controls remain available" }),
       /* @__PURE__ */ jsx("div", { className: "flex flex-wrap gap-1.5", children: interactive.map((node) => /* @__PURE__ */ jsx(
         "button",
@@ -2104,13 +2192,14 @@ function ComputerViewportFallback(props) {
   ] }) });
 }
 function ComputerSemanticPanel(props) {
-  const nodes = semanticNodes(props.observation).slice(0, 100);
+  const nodes = semanticControls(props.observation).slice(0, 100);
   return /* @__PURE__ */ jsxs("aside", { className: "box-border max-h-[40%] w-full shrink-0 overflow-y-auto border-t border-og-border bg-og-surface-1 p-2 @xl/computer-viewer:max-h-none @xl/computer-viewer:w-64 @xl/computer-viewer:border-t-0 @xl/computer-viewer:border-l", children: [
     /* @__PURE__ */ jsxs("div", { className: "mb-2 flex items-center gap-1.5 px-1 text-og-xs font-medium uppercase tracking-[0.1em] text-og-fg-subtle", children: [
       /* @__PURE__ */ jsx(KeyboardIcon, { className: "size-3" }),
       " App controls"
     ] }),
-    nodes.length === 0 ? /* @__PURE__ */ jsx("p", { className: "px-1 py-2 text-og-control leading-5 text-og-fg-muted", children: "This view has no accessibility controls. Use the live image instead." }) : /* @__PURE__ */ jsx("div", { className: "space-y-0.5", children: nodes.map((node) => /* @__PURE__ */ jsx(
+    props.controlUnavailable ? /* @__PURE__ */ jsx("p", { className: "px-1 py-2 text-og-control leading-5 text-og-fg-muted", children: "Reconnect to use app controls." }) : null,
+    nodes.length === 0 ? props.controlUnavailable ? null : /* @__PURE__ */ jsx("p", { className: "px-1 py-2 text-og-control leading-5 text-og-fg-muted", children: "This view has no accessibility controls. Use the live image instead." }) : /* @__PURE__ */ jsx("div", { className: "space-y-0.5", children: nodes.map((node) => /* @__PURE__ */ jsx(
       ComputerSemanticControl,
       {
         node,
@@ -2199,8 +2288,8 @@ function ComputerStatusBar(props) {
         )
       }
     ),
-    /* @__PURE__ */ jsx("span", { children: props.connectionState === "live" ? "Live" : computerConnectionLabel(props.connectionState) }),
-    /* @__PURE__ */ jsx("span", { className: "min-w-0 flex-1 truncate", children: screen ? "Full screen \xB7 input may move pointer and focus" : props.session?.capabilities?.backgroundActions ? "Window \xB7 app controls work in the background" : props.target?.kind ?? "Desktop" }),
+    /* @__PURE__ */ jsx("span", { children: props.controlUnavailable ? "Controls unavailable" : props.connectionState === "live" ? "Live" : computerConnectionLabel(props.connectionState) }),
+    /* @__PURE__ */ jsx("span", { className: "min-w-0 flex-1 truncate", children: props.controlUnavailable ? "Reconnect to use desktop input" : screen ? "Full screen \xB7 input may move pointer and focus" : props.session?.capabilities?.backgroundInput ? "Window \xB7 clicks and typing stay in the background" : props.session?.capabilities?.backgroundActions ? "Window \xB7 app controls work in the background" : props.target?.kind ?? "Desktop" }),
     screen ? /* @__PURE__ */ jsx(MousePointer2Icon, { className: "size-3", "aria-hidden": true }) : null,
     props.refreshing ? /* @__PURE__ */ jsx(LoaderCircleIcon, { className: "size-3 animate-spin" }) : null,
     /* @__PURE__ */ jsxs(
@@ -2236,11 +2325,30 @@ function ComputerNotice(props) {
     }
   );
 }
-function semanticNodes(observation) {
+var structuralRoles = /* @__PURE__ */ new Set([
+  "frame",
+  "panel",
+  "section",
+  "document",
+  "document web",
+  "static",
+  "static text",
+  "heading",
+  "paragraph",
+  "notification",
+  "separator",
+  "tool bar",
+  "page tab list"
+]);
+function semanticControls(observation) {
   if (observation?.semantic?.kind !== "snapshot") return [];
   const result = [];
   const visit = (node) => {
-    result.push(node);
+    const role = node.role.trim().toLowerCase();
+    const label = node.name?.trim() || node.identifier?.trim();
+    if (node.actions.includes("set_value") || label && label.toLowerCase() !== role && !structuralRoles.has(role) && semanticAction(node) !== null) {
+      result.push(node);
+    }
     node.children?.forEach(visit);
   };
   observation.semantic.roots.forEach(visit);
@@ -2261,11 +2369,24 @@ function semanticValue(node) {
   return typeof node.value === "string" ? node.value : null;
 }
 function paintCanvas(canvas, image, width, height) {
+  const dimensionsChanged = canvas.width !== width || canvas.height !== height;
   canvas.width = width;
   canvas.height = height;
   const context = canvas.getContext("2d", { alpha: false });
   if (!context) throw new Error("Desktop canvas is unavailable.");
   context.drawImage(image, 0, 0, width, height);
+  if (dimensionsChanged) fitComputerCanvas(canvas);
+}
+function fitComputerCanvas(canvas) {
+  const viewport = canvas.parentElement;
+  if (!viewport || canvas.width <= 0 || canvas.height <= 0) return;
+  const scale = Math.min(
+    viewport.clientWidth / canvas.width,
+    viewport.clientHeight / canvas.height
+  );
+  if (!Number.isFinite(scale) || scale <= 0) return;
+  canvas.style.width = `${canvas.width * scale}px`;
+  canvas.style.height = `${canvas.height * scale}px`;
 }
 function loadImage(url) {
   return new Promise((resolve, reject) => {
@@ -2423,4 +2544,4 @@ export {
   ComputerViewer,
   computerKey
 };
-//# sourceMappingURL=chunk-6NEZPF53.js.map
+//# sourceMappingURL=chunk-YN7I3STW.js.map