@opengeni/react 7.1.2 → 7.2.0-canary.36239117573001

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. package/dist/{browser-viewer-TZ6T76E3.js → browser-viewer-GEYFZE2W.js} +2 -2
  2. package/dist/{chunk-CP6IFD2Z.js → chunk-6IUUN2AQ.js} +124 -58
  3. package/dist/chunk-6IUUN2AQ.js.map +1 -0
  4. package/dist/{chunk-VHEY4CB3.js → chunk-RSFCYKA7.js} +5 -3
  5. package/dist/{chunk-VHEY4CB3.js.map → chunk-RSFCYKA7.js.map} +1 -1
  6. package/dist/{chunk-VI7HELBI.js → chunk-SJ5HE4ZA.js} +21 -2
  7. package/dist/chunk-SJ5HE4ZA.js.map +1 -0
  8. package/dist/{chunk-4ACTWOOW.js → chunk-VDSZ3AVO.js} +41 -18
  9. package/dist/chunk-VDSZ3AVO.js.map +1 -0
  10. package/dist/{chunk-2O2HXRGP.js → chunk-Y6DMQTLA.js} +130 -58
  11. package/dist/chunk-Y6DMQTLA.js.map +1 -0
  12. package/dist/composer.js +1 -1
  13. package/dist/{computer-viewer-FX4WOZMX.js → computer-viewer-BNHZ4PYO.js} +2 -2
  14. package/dist/index.js +5 -5
  15. package/dist/interaction.js +2 -2
  16. package/dist/session-ui.js +3 -3
  17. package/dist/session.js +1 -1
  18. package/dist/timeline/types.d.ts +2 -0
  19. package/package.json +5 -5
  20. package/src/components/browser-viewer.tsx +146 -57
  21. package/src/components/composer-transcription-control.tsx +4 -2
  22. package/src/components/composer.tsx +45 -20
  23. package/src/components/computer-viewer.tsx +161 -57
  24. package/src/hooks/use-browser-frame-stream.ts +3 -1
  25. package/src/hooks/use-voice-input.ts +12 -1
  26. package/src/timeline/compaction-copy.ts +2 -0
  27. package/src/timeline/projection.ts +31 -5
  28. package/src/timeline/types.ts +2 -0
  29. package/dist/chunk-2O2HXRGP.js.map +0 -1
  30. package/dist/chunk-4ACTWOOW.js.map +0 -1
  31. package/dist/chunk-CP6IFD2Z.js.map +0 -1
  32. package/dist/chunk-VI7HELBI.js.map +0 -1
  33. /package/dist/{browser-viewer-TZ6T76E3.js.map → browser-viewer-GEYFZE2W.js.map} +0 -0
  34. /package/dist/{computer-viewer-FX4WOZMX.js.map → computer-viewer-BNHZ4PYO.js.map} +0 -0
@@ -48,6 +48,7 @@ import {
48
48
  import {
49
49
  useCallback as useCallback4,
50
50
  useEffect as useEffect4,
51
+ useLayoutEffect,
51
52
  useMemo as useMemo2,
52
53
  useRef as useRef4,
53
54
  useState as useState4
@@ -1080,7 +1081,7 @@ function ComputerViewer({
1080
1081
  },
1081
1082
  ...webSocketFactory ? { webSocketFactory } : {}
1082
1083
  });
1083
- const displayedFrame = frames.frame && frames.frame.computerSessionId === selection?.sessionId && frames.frame.targetId === computer.selectedTarget?.id ? frames.frame : null;
1084
+ const displayedFrame = frames.frame && frames.frame.computerSessionId === selection?.sessionId && frames.frame.targetId === computer.selectedTarget?.id && frames.frame.targetGeneration === computer.selectedTarget?.targetGeneration ? frames.frame : null;
1084
1085
  const rfbStream = frames.attachment?.stream.kind === "direct_rfb" ? frames.attachment.stream : null;
1085
1086
  const rfbCapability = useMemo2(() => {
1086
1087
  if (!rfbStream || !frames.attachment) return null;
@@ -1190,11 +1191,15 @@ function ComputerViewer({
1190
1191
  ]);
1191
1192
  const perform = useCallback4(
1192
1193
  async (action, frame) => {
1194
+ let receipt;
1193
1195
  if (action.type === "pointer") {
1194
1196
  if (!frame) throw new Error("Desktop view is not ready for pointer input.");
1195
- await actFromFrame(action, frame);
1197
+ receipt = await actFromFrame(action, frame);
1196
1198
  } else {
1197
- await act(action);
1199
+ receipt = await act(action);
1200
+ }
1201
+ if (receipt.state !== "completed") {
1202
+ throw new Error(receipt.error?.message ?? "Desktop input did not complete.");
1198
1203
  }
1199
1204
  },
1200
1205
  [act, actFromFrame]
@@ -1355,7 +1360,13 @@ function ComputerViewer({
1355
1360
  onReadClipboard: computer.readClipboard,
1356
1361
  onReconnect: generationLossSelected || generationLossFrames ? () => void 0 : frames.reconnect,
1357
1362
  onError: (cause) => notifyError(cause, "Desktop input failed.")
1358
- }
1363
+ },
1364
+ JSON.stringify([
1365
+ selection?.sessionId,
1366
+ computer.selectedTarget?.id,
1367
+ computer.selectedTarget?.targetGeneration,
1368
+ frames.attachment?.controllerGeneration
1369
+ ])
1359
1370
  ),
1360
1371
  showControls ? /* @__PURE__ */ jsx(
1361
1372
  ComputerSemanticPanel,
@@ -1608,7 +1619,11 @@ function ComputerViewport(props) {
1608
1619
  const readClipboardRef = useRef4(props.onReadClipboard);
1609
1620
  const errorRef = useRef4(props.onError);
1610
1621
  const actionTailRef = useRef4(Promise.resolve());
1622
+ const actionQueueEpochRef = useRef4(0);
1611
1623
  const queuedFrameRef = useRef4(null);
1624
+ const currentFrameRef = useRef4(props.frame);
1625
+ const paintedFrameRef = useRef4(null);
1626
+ const [paintedFrame, setPaintedFrame] = useState4(null);
1612
1627
  const decodingFrameRef = useRef4(false);
1613
1628
  const mountedRef = useRef4(true);
1614
1629
  actionRef.current = props.onAction;
@@ -1616,6 +1631,17 @@ function ComputerViewport(props) {
1616
1631
  errorRef.current = props.onError;
1617
1632
  const streamFailed = props.connectionState === "error";
1618
1633
  const rawInputEnabled = !streamFailed && !props.machineLocked && (!props.backgroundActions || props.target?.kind === "screen" || props.target?.focused === true);
1634
+ const clearBufferedInput = useCallback4(() => {
1635
+ if (clickTimerRef.current) clearTimeout(clickTimerRef.current);
1636
+ if (wheelRef.current?.timer) clearTimeout(wheelRef.current.timer);
1637
+ if (pendingTextRef.current?.timer) clearTimeout(pendingTextRef.current.timer);
1638
+ clickTimerRef.current = null;
1639
+ wheelRef.current = null;
1640
+ pendingTextRef.current = null;
1641
+ pointerStartRef.current = null;
1642
+ lastClickRef.current = null;
1643
+ if (inputRef.current) inputRef.current.value = "";
1644
+ }, []);
1619
1645
  const paintQueuedFrames = useCallback4(() => {
1620
1646
  if (decodingFrameRef.current) return;
1621
1647
  decodingFrameRef.current = true;
@@ -1632,8 +1658,10 @@ function ComputerViewport(props) {
1632
1658
  const bitmap = await createImageBitmap(blob);
1633
1659
  try {
1634
1660
  const canvas2 = canvasRef.current;
1635
- if (mountedRef.current && canvas2) {
1661
+ if (mountedRef.current && canvas2 && currentFrameRef.current && sameComputerTarget(frame, currentFrameRef.current)) {
1636
1662
  paintCanvas(canvas2, bitmap, frame.width, frame.height);
1663
+ paintedFrameRef.current = frame;
1664
+ setPaintedFrame(frame);
1637
1665
  }
1638
1666
  } finally {
1639
1667
  bitmap.close();
@@ -1643,11 +1671,14 @@ function ComputerViewport(props) {
1643
1671
  objectUrl = URL.createObjectURL(blob);
1644
1672
  const image = await loadImage(objectUrl);
1645
1673
  const canvas = canvasRef.current;
1646
- if (mountedRef.current && canvas) {
1674
+ if (mountedRef.current && canvas && currentFrameRef.current && sameComputerTarget(frame, currentFrameRef.current)) {
1647
1675
  paintCanvas(canvas, image, frame.width, frame.height);
1676
+ paintedFrameRef.current = frame;
1677
+ setPaintedFrame(frame);
1648
1678
  }
1649
1679
  } catch (cause) {
1650
- if (mountedRef.current) errorRef.current(cause);
1680
+ if (mountedRef.current && currentFrameRef.current && sameComputerTarget(frame, currentFrameRef.current))
1681
+ errorRef.current(cause);
1651
1682
  } finally {
1652
1683
  if (objectUrl) URL.revokeObjectURL(objectUrl);
1653
1684
  }
@@ -1658,34 +1689,54 @@ function ComputerViewport(props) {
1658
1689
  }
1659
1690
  })();
1660
1691
  }, []);
1661
- useEffect4(() => {
1692
+ useLayoutEffect(() => {
1662
1693
  mountedRef.current = true;
1663
1694
  return () => {
1664
1695
  mountedRef.current = false;
1696
+ actionQueueEpochRef.current += 1;
1665
1697
  queuedFrameRef.current = null;
1698
+ clearBufferedInput();
1666
1699
  };
1667
- }, []);
1668
- useEffect4(() => {
1669
- if (!props.frame) return;
1700
+ }, [clearBufferedInput]);
1701
+ useLayoutEffect(() => {
1702
+ if (currentFrameRef.current && !props.frame) {
1703
+ actionQueueEpochRef.current += 1;
1704
+ clearBufferedInput();
1705
+ }
1706
+ currentFrameRef.current = props.frame;
1707
+ if (!props.frame) {
1708
+ queuedFrameRef.current = null;
1709
+ paintedFrameRef.current = null;
1710
+ setPaintedFrame(null);
1711
+ return;
1712
+ }
1670
1713
  queuedFrameRef.current = props.frame;
1671
1714
  paintQueuedFrames();
1672
- }, [paintQueuedFrames, props.frame]);
1673
- useEffect4(
1674
- () => () => {
1675
- if (clickTimerRef.current) clearTimeout(clickTimerRef.current);
1676
- if (wheelRef.current?.timer) clearTimeout(wheelRef.current.timer);
1677
- if (pendingTextRef.current?.timer) clearTimeout(pendingTextRef.current.timer);
1678
- },
1679
- []
1680
- );
1715
+ }, [clearBufferedInput, paintQueuedFrames, props.frame]);
1716
+ useLayoutEffect(() => {
1717
+ if (!rawInputEnabled) {
1718
+ actionQueueEpochRef.current += 1;
1719
+ clearBufferedInput();
1720
+ }
1721
+ }, [clearBufferedInput, rawInputEnabled]);
1681
1722
  const enqueue = useCallback4(
1682
1723
  (action, frame, after) => {
1724
+ const epoch = actionQueueEpochRef.current;
1725
+ const dispatch = actionRef.current;
1726
+ const isCurrent = () => mountedRef.current && epoch === actionQueueEpochRef.current;
1683
1727
  actionTailRef.current = actionTailRef.current.catch(() => void 0).then(async () => {
1684
- await actionRef.current(action, frame);
1685
- await after?.();
1686
- }).catch((cause) => errorRef.current(cause));
1728
+ if (!isCurrent()) return;
1729
+ await dispatch(action, frame);
1730
+ if (!isCurrent()) return;
1731
+ await after?.(isCurrent);
1732
+ }).catch((cause) => {
1733
+ if (!isCurrent()) return;
1734
+ actionQueueEpochRef.current += 1;
1735
+ clearBufferedInput();
1736
+ errorRef.current(cause);
1737
+ });
1687
1738
  },
1688
- []
1739
+ [clearBufferedInput]
1689
1740
  );
1690
1741
  const point = useCallback4(
1691
1742
  (frame, clientX, clientY) => computerPoint(canvasRef.current, frame, clientX, clientY),
@@ -1715,14 +1766,35 @@ function ComputerViewport(props) {
1715
1766
  pending.frame
1716
1767
  );
1717
1768
  }, [enqueue]);
1769
+ const flushPendingWheel = useCallback4(() => {
1770
+ const batch = wheelRef.current;
1771
+ if (!batch) return;
1772
+ if (batch.timer) clearTimeout(batch.timer);
1773
+ wheelRef.current = null;
1774
+ enqueue(
1775
+ {
1776
+ type: "pointer",
1777
+ frameId: batch.frame.frameId,
1778
+ action: "scroll",
1779
+ x: batch.x,
1780
+ y: batch.y,
1781
+ deltaX: batch.deltaX,
1782
+ deltaY: batch.deltaY
1783
+ },
1784
+ batch.frame
1785
+ );
1786
+ }, [enqueue]);
1718
1787
  const pointerDown = (event) => {
1719
- if (!props.frame || props.mutating || !rawInputEnabled || event.button !== 0) return;
1788
+ const frame = paintedFrameRef.current;
1789
+ if (!frame || props.mutating || !rawInputEnabled || event.button !== 0) return;
1790
+ event.preventDefault();
1791
+ flushPendingWheel();
1720
1792
  flushPendingText();
1721
1793
  pointerStartRef.current = {
1722
1794
  x: event.clientX,
1723
1795
  y: event.clientY,
1724
1796
  pointerId: event.pointerId,
1725
- frame: props.frame
1797
+ frame
1726
1798
  };
1727
1799
  event.currentTarget.setPointerCapture?.(event.pointerId);
1728
1800
  inputRef.current?.focus({ preventScroll: true });
@@ -1788,8 +1860,9 @@ function ComputerViewport(props) {
1788
1860
  const contextMenu = (event) => {
1789
1861
  event.preventDefault();
1790
1862
  if (!rawInputEnabled) return;
1791
- const frame = props.frame;
1863
+ const frame = paintedFrameRef.current;
1792
1864
  if (!frame) return;
1865
+ flushPendingWheel();
1793
1866
  flushPendingText();
1794
1867
  flushPendingClick();
1795
1868
  const at = point(frame, event.clientX, event.clientY);
@@ -1809,43 +1882,30 @@ function ComputerViewport(props) {
1809
1882
  };
1810
1883
  const wheel = (event) => {
1811
1884
  if (!rawInputEnabled) return;
1812
- const frame = props.frame;
1885
+ const frame = paintedFrameRef.current;
1813
1886
  if (!frame) return;
1814
1887
  const at = point(frame, event.clientX, event.clientY);
1815
1888
  if (!at) return;
1816
1889
  flushPendingText();
1817
1890
  flushPendingClick();
1818
1891
  event.preventDefault();
1819
- const pending = wheelRef.current;
1820
- if (pending?.timer) clearTimeout(pending.timer);
1892
+ let pending = wheelRef.current;
1893
+ if (pending && (!sameComputerTarget(pending.frame, frame) || Math.hypot(pending.x - at.x, pending.y - at.y) > 6)) {
1894
+ flushPendingWheel();
1895
+ pending = null;
1896
+ }
1821
1897
  wheelRef.current = {
1822
1898
  x: at.x,
1823
1899
  y: at.y,
1824
- deltaX: (pending && sameFrameFence(pending.frame, frame) ? pending.deltaX : 0) + event.deltaX,
1825
- deltaY: (pending && sameFrameFence(pending.frame, frame) ? pending.deltaY : 0) + event.deltaY,
1900
+ deltaX: (pending?.deltaX ?? 0) + event.deltaX,
1901
+ deltaY: (pending?.deltaY ?? 0) + event.deltaY,
1826
1902
  frame,
1827
- timer: setTimeout(() => {
1828
- const batch = wheelRef.current;
1829
- wheelRef.current = null;
1830
- if (batch) {
1831
- enqueue(
1832
- {
1833
- type: "pointer",
1834
- frameId: batch.frame.frameId,
1835
- action: "scroll",
1836
- x: batch.x,
1837
- y: batch.y,
1838
- deltaX: batch.deltaX,
1839
- deltaY: batch.deltaY
1840
- },
1841
- batch.frame
1842
- );
1843
- }
1844
- }, 45)
1903
+ timer: pending?.timer ?? setTimeout(flushPendingWheel, 45)
1845
1904
  };
1846
1905
  };
1847
1906
  const keyDown = (event) => {
1848
1907
  if (!rawInputEnabled) return;
1908
+ flushPendingWheel();
1849
1909
  const command = event.metaKey || event.ctrlKey;
1850
1910
  if (props.clipboardEnabled && command && !event.altKey && ["c", "v"].includes(event.key.toLowerCase())) {
1851
1911
  flushPendingText();
@@ -1861,10 +1921,13 @@ function ComputerViewport(props) {
1861
1921
  const copy = (event) => {
1862
1922
  if (!props.clipboardEnabled || !rawInputEnabled) return;
1863
1923
  event.preventDefault();
1924
+ flushPendingWheel();
1864
1925
  flushPendingText();
1865
1926
  flushPendingClick();
1866
- enqueue({ type: "clipboard", operation: "copy" }, null, async () => {
1867
- const clipboard = await readClipboardRef.current();
1927
+ const readClipboard = readClipboardRef.current;
1928
+ enqueue({ type: "clipboard", operation: "copy" }, null, async (isCurrent) => {
1929
+ const clipboard = await readClipboard();
1930
+ if (!isCurrent()) return;
1868
1931
  if (!clipboard.text) return;
1869
1932
  if (!await copyTextToClipboard(clipboard.text)) {
1870
1933
  throw new Error("Desktop text could not be copied to the local clipboard");
@@ -1874,16 +1937,22 @@ function ComputerViewport(props) {
1874
1937
  const paste = (event) => {
1875
1938
  if (!props.clipboardEnabled || !rawInputEnabled) return;
1876
1939
  event.preventDefault();
1940
+ flushPendingWheel();
1877
1941
  flushPendingText();
1878
1942
  flushPendingClick();
1879
1943
  const text = event.clipboardData.getData("text/plain");
1880
- enqueue({ type: "clipboard", operation: "write", text }, null, async () => {
1881
- assertExactComputerClipboard(await readClipboardRef.current(), text);
1882
- await actionRef.current({ type: "clipboard", operation: "paste" }, null);
1944
+ const readClipboard = readClipboardRef.current;
1945
+ const dispatch = actionRef.current;
1946
+ enqueue({ type: "clipboard", operation: "write", text }, null, async (isCurrent) => {
1947
+ const clipboard = await readClipboard();
1948
+ if (!isCurrent()) return;
1949
+ assertExactComputerClipboard(clipboard, text);
1950
+ await dispatch({ type: "clipboard", operation: "paste" }, null);
1883
1951
  });
1884
1952
  };
1885
1953
  const input = (value) => {
1886
1954
  if (!value || !rawInputEnabled) return;
1955
+ flushPendingWheel();
1887
1956
  flushPendingClick();
1888
1957
  const pending = pendingTextRef.current;
1889
1958
  if (pending) {
@@ -1898,7 +1967,7 @@ function ComputerViewport(props) {
1898
1967
  }
1899
1968
  if (inputRef.current) inputRef.current.value = "";
1900
1969
  };
1901
- const showCanvas = props.frame !== null && !props.machineLocked && !streamFailed;
1970
+ const showCanvas = props.frame !== null && paintedFrame !== null && sameComputerTarget(props.frame, paintedFrame) && !props.machineLocked && !streamFailed;
1902
1971
  return /* @__PURE__ */ jsxs("div", { className: "relative min-h-0 flex-1 overflow-hidden bg-black", children: [
1903
1972
  /* @__PURE__ */ jsx(
1904
1973
  "canvas",
@@ -2223,7 +2292,10 @@ function computerKey(event) {
2223
2292
  return parts.join("+");
2224
2293
  }
2225
2294
  function sameFrameFence(left, right) {
2226
- return left.frameId === right.frameId && left.computerSessionId === right.computerSessionId && left.targetId === right.targetId && left.targetGeneration === right.targetGeneration;
2295
+ return left.frameId === right.frameId && sameComputerTarget(left, right);
2296
+ }
2297
+ function sameComputerTarget(left, right) {
2298
+ return left.computerSessionId === right.computerSessionId && left.controllerGeneration === right.controllerGeneration && left.targetId === right.targetId && left.targetGeneration === right.targetGeneration;
2227
2299
  }
2228
2300
  function isLiveComputer2(session) {
2229
2301
  return !["ending", "ended", "failed", "lost"].includes(session.lifecycle);
@@ -2331,4 +2403,4 @@ export {
2331
2403
  ComputerViewer,
2332
2404
  computerKey
2333
2405
  };
2334
- //# sourceMappingURL=chunk-2O2HXRGP.js.map
2406
+ //# sourceMappingURL=chunk-Y6DMQTLA.js.map