@opengeni/react 0.5.0 → 0.6.0

package/src/hooks/use-relay-frame-stream.ts

@@ -0,0 +1,335 @@
+import {
+  decodeStreamFrame,
+  decodeStreamOpenAck,
+  encodeStreamOpen,
+  STREAM_KIND_DESKTOP,
+  STREAM_ROLE_CLIENT,
+} from "../lib/relay-wire";
+import type { DesktopConnectionState, DesktopStreamCapability } from "@opengeni/sdk";
+import { type RefObject, useEffect, useRef, useState } from "react";
+
+/**
+ * The minimal WebSocket surface the frame renderer drives. Lets tests (and a
+ * future transport swap) inject a fake without a live socket — the exact
+ * analogue of `DesktopRfbFactory` for the noVNC path. Structurally a subset of
+ * the browser `WebSocket`.
+ */
+export interface DesktopWebSocketLike {
+  binaryType: string;
+  send(data: ArrayBuffer): void;
+  close(): void;
+  addEventListener(
+    type: "open" | "message" | "error" | "close",
+    listener: (ev: { data?: unknown } & Record<string, unknown>) => void,
+  ): void;
+  removeEventListener(
+    type: "open" | "message" | "error" | "close",
+    listener: (ev: { data?: unknown } & Record<string, unknown>) => void,
+  ): void;
+}
+
+export type DesktopWebSocketFactory = (url: string) => DesktopWebSocketLike;
+
+export type UseRelayFrameStreamOptions = {
+  /** The desktop cell of the negotiated capabilities (`capabilities.DesktopStream`). */
+  capability: DesktopStreamCapability | null;
+  /** The mount target. A `<canvas>` is appended here on connect. */
+  containerRef: RefObject<HTMLDivElement | null>;
+  /** Custom socket factory (tests / a transport swap). Defaults to `new WebSocket(url)`. */
+  webSocketFactory?: DesktopWebSocketFactory | undefined;
+};
+
+export type UseRelayFrameStreamResult = {
+  state: DesktopConnectionState;
+  error: Error | null;
+  /** Tear down + reopen the socket (e.g. after a drop, once a fresh url arrives). */
+  reconnect: () => void;
+};
+
+// Relay datagram tags: byte[0] of every binary message. Mirrors the proven wire
+// protocol in `apps/api/scripts/diagnose-mac-desktop-stream.ts`.
+const TAG_OPEN = 1;
+const TAG_OPENACK = 2;
+const TAG_FRAME = 3;
+
+/** Build a relay datagram: a fresh Uint8Array of `body.length + 1` with the tag
+ *  at [0] and the protobuf body copied at offset 1 (so `.buffer` is exact). */
+function datagram(tag: number, body: Uint8Array): Uint8Array {
+  const out = new Uint8Array(body.length + 1);
+  out[0] = tag;
+  out.set(body, 1);
+  return out;
+}
+
+function defaultWebSocketFactory(url: string): DesktopWebSocketLike {
+  // The browser `WebSocket` is structurally a superset of `DesktopWebSocketLike`
+  // (its event maps are stricter); cast so the seam stays loosely typed for fakes.
+  return new WebSocket(url) as unknown as DesktopWebSocketLike;
+}
+
+/**
+ * VIEW-ONLY PNG-frame renderer for a SELF-HOSTED desktop stream — the relay
+ * transport (`transport: "relay-frames"`, `client: "frames"`). The self-hosted
+ * agent produces one PNG per frame as a protobuf datagram over the relay; there
+ * is no RFB/noVNC here. This hook opens the relay DESKTOP channel as a CLIENT
+ * (mirroring the proven diagnostic wire protocol), decodes each PNG onto a
+ * `<canvas>` mounted into `containerRef`, and drives the connection state
+ * machine: idle → connecting (ws opening) → connected (first painted frame) →
+ * error (ws error/close/ack rejection).
+ *
+ * It is DORMANT for any other transport (or no url): it stays idle and touches
+ * nothing, so the noVNC path owns the surface. SSR-safe: the socket + DOM attach
+ * live inside `useEffect`.
+ *
+ * Backpressure (critical — frames are ~1.8MB at ~10fps): at most one frame is
+ * decoded at a time; a frame arriving mid-decode replaces the pending one
+ * (latest-wins), so a slow decode can never build an unbounded queue.
+ */
+export function useRelayFrameStream(options: UseRelayFrameStreamOptions): UseRelayFrameStreamResult {
+  const { capability, containerRef, webSocketFactory } = options;
+  const [state, setState] = useState<DesktopConnectionState>("idle");
+  const [error, setError] = useState<Error | null>(null);
+  const [nonce, setNonce] = useState(0);
+  const stateRef = useRef<DesktopConnectionState>("idle");
+  const setBoth = (next: DesktopConnectionState) => {
+    stateRef.current = next;
+    setState(next);
+  };
+
+  const reconnect = () => setNonce((n) => n + 1);
+
+  const transport = capability?.transport ?? null;
+  const url = capability?.url ?? null;
+  const token = capability?.token ?? null;
+
+  // The factory is read via a ref so swapping it never re-opens the socket (it is
+  // not a connect-effect dependency), symmetric with the noVNC `rfbFactory`.
+  const factoryRef = useRef(webSocketFactory);
+  factoryRef.current = webSocketFactory;
+
+  useEffect(() => {
+    // SSR / no DOM: stay idle and show the placeholder.
+    if (typeof window === "undefined") return;
+    // This hook OWNS the surface ONLY for the frame transport; anything else (or
+    // no live url) → dormant idle so the noVNC path can drive.
+    if (transport !== "relay-frames" || !url) {
+      setBoth("idle");
+      return;
+    }
+    const container = containerRef.current;
+    if (!container) {
+      // Mount target not in the DOM yet (tab hidden) — stay idle so a re-attach
+      // nudge can re-run this effect once the container exists.
+      setBoth("idle");
+      return;
+    }
+
+    // Parse the 4 relay query params the channel is keyed on.
+    let channel: {
+      channelId: string;
+      workspaceId: string;
+      agentId: string;
+      kind: number;
+      port: number;
+    };
+    try {
+      const u = new URL(url);
+      channel = {
+        channelId: u.searchParams.get("channel") ?? "",
+        workspaceId: u.searchParams.get("ws") ?? "",
+        agentId: u.searchParams.get("agent") ?? "",
+        kind: STREAM_KIND_DESKTOP,
+        port: Number(u.searchParams.get("port") ?? "0"),
+      };
+    } catch (cause) {
+      setError(cause instanceof Error ? cause : new Error(String(cause)));
+      setBoth("error");
+      return;
+    }
+
+    setError(null);
+    setBoth("connecting");
+
+    // Canvas mount — mirror the noVNC surface conventions: absolute-fill the
+    // panel and CONTAIN the framebuffer (aspect-preserved, centered, letterboxed)
+    // — never distorted, never overflowing. `max-*: 100%` + `margin: auto` +
+    // intrinsic size gives fit-to-panel without a resize observer.
+    const canvas = document.createElement("canvas");
+    canvas.setAttribute("data-opengeni-desktop-frames", "");
+    const style = canvas.style;
+    style.position = "absolute";
+    style.top = "0";
+    style.left = "0";
+    style.right = "0";
+    style.bottom = "0";
+    style.margin = "auto";
+    style.maxWidth = "100%";
+    style.maxHeight = "100%";
+    style.width = "auto";
+    style.height = "auto";
+    style.display = "block";
+    style.imageRendering = "auto";
+    container.appendChild(canvas);
+    const ctx = canvas.getContext("2d");
+
+    let disposed = false;
+    let acked = false;
+    // Backpressure: decode at most ONE frame at a time; a frame arriving mid-
+    // decode replaces `pending` (keep only the LATEST) — never a queue.
+    let decoding = false;
+    let pending: Uint8Array | null = null;
+
+    const drainLatest = async () => {
+      if (decoding) return;
+      decoding = true;
+      try {
+        while (!disposed && pending) {
+          const data = pending;
+          pending = null;
+          let bmp: ImageBitmap;
+          try {
+            // Copy the exact frame bytes into a fresh ArrayBuffer-backed view:
+            // ts-proto's `bytes()` hands back a subarray VIEW into the larger
+            // message buffer, so slicing isolates just this PNG (and satisfies the
+            // `BlobPart` typing, which rejects the generic `ArrayBufferLike`).
+            bmp = await createImageBitmap(new Blob([new Uint8Array(data)], { type: "image/png" }));
+          } catch {
+            // A corrupt/partial frame is skipped; latest-wins keeps us live.
+            continue;
+          }
+          if (disposed) {
+            bmp.close();
+            break;
+          }
+          // Size the backing store to the frame's natural WxH on the first frame
+          // (or when the resolution changes); CSS then scales the element to fit.
+          if (canvas.width !== bmp.width || canvas.height !== bmp.height) {
+            canvas.width = bmp.width;
+            canvas.height = bmp.height;
+          }
+          ctx?.drawImage(bmp, 0, 0, canvas.width, canvas.height);
+          bmp.close();
+          // First painted frame → connected.
+          if (!disposed && stateRef.current !== "connected") setBoth("connected");
+        }
+      } finally {
+        decoding = false;
+      }
+    };
+
+    let ws: DesktopWebSocketLike;
+    try {
+      ws = (factoryRef.current ?? defaultWebSocketFactory)(url);
+    } catch (cause) {
+      setError(cause instanceof Error ? cause : new Error(String(cause)));
+      setBoth("error");
+      canvas.remove();
+      return;
+    }
+    ws.binaryType = "arraybuffer";
+
+    const onOpen = () => {
+      if (disposed) return;
+      const body = encodeStreamOpen({
+        channel,
+        token: token ?? "",
+        role: STREAM_ROLE_CLIENT,
+        resumeFromSeq: "0",
+      });
+      try {
+        ws.send(datagram(TAG_OPEN, body).buffer as ArrayBuffer);
+      } catch {
+        // Socket already closing; the close/error handler surfaces it.
+      }
+    };
+
+    const onMessage = (ev: { data?: unknown }) => {
+      if (disposed) return;
+      const data = ev.data;
+      if (!(data instanceof ArrayBuffer)) return;
+      const buf = new Uint8Array(data);
+      if (buf.length === 0) return;
+      const tag = buf[0];
+      const rest = buf.subarray(1);
+      if (tag === TAG_OPENACK) {
+        let ack: ReturnType<typeof decodeStreamOpenAck>;
+        try {
+          ack = decodeStreamOpenAck(rest);
+        } catch {
+          return;
+        }
+        if (!ack.accepted) {
+          setError(
+            new Error(
+              ack.error?.message
+                ? `desktop stream rejected: ${ack.error.message}`
+                : "desktop stream rejected by the relay",
+            ),
+          );
+          setBoth("error");
+          try {
+            ws.close();
+          } catch {
+            // already closed
+          }
+          return;
+        }
+        // Accepted — stay "connecting" until the first frame paints.
+        acked = true;
+      } else if (tag === TAG_FRAME) {
+        let fr: ReturnType<typeof decodeStreamFrame>;
+        try {
+          fr = decodeStreamFrame(rest);
+        } catch {
+          return;
+        }
+        if (fr.data && fr.data.length > 0) {
+          pending = fr.data; // latest-wins backpressure
+          void drainLatest();
+        }
+      }
+    };
+
+    const onError = () => {
+      if (disposed) return;
+      setError((prev) => prev ?? new Error("desktop stream connection error"));
+      setBoth("error");
+    };
+
+    const onClose = () => {
+      if (disposed) return;
+      setError(
+        (prev) =>
+          prev ??
+          new Error(acked ? "desktop stream closed" : "desktop stream closed before it opened"),
+      );
+      setBoth("error");
+    };
+
+    ws.addEventListener("open", onOpen);
+    ws.addEventListener("message", onMessage);
+    ws.addEventListener("error", onError);
+    ws.addEventListener("close", onClose);
+
+    return () => {
+      disposed = true;
+      pending = null;
+      ws.removeEventListener("open", onOpen);
+      ws.removeEventListener("message", onMessage);
+      ws.removeEventListener("error", onError);
+      ws.removeEventListener("close", onClose);
+      try {
+        ws.close();
+      } catch {
+        // ignore teardown errors
+      }
+      canvas.remove();
+    };
+    // ONLY a real transport change reopens: a fresh url (rotation), a new token,
+    // the transport flipping, or a manual reconnect (`nonce`). The factory is read
+    // via a ref and must never re-open the socket.
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [url, token, transport, nonce]);
+
+  return { state, error, reconnect };
+}

package/src/index.ts

@@ -55,6 +55,13 @@ export type {
 } from "./hooks/use-session-capabilities";
 export { useDesktopStream } from "./hooks/use-desktop-stream";
 export type { UseDesktopStreamOptions, UseDesktopStreamResult } from "./hooks/use-desktop-stream";
+export { useRelayFrameStream } from "./hooks/use-relay-frame-stream";
+export type {
+  DesktopWebSocketFactory,
+  DesktopWebSocketLike,
+  UseRelayFrameStreamOptions,
+  UseRelayFrameStreamResult,
+} from "./hooks/use-relay-frame-stream";
 export { useTerminalStream } from "./hooks/use-terminal-stream";
 export type {
   TerminalStreamStatus,
@@ -99,6 +106,7 @@ export type {
   SessionStatusItem,
   TimelineGroup,
   TimelineItem,
+  TurnEndItem,
   ToolCallItem,
   UserMessageItem,
   WorkerItem,
@@ -221,6 +229,13 @@ export type { DesktopViewerProps } from "./components/desktop-viewer";
 export { WorkspaceDock } from "./components/workspace-dock";
 export type { WorkspaceDockProps, WorkspaceTab } from "./components/workspace-dock";
 
+// Connected-machine UI moved to the "@opengeni/react/machines" subpath; re-exported
+// here for back-compat (#144).
+export * from "./machines";
+// Multi-account Codex (P1): accounts list + active-switch hook.
+export { useCodexAccounts, isCodexAccountEvent } from "./hooks/use-codex-accounts";
+export type { CodexAccountsClientLike, UseCodexAccountsOptions, UseCodexAccountsResult } from "./hooks/use-codex-accounts";
+
 // Sandbox helpers
 export { gitFileDiffToPatch } from "./lib/git-patch";
 export { xtermThemeFromTokens } from "./lib/xterm-theme";

package/src/lib/relay-wire.ts

@@ -0,0 +1,116 @@
+// Minimal HAND-MIRROR of the relay stream wire messages the desktop frame client
+// needs (`StreamOpen` encode; `StreamOpenAck` + `StreamFrame` decode). It exists
+// because the publish-closure guard (scripts/publish-closure-guard.ts) forbids
+// `@opengeni/react` from depending on `@opengeni/agent-proto` — the published SDK
+// closure may only reach `@opengeni/sdk` among `@opengeni/*` packages. So, exactly
+// as the guard prescribes ("hand-mirror it"), we re-encode just these three
+// messages against the third-party `@bufbuild/protobuf/wire` runtime (the same
+// runtime `@opengeni/agent-proto` generates against), byte-compatible with the
+// generated code. Field numbers copied verbatim from
+// `packages/agent-proto/src/gen/opengeni_agent.ts`:
+//   StreamChannel: channelId=1, workspaceId=2, agentId=3, kind=4(int32), port=5(uint32)
+//   StreamOpen:    channel=1(msg), token=2, role=3(int32), resumeFromSeq=4(uint64, omitted when "0")
+//   StreamOpenAck: accepted=1(bool), error=2(AgentError msg), resumeFromSeq=3
+//   AgentError:    code=1, message=2, retryable=3, detail=4  (we read only message)
+//   StreamFrame:   channelId=1, seq=2, data=3(bytes), producedAtMs=4  (we read only data)
+import { BinaryReader, BinaryWriter } from "@bufbuild/protobuf/wire";
+
+/** `StreamKind.STREAM_KIND_DESKTOP` (agent-proto enum value). */
+export const STREAM_KIND_DESKTOP = 2;
+/** `StreamRole.STREAM_ROLE_CLIENT` (agent-proto enum value). */
+export const STREAM_ROLE_CLIENT = 2;
+
+export interface RelayChannel {
+  channelId: string;
+  workspaceId: string;
+  agentId: string;
+  kind: number;
+  port: number;
+}
+
+export interface RelayStreamOpen {
+  channel: RelayChannel;
+  token: string;
+  role: number;
+  resumeFromSeq: string;
+}
+
+function encodeChannel(message: RelayChannel, writer: BinaryWriter): BinaryWriter {
+  if (message.channelId !== "") writer.uint32(10).string(message.channelId);
+  if (message.workspaceId !== "") writer.uint32(18).string(message.workspaceId);
+  if (message.agentId !== "") writer.uint32(26).string(message.agentId);
+  if (message.kind !== 0) writer.uint32(32).int32(message.kind);
+  if (message.port !== 0) writer.uint32(40).uint32(message.port);
+  return writer;
+}
+
+/** Encode a `StreamOpen`, byte-compatible with `StreamOpen.encode(...).finish()`. */
+export function encodeStreamOpen(message: RelayStreamOpen): Uint8Array {
+  const writer = new BinaryWriter();
+  encodeChannel(message.channel, writer.uint32(10).fork()).join();
+  if (message.token !== "") writer.uint32(18).string(message.token);
+  if (message.role !== 0) writer.uint32(24).int32(message.role);
+  if (message.resumeFromSeq !== "0") writer.uint32(32).uint64(message.resumeFromSeq);
+  return writer.finish();
+}
+
+/** The only `AgentError` field we surface from a rejected ack. */
+function decodeAgentErrorMessage(reader: BinaryReader, length: number): { message?: string } {
+  const end = reader.pos + length;
+  let message: string | undefined;
+  while (reader.pos < end) {
+    const tag = reader.uint32();
+    if (tag >>> 3 === 2 && tag === 18) {
+      message = reader.string();
+      continue;
+    }
+    if ((tag & 7) === 4 || tag === 0) break;
+    reader.skip(tag & 7);
+  }
+  return message !== undefined ? { message } : {};
+}
+
+/** Decode a `StreamOpenAck` (we need `accepted` + the error message). */
+export function decodeStreamOpenAck(bytes: Uint8Array): {
+  accepted: boolean;
+  error?: { message?: string };
+} {
+  const reader = new BinaryReader(bytes);
+  const end = reader.len;
+  let accepted = false;
+  let error: { message?: string } | undefined;
+  while (reader.pos < end) {
+    const tag = reader.uint32();
+    if (tag >>> 3 === 1 && tag === 8) {
+      accepted = reader.bool();
+      continue;
+    }
+    if (tag >>> 3 === 2 && tag === 18) {
+      error = decodeAgentErrorMessage(reader, reader.uint32());
+      continue;
+    }
+    if ((tag & 7) === 4 || tag === 0) break;
+    reader.skip(tag & 7);
+  }
+  return error !== undefined ? { accepted, error } : { accepted };
+}
+
+/** Decode a `StreamFrame`, extracting only the `data` (framebuffer PNG) bytes. */
+export function decodeStreamFrame(bytes: Uint8Array): { data: Uint8Array } {
+  const reader = new BinaryReader(bytes);
+  const end = reader.len;
+  let data = new Uint8Array(0);
+  while (reader.pos < end) {
+    const tag = reader.uint32();
+    if (tag >>> 3 === 3 && tag === 26) {
+      // Copy into a fresh ArrayBuffer-backed view: BinaryReader.bytes() returns
+      // `Uint8Array<ArrayBufferLike>` (a view into the message buffer), which both
+      // narrows the type to `Uint8Array<ArrayBuffer>` and isolates just this frame.
+      data = new Uint8Array(reader.bytes());
+      continue;
+    }
+    if ((tag & 7) === 4 || tag === 0) break;
+    reader.skip(tag & 7);
+  }
+  return { data };
+}

package/src/machines.ts

@@ -0,0 +1,50 @@
+// @opengeni/react/machines — Bring-your-own-compute: Machines dashboard +
+// enrollment flow + status surfacing (M9). View-model types mirror the M10
+// contract shape (MachineView / MetricSample / MachinesResponse) — see
+// types/machines.ts.
+//
+// This is a self-contained island carved out of the package root so consumers
+// that don't surface connected machines never pull it in. The root still
+// re-exports everything here for backwards compatibility (deprecated).
+export type {
+  ConnectionStatus,
+  MachineKind,
+  MachineMetricsSeriesResponse,
+  MachineState,
+  MachineView,
+  MachinesResponse,
+  MetricSample,
+} from "./types/machines";
+export { connectionStatusForState } from "./types/machines";
+export {
+  ConnectionDot,
+  ConnectionStatusPill,
+  MachineStatusPill,
+  CONNECTION_STATUS_META,
+  MACHINE_STATE_BADGE_META,
+} from "./components/machine-status-pill";
+export type {
+  ConnectionDotProps,
+  ConnectionStatusMeta,
+  ConnectionStatusPillProps,
+  MachineStateBadgeMeta,
+  MachineStatusPillProps,
+} from "./components/machine-status-pill";
+export { MachineMetrics } from "./components/machine-metrics";
+export type { MachineMetricsProps } from "./components/machine-metrics";
+export { MachineCard } from "./components/machine-card";
+export type { MachineCardProps } from "./components/machine-card";
+export { MachinesDashboard } from "./components/machines-dashboard";
+export type { MachinesDashboardProps } from "./components/machines-dashboard";
+export { MachineDockBar, SharedMachineDisclosure } from "./components/machine-dock-bar";
+export type { MachineDockBarProps, SharedMachineDisclosureProps } from "./components/machine-dock-bar";
+export { EnrollmentDeviceFlow } from "./components/enrollment-device-flow";
+export type { DeviceFlowPhase, EnrollmentDeviceFlowProps } from "./components/enrollment-device-flow";
+export { EnrollmentConsent } from "./components/enrollment-consent";
+export type {
+  EnrollmentConsentMachine,
+  EnrollmentConsentPhase,
+  EnrollmentConsentProps,
+} from "./components/enrollment-consent";
+export { useMachines } from "./hooks/use-machines";
+export type { MachinesClientLike, UseMachinesOptions, UseMachinesResult } from "./hooks/use-machines";

package/src/timeline/activity-rail.tsx

@@ -145,13 +145,19 @@ function WorkerRow({ item, onOpenSession }: { item: WorkerItem; onOpenSession?:
           : cancelled
             ? "Worker interrupted"
             : "Worker spawned"
-      : running
-        ? "Messaging worker"
-        : failed
-          ? "Worker message failed"
-          : cancelled
-            ? "Worker interrupted"
-            : "Worker messaged";
+      : item.action === "interrupt"
+        ? (() => {
+            const verb = item.mode === "steer" ? "Steering" : "Stopping";
+            const done = item.mode === "steer" ? "Worker steered" : "Worker stopped";
+            return running ? `${verb} worker` : failed ? "Worker interrupt failed" : cancelled ? "Worker interrupted" : done;
+          })()
+        : running
+          ? "Messaging worker"
+          : failed
+            ? "Worker message failed"
+            : cancelled
+              ? "Worker interrupted"
+              : "Worker messaged";
   return (
     <div
       className={cn(

package/src/timeline/index.ts

@@ -29,6 +29,7 @@ export type {
   SessionStatusItem,
   TimelineGroup,
   TimelineItem,
+  TurnEndItem,
   ToolCallItem,
   UserMessageItem,
   WorkerItem,