@unoverse-platform/studio 0.1.0

package/vendor/sdk/lib/voice.tsx

@@ -0,0 +1,311 @@
+/**
+ * useVoiceService — the Unoverse `voice` NATIVE SERVICE (UNOVERSE_SPEC §2e-1 Tier-3).
+ *
+ * Audio I/O is inherently native (getUserMedia / AudioContext / the /ws/gravity WS
+ * lane), so it cannot be expressed as definition data — it is the spec's sanctioned
+ * escape hatch: a fixed `{ state, actions }` capability the SDK provides and a template
+ * REFERENCES (`service: "voice"`), binding its neutral state + dispatching its actions.
+ *
+ * This composes the ported realtime hooks (capture / playback / the WS audio lane) +
+ * the orchestration from the legacy useVoiceCall + useAudioContext into ONE neutral
+ * projection. The channel spreads `state` into the template's data scope and routes the
+ * voice action names to the actions — see the workbench wiring. The SDK owns NO styles
+ * and exposes only NEUTRAL model facts here (never UX flags).
+ *
+ * Transport split (docs/UNOVERSE_MCP_TEMPLATE_PROTOCOL §5): the live PCM frames AND the
+ * synchronized audio-state control events both ride the /ws/gravity WS lane (they stay
+ * frame-synchronized); only structured component data is on MCP, untouched by voice.
+ */
+
+import { useState, useCallback, useEffect, useRef, useMemo } from "react";
+import { useRealtimeWebSocket } from "./realtime/useRealtimeWebSocket";
+import { useAudioCapture } from "./realtime/useAudioCapture";
+import { useAudioPlayback } from "./realtime/useAudioPlayback";
+import type { AudioStateEvent } from "./realtime/types";
+import type { ComponentStore } from "./core";
+
+export type VoiceConnectionStatus = "idle" | "connecting" | "connected" | "ended" | "error";
+
+export interface VoiceSession {
+  userId: string;
+  conversationId: string;
+  chatId?: string;
+  workflowId: string;
+  targetTriggerNode: string;
+}
+
+export interface UseVoiceServiceConfig {
+  /** Gateway REST base, e.g. http://localhost:4100 (the workflow `…/execute` host). */
+  apiUrl: string;
+  /** Audio WS base, e.g. ws://localhost:4100. Defaults to `apiUrl` with http→ws. */
+  wsUrl?: string;
+  /** Fresh access token provider (rides the WS lane + the REST Authorization header). */
+  getAccessToken?: () => Promise<string | null> | string | null;
+  session: VoiceSession;
+  /** The shared store. When given, the service acts as a PRODUCER (STATE_MODEL §4):
+   *  every state change is merged into TEMPLATE STATE, so templates read voice exactly
+   *  like any global key (`visibleWhen: { field: "callState", … }`) with zero host
+   *  glue — the voice twin of `lifecycle`/`isThinking`/`isStreaming`. */
+  store?: ComponentStore;
+}
+
+/** The single call-phase VALUE a voice template branches its states on — the standard voice
+ *  state vocabulary (`visibleWhen: { field: "callState", eq: … }`). Derived from the neutral
+ *  facts below so every voice template keys off one field, exactly like `mode` for focus. */
+export type VoiceCallState = "idle" | "active" | "agentSpeaking" | "userSpeaking";
+
+/** NEUTRAL voice projections — flat, so the channel can spread them into the template scope. */
+export interface VoiceServiceState {
+  connectionStatus: VoiceConnectionStatus;
+  /** The single phase the template's states key off (idle | active | agentSpeaking | userSpeaking). */
+  callState: VoiceCallState;
+  isIdle: boolean;
+  isCallActive: boolean;
+  isConnecting: boolean;
+  isConnected: boolean;
+  isAssistantSpeaking: boolean;
+  isUserSpeaking: boolean;
+  /** user speaking AND assistant not — mutual exclusion (the template can't express NOT). */
+  showUserSpeaking: boolean;
+  isMuted: boolean;
+  isLookingUp: boolean;
+  lookupLabel: string;
+  callDuration: number;
+  durationText: string;
+  statusText: string;
+  error: string | null;
+}
+
+export interface VoiceService {
+  /** Spread into the template's data scope (channel overrides → rootData). */
+  state: VoiceServiceState;
+  startCall: () => Promise<void>;
+  endCall: () => Promise<void>;
+  toggleMute: () => void;
+}
+
+function formatDuration(totalSeconds: number): string {
+  const m = Math.floor(totalSeconds / 60)
+    .toString()
+    .padStart(2, "0");
+  const s = (totalSeconds % 60).toString().padStart(2, "0");
+  return `${m}:${s}`;
+}
+
+export function useVoiceService(config: UseVoiceServiceConfig): VoiceService {
+  const { apiUrl, session, getAccessToken } = config;
+  const wsUrl = config.wsUrl ?? apiUrl.replace(/^http/, "ws");
+
+  const [connectionStatus, setConnectionStatus] = useState<VoiceConnectionStatus>("idle");
+  const [isMuted, setIsMuted] = useState(false);
+  const [callDuration, setCallDuration] = useState(0);
+  const [error, setError] = useState<string | null>(null);
+  const [assistantHint, setAssistantHint] = useState(false);
+  const [userSpeaking, setUserSpeaking] = useState(false);
+  const [isLookingUp, setIsLookingUp] = useState(false);
+  const [lookupTool, setLookupTool] = useState<string | null>(null);
+
+  const timerRef = useRef<ReturnType<typeof setInterval> | null>(null);
+  const sendAudioRef = useRef<(d: ArrayBuffer) => void>(() => {});
+
+  const playback = useAudioPlayback();
+
+  // Audio-state events arrive on the WS lane, in lockstep with the frames.
+  const handleAudioState = useCallback(
+    (event: AudioStateEvent) => {
+      const { state, metadata } = event;
+      if (state === "SPEECH_STARTED") {
+        setAssistantHint(true);
+      } else if (state === "SPEECH_ENDED") {
+        setAssistantHint(false);
+        playback.markAsLastChunk();
+      } else if (state === "USER_SPEECH_STARTED") {
+        setUserSpeaking(true);
+        playback.stopAll(); // barge-in: cut assistant playback the moment the user speaks
+      } else if (state === "USER_SPEECH_ENDED") {
+        setUserSpeaking(false);
+      } else if (state === "SESSION_READY") {
+        setConnectionStatus("connected");
+      } else if (state === "SESSION_ENDED") {
+        setConnectionStatus("ended");
+        setAssistantHint(false);
+        setUserSpeaking(false);
+        playback.markAsLastChunk();
+      } else if (state === "TOOL_USE") {
+        setIsLookingUp(true);
+        setLookupTool((metadata?.toolName as string | undefined) ?? null);
+      } else if (state === "TOOL_USE_COMPLETED") {
+        setIsLookingUp(false);
+        setLookupTool(null);
+      }
+    },
+    [playback],
+  );
+
+  const ws = useRealtimeWebSocket({
+    sessionId: session.conversationId,
+    userId: session.userId,
+    chatId: session.chatId,
+    wsUrl,
+    getAccessToken,
+    onAudioReceived: playback.playAudio,
+    onAudioState: handleAudioState,
+  });
+
+  // Capture → WS via a ref so the capture graph isn't rebuilt when sendAudio changes.
+  const handleAudioData = useCallback((d: ArrayBuffer) => sendAudioRef.current(d), []);
+  useEffect(() => {
+    sendAudioRef.current = ws.sendAudio;
+  }, [ws.sendAudio]);
+
+  const capture = useAudioCapture({ onAudioData: handleAudioData });
+
+  const startCall = useCallback(async () => {
+    setConnectionStatus("connecting");
+    setError(null);
+    setCallDuration(0);
+    try {
+      // 1. Open the audio lane (sends INIT_SESSION on open) BEFORE the trigger,
+      //    so the server can route audio the moment the Nova/Grok session is up.
+      await ws.connect();
+
+      // 2. START_CALL via REST — fire-and-forget; the workflow stays open for the call.
+      const token = getAccessToken ? await getAccessToken() : null;
+      const headers: Record<string, string> = { "Content-Type": "application/json" };
+      if (token) headers["Authorization"] = `Bearer ${token}`;
+      const chatId = session.chatId ?? `voice_${session.conversationId}`;
+      fetch(`${apiUrl}/api/workflows/${session.workflowId}/execute`, {
+        method: "POST",
+        headers,
+        body: JSON.stringify({
+          input: {
+            message: "Start call",
+            chatId,
+            conversationId: session.conversationId,
+            userId: session.userId,
+            providerId: "gravity-voice",
+            isAudio: true,
+            metadata: {
+              targetTriggerNode: session.targetTriggerNode,
+              action: "START_CALL",
+              isAction: true,
+              workflowId: session.workflowId,
+              continuousStream: true,
+            },
+          },
+          conversationId: session.conversationId,
+        }),
+      }).catch((e) => console.error("[voice] START_CALL error:", e));
+
+      // 3. Start streaming the mic.
+      await capture.startCapture();
+      timerRef.current = setInterval(() => setCallDuration((p) => p + 1), 1000);
+      setConnectionStatus("connected");
+    } catch (err: unknown) {
+      setError(err instanceof Error ? err.message : "Failed to start call");
+      setConnectionStatus("error");
+    }
+  }, [ws, capture, apiUrl, session, getAccessToken]);
+
+  const endCall = useCallback(async () => {
+    if (timerRef.current) {
+      clearInterval(timerRef.current);
+      timerRef.current = null;
+    }
+    await capture.stopCapture();
+    playback.stopAll();
+    // END via WS control — NOT a new REST execute (that would start a new workflow).
+    ws.sendControl("AUDIO_CONTROL", { command: "stop", workflowId: session.workflowId });
+    ws.disconnect();
+    setConnectionStatus("ended");
+    setIsMuted(false);
+  }, [capture, playback, ws, session.workflowId]);
+
+  const toggleMute = useCallback(() => {
+    setIsMuted((prev) => {
+      const next = !prev;
+      capture.setMuted(next);
+      return next;
+    });
+  }, [capture]);
+
+  // Cleanup on unmount.
+  useEffect(() => {
+    return () => {
+      if (timerRef.current) clearInterval(timerRef.current);
+      ws.disconnect();
+    };
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, []);
+
+  const isAssistantSpeaking = assistantHint || playback.isPlaying;
+  const isUserSpeaking = userSpeaking || capture.isSpeaking;
+  const isCallActive = connectionStatus === "connecting" || connectionStatus === "connected";
+  const lookupLabel =
+    lookupTool === "findIntent" ? "Searching the knowledge base…" : `Looking up ${lookupTool ?? "information"}…`;
+  const durationText = formatDuration(callDuration);
+  const statusText =
+    connectionStatus === "idle"
+      ? "Ready to call"
+      : connectionStatus === "connecting"
+        ? "Connecting…"
+        : connectionStatus === "connected"
+          ? durationText
+          : connectionStatus === "error"
+            ? (error ?? "Connection error")
+            : "Call ended";
+
+  const state: VoiceServiceState = useMemo(
+    () => ({
+      connectionStatus,
+      // The one phase the template branches on — derived from the neutral facts. Order matters:
+      // agent speaking wins over user (barge-in), both win over plain "active".
+      callState: (!isCallActive
+        ? "idle"
+        : isAssistantSpeaking
+          ? "agentSpeaking"
+          : isUserSpeaking
+            ? "userSpeaking"
+            : "active") as VoiceCallState,
+      isIdle: !isCallActive,
+      isCallActive,
+      isConnecting: connectionStatus === "connecting",
+      isConnected: connectionStatus === "connected",
+      isAssistantSpeaking,
+      isUserSpeaking,
+      showUserSpeaking: isUserSpeaking && !isAssistantSpeaking,
+      isMuted,
+      isLookingUp,
+      lookupLabel,
+      callDuration,
+      durationText,
+      statusText,
+      error: error ?? capture.error ?? null,
+    }),
+    [
+      connectionStatus,
+      isCallActive,
+      isAssistantSpeaking,
+      isUserSpeaking,
+      isMuted,
+      isLookingUp,
+      lookupLabel,
+      callDuration,
+      durationText,
+      statusText,
+      error,
+      capture.error,
+    ],
+  );
+
+  // PRODUCER write (STATE_MODEL §4): the service feeds the state, it is not state.
+  // Call state lands in TEMPLATE STATE like any dev-named key, so every template —
+  // studio, canvas, native — reads voice from the store's global bag with no
+  // host-side props spread. Low-frequency (call phases + a 1Hz duration tick).
+  const store = config.store;
+  useEffect(() => {
+    store?.mergeTemplateState(state as unknown as Record<string, unknown>);
+  }, [store, state]);
+
+  return { state, startCall, endCall, toggleMute };
+}

package/vendor/sdk/main.tsx

@@ -0,0 +1,96 @@
+/**
+ * The `ui://` app entry (§6d) — what a HOST loads (the unoverse server's SAB host in an
+ * iframe, or Claude/ChatGPT). Boots the AppEngine on the LOCAL vendored SDK. The host
+ * provides config (server, bearer, session facts) via `window.__UNOVERSE_APP_CONFIG` or a
+ * postMessage `{ type: "unoverse:config", config }` (also the token-refresh channel).
+ */
+import { createRoot, type Root } from "react-dom/client";
+import { AppEngine, type AppConfig } from "./appEngine";
+
+type HostConfig = Omit<AppConfig, "getAccessToken"> & { token?: string };
+
+let root: Root | null = null;
+let current: HostConfig | null = null;
+
+function render() {
+  if (!current) return;
+  const el = document.getElementById("root")!;
+  if (!root) root = createRoot(el);
+  root.render(
+    <AppEngine
+      serverUrl={current.serverUrl}
+      apiUrl={current.apiUrl}
+      templateId={current.templateId}
+      userId={current.userId}
+      conversationId={current.conversationId}
+      getAccessToken={() => current?.token ?? null}
+    />,
+  );
+}
+
+function apply(cfg: HostConfig) {
+  const first = !current;
+  current = { ...current, ...cfg };
+  if (first) render(); // mount once; token refresh just updates `current` (getAccessToken reads it)
+}
+
+declare global {
+  interface Window {
+    __UNOVERSE_APP_CONFIG?: HostConfig;
+  }
+}
+
+if (window.__UNOVERSE_APP_CONFIG) apply(window.__UNOVERSE_APP_CONFIG);
+window.addEventListener("message", (e) => {
+  const msg = e?.data;
+  if (msg?.type === "unoverse:config" && msg.config) apply(msg.config);
+});
+
+// ── ChatGPT (OpenAI Apps) bootstrap ────────────────────────────────────────
+// ChatGPT does NOT send our host's `unoverse:config` postMessage; it exposes a
+// `window.openai` bridge. Without this the SDK never receives config → #root stays
+// empty → blank widget (observed live). `toolOutput` = the tool's structuredContent
+// (app id + conversationId); `toolResponseMetadata` = the tool result `_meta`
+// (server-forwarded serverUrl + token + userId — widget-only, never seen by the model).
+// DIAGNOSTIC-FIRST: log the real runtime shape so we wire against ground truth, not the
+// docs' assumed names. `window.openai` may arrive after load, so also listen for its ready
+// event. This branch is inert in our own hosts (no window.openai there).
+function bootstrapFromOpenAI(): boolean {
+  const oai = (window as unknown as { openai?: Record<string, unknown> }).openai;
+  if (!oai) return false;
+  const out = (oai.toolOutput ?? {}) as Record<string, unknown>;
+  const meta = (oai.toolResponseMetadata ?? {}) as Record<string, unknown>;
+  // eslint-disable-next-line no-console
+  console.log("[unoverse:boot] window.openai present", {
+    keys: Object.keys(oai),
+    toolOutput: out,
+    toolResponseMetadataKeys: Object.keys(meta),
+    hasToken: Boolean(meta["unoverse/token"]),
+  });
+  // serverUrl is SERVER-PROVIDED (the tool result's `_meta.unoverse/serverUrl` = the origin
+  // that served this bundle). NEVER hardcode/guess it — a baked URL ties the bundle to one
+  // environment and hides a missing config. If it's absent, fail loudly and don't mount.
+  const serverUrl = meta["unoverse/serverUrl"] as string;
+  const templateId = (out.app as string) || (meta["unoverse/app"] as string);
+  const conversationId = (out.conversationId as string) || (meta["unoverse/conversationId"] as string);
+  const token = (meta["unoverse/token"] as string) || undefined;
+  const userId = (meta["unoverse/userId"] as string) || undefined;
+  if (!templateId || !serverUrl) {
+    // eslint-disable-next-line no-console
+    console.warn("[unoverse:boot] cannot mount — missing serverUrl or app id from window.openai", {
+      serverUrl,
+      templateId,
+      toolOutput: out,
+      toolResponseMetadataKeys: Object.keys(meta),
+    });
+    return false;
+  }
+  // eslint-disable-next-line no-console
+  console.log("[unoverse:boot] mounting", { serverUrl, templateId, conversationId, userId, hasToken: Boolean(token) });
+  apply({ serverUrl, templateId, conversationId, userId, token });
+  return true;
+}
+if (!bootstrapFromOpenAI()) {
+  // window.openai can be injected after the bundle loads — retry on its ready signal.
+  window.addEventListener("openai:set_globals", () => bootstrapFromOpenAI());
+}

package/vite.config.ts

@@ -0,0 +1,70 @@
+/**
+ * Local Studio — the standalone developer tool (docs/architecture/LOCAL_STUDIO.md).
+ *
+ * Runs entirely offline against the developer's own project folder. No Postgres, no
+ * Redis, no engine, no Docker: the catalog routes are served straight off disk by the
+ * local host plugin, using the SAME loaders the universe server uses.
+ */
+import { defineConfig } from "vite";
+import react from "@vitejs/plugin-react";
+import tailwindcss from "@tailwindcss/vite";
+import { resolve } from "path";
+import { existsSync } from "fs";
+import { localStudioHost } from "./local/plugin.js";
+
+export default defineConfig({
+  plugins: [react(), tailwindcss(), localStudioHost()],
+  server: {
+    // 4108. The ports below it are all taken by the dev stack:
+    //   4105 unoverse public   4106 the internal node-runtime listener (UNOVERSE_RUNTIME_PORT)
+    //   4107 apps/unoverse-runtime (`PORT || 4107`, main.ts)
+    // This was 4107, chosen to dodge 4106 and unaware that unoverse-runtime already held
+    // it. Running the full stack put two servers on one port, and Vite silently drifts to
+    // the next free one, so Studio moved and nothing said so.
+    port: 4108,
+    // Fail rather than drift. A tool that quietly answers on a different port than it
+    // printed is worse than one that refuses to start.
+    strictPort: true,
+    open: true,
+    fs: {
+      // The screens and the SDK are SOURCE outside this app's root, and the local host
+      // reads the developer's project. Allow the workspace root above us.
+      allow: [resolve(__dirname, "../..")],
+    },
+  },
+  resolve: {
+    dedupe: ["react", "react-dom"],
+    alias: {
+      // The Studio SCREENS, owned once and shared with the Canvas Studio.
+      "@unoverse/studio": resolve(__dirname, "screens/index.ts"),
+      // The host seam. Canvas aliases this to a universe; here it is the local disk.
+      "studio-host": resolve(__dirname, "src/host.ts"),
+      // The renderer, vendored source (same convention as Canvas and the old studio).
+      // THE VENDORED COPY WHEN THERE IS ONE. The monorepo path below exists only here;
+      // an installed Studio has `vendor/sdk`, written by prepack (scripts/vendor-sdk.mjs).
+      // Preferring the vendor directory is what makes the published package work at all,
+      // and falling back keeps the monorepo editing the real source with HMR.
+      "@unoverse/sdk": existsSync(resolve(__dirname, "vendor/sdk/lib/index.ts"))
+        ? resolve(__dirname, "vendor/sdk/lib/index.ts")
+        : resolve(__dirname, "../unoverse/web/sdk/src/lib/index.ts"),
+    },
+  },
+  ssr: {
+    // The local host loads the server's disk loaders through Vite's SSR runner, and two
+    // packages here are CommonJS, which the ESM runner cannot evaluate. Externalise them
+    // so Node requires them natively instead.
+    //
+    //   plugin-base   re-exported by inputs.ts
+    //   marketplace   the DESIGN SYSTEM. Studio is not optional about this: a developer's
+    //                 components are built ON TOP of marketplace atoms and components, so
+    //                 without it every shared reference resolves to nothing and Studio
+    //                 shows holes where the design system should be. It failed with
+    //                 "__dirname is not defined", the CommonJS giveaway.
+    external: ["@unoverse-platform/plugin-base", "@unoverse-platform/marketplace"],
+  },
+  optimizeDeps: {
+    // The SDK pulls in the MCP SDK, whose ajv-provider does `import Ajv from "ajv"`.
+    // ajv is CJS with no default ESM export, so pre-bundle for the interop shim.
+    include: ["ajv", "@modelcontextprotocol/sdk/client/index.js", "@modelcontextprotocol/sdk/client/streamableHttp.js"],
+  },
+});