@unoverse-platform/studio 0.1.102 → 0.1.104

package/vendor/sdk/lib/voiceService.ts

@@ -0,0 +1,344 @@
+/**
+ * The Unoverse `voice` NATIVE SERVICE (UNOVERSE_SPEC §2e-1 Tier-3), plain TypeScript.
+ *
+ * Audio I/O is inherently native (getUserMedia / AudioContext / the /ws/gravity WS lane), so
+ * it cannot be expressed as definition data: it is the spec's sanctioned escape hatch, a fixed
+ * `{ state, actions }` capability the SDK provides and a template REFERENCES
+ * (`service: "voice"`), binding its neutral state and dispatching its actions. The SDK owns NO
+ * styles and exposes only NEUTRAL model facts (never UX flags).
+ *
+ * Transport split (UNOVERSE_MCP_APP_PROTOCOL §5): the live PCM frames AND the synchronized
+ * audio-state events both ride the /ws/gravity lane; only component data is on MCP.
+ */
+import { RealtimeSocket } from "./realtime/socket";
+import { AudioCapture } from "./realtime/capture";
+import { AudioPlayback } from "./realtime/playback";
+import type { AudioStateEvent } from "./realtime/types";
+import type { ComponentStore } from "./core/store";
+import { sessionUserId } from "./dom/connection";
+
+export type VoiceConnectionStatus = "idle" | "connecting" | "connected" | "ended" | "error";
+
+export interface VoiceSession {
+  userId: string;
+  conversationId: string;
+  chatId?: string;
+  workflowId: string;
+  targetTriggerNode: string;
+}
+
+export interface VoiceServiceConfig {
+  /** Gateway REST base, e.g. http://localhost:4100 (the workflow `…/execute` host). */
+  apiUrl: string;
+  /** Audio WS base, e.g. ws://localhost:4100. Defaults to `apiUrl` with http→ws. */
+  wsUrl?: string;
+  /** Fresh access token provider (rides the WS lane + the REST Authorization header). */
+  getAccessToken?: () => Promise<string | null> | string | null;
+  session: VoiceSession;
+  /** The shared store. Given, the service is a PRODUCER (STATE_MODEL §4): every state change
+   *  is merged into TEMPLATE STATE, so templates read voice like any global key. */
+  store?: ComponentStore;
+}
+
+/** The single call phase a voice template branches its states on (`callState`). */
+export type VoiceCallState = "idle" | "active" | "speaking" | "listening" | "thinking";
+
+/** NEUTRAL voice projections, flat, so they spread into the template scope. */
+export interface VoiceServiceState {
+  connectionStatus: VoiceConnectionStatus;
+  callState: VoiceCallState;
+  isIdle: boolean;
+  isCallActive: boolean;
+  isConnecting: boolean;
+  isConnected: boolean;
+  isAssistantSpeaking: boolean;
+  isUserSpeaking: boolean;
+  /** user speaking AND assistant not: mutual exclusion (the template can't express NOT). */
+  showUserSpeaking: boolean;
+  isMuted: boolean;
+  isLookingUp: boolean;
+  lookupLabel: string;
+  callDuration: number;
+  durationText: string;
+  statusText: string;
+  error: string | null;
+}
+
+export interface StartCall {
+  message?: string;
+  metadata?: Record<string, unknown>;
+  workflowId?: string;
+  targetTriggerNode?: string;
+}
+
+export interface VoiceService {
+  /** The current state, to spread into the template's data scope. */
+  readonly state: VoiceServiceState;
+  /** Called on every state change; returns the unsubscribe. */
+  subscribe(listener: () => void): () => void;
+  /** Start a call (or MOVE a live one to another workflow: same line, the server closes the
+   *  session before it, VOICE_STREAMING_GUIDE.md "One Live Session Per Conversation"). */
+  startCall: (start?: StartCall) => Promise<void>;
+  /** Join a call already running (a canvas Run, a button, `JOIN_CALL`) without starting it. */
+  joinCall: (workflowId?: string) => Promise<void>;
+  /** Wake the speaker. Call it inside the click that may start a call. */
+  unlock: () => void;
+  endCall: () => Promise<void>;
+  toggleMute: () => void;
+  /** Release the microphone, the speaker and the lane. */
+  destroy: () => void;
+}
+
+const formatDuration = (s: number) => `${Math.floor(s / 60).toString().padStart(2, "0")}:${(s % 60).toString().padStart(2, "0")}`;
+
+export function createVoiceService(config: VoiceServiceConfig): VoiceService {
+  const { apiUrl, getAccessToken } = config;
+  const wsUrl = config.wsUrl ?? apiUrl.replace(/^http/, "ws");
+  // Same identity as the data plane: without a token the audio socket is a guest socket.
+  const session = { ...config.session, userId: sessionUserId(config.session.userId, getAccessToken) };
+
+  let connectionStatus: VoiceConnectionStatus = "idle";
+  let isMuted = false;
+  let callDuration = 0;
+  let error: string | null = null;
+  let assistantHint = false;
+  let userSpeaking = false;
+  let isLookingUp = false;
+  let lookupTool: string | null = null;
+  let timer: ReturnType<typeof setInterval> | null = null;
+  /** The workflow the live call is on, which a hang-up names; null with no call. */
+  let callWorkflow: string | null = null;
+  const listeners = new Set<() => void>();
+  // Set once the speaker and microphone exist (below); `derive` reads them.
+  let state!: VoiceServiceState;
+
+  function derive(): VoiceServiceState {
+    const isAssistantSpeaking = assistantHint || playback.isPlaying;
+    const isUserSpeaking = userSpeaking || capture.isSpeaking;
+    const isCallActive = connectionStatus === "connecting" || connectionStatus === "connected";
+    const durationText = formatDuration(callDuration);
+    return {
+      connectionStatus,
+      // The one phase the template branches on. A lookup wins (it has a true start and end,
+      // while speaking may never end: GPT-Live sends no end-of-speech), then the agent speaking
+      // over the user (barge-in), and all three win over plain "active".
+      callState: !isCallActive ? "idle" : isLookingUp ? "thinking" : isAssistantSpeaking ? "speaking" : isUserSpeaking ? "listening" : "active",
+      isIdle: !isCallActive,
+      isCallActive,
+      isConnecting: connectionStatus === "connecting",
+      isConnected: connectionStatus === "connected",
+      isAssistantSpeaking,
+      isUserSpeaking,
+      showUserSpeaking: isUserSpeaking && !isAssistantSpeaking,
+      isMuted,
+      isLookingUp,
+      lookupLabel: lookupTool === "findIntent" ? "Searching the knowledge base…" : `Looking up ${lookupTool ?? "information"}…`,
+      callDuration,
+      durationText,
+      statusText:
+        connectionStatus === "idle"
+          ? "Ready to call"
+          : connectionStatus === "connecting"
+            ? "Connecting…"
+            : connectionStatus === "connected"
+              ? durationText
+              : connectionStatus === "error"
+                ? (error ?? "Connection error")
+                : "Call ended",
+      error: error ?? capture.error ?? null,
+    };
+  }
+
+  /** Recompute; on any change, tell listeners and (PRODUCER) write it into template state. */
+  function changed(): void {
+    const next = derive();
+    const keys = Object.keys(next) as (keyof VoiceServiceState)[];
+    if (keys.every((k) => next[k] === state[k])) return;
+    state = next;
+    config.store?.mergeAppState(state as unknown as Record<string, unknown>);
+    listeners.forEach((l) => l());
+  }
+
+  const playback: AudioPlayback = new AudioPlayback(changed);
+  // Audio-state events arrive on the WS lane, in lockstep with the frames.
+  const onAudioState = ({ state: s, metadata }: AudioStateEvent) => {
+    if (s === "SPEECH_STARTED") assistantHint = true;
+    else if (s === "SPEECH_ENDED") {
+      assistantHint = false;
+      playback.markAsLastChunk();
+    } else if (s === "USER_SPEECH_STARTED") {
+      userSpeaking = true;
+      playback.stopAll(); // barge-in: cut assistant playback the moment the user speaks
+    } else if (s === "USER_SPEECH_ENDED") userSpeaking = false;
+    else if (s === "SESSION_READY") connectionStatus = "connected";
+    else if (s === "SESSION_ENDED") {
+      // THE SERVER ENDED THE CALL: the microphone stops and the lane closes.
+      release();
+      connectionStatus = "ended";
+      assistantHint = false;
+      userSpeaking = false;
+      isLookingUp = false;
+      lookupTool = null;
+      playback.markAsLastChunk();
+    } else if (s === "TOOL_USE") {
+      isLookingUp = true;
+      lookupTool = (metadata?.toolName as string | undefined) ?? null;
+    } else if (s === "TOOL_USE_COMPLETED") {
+      isLookingUp = false;
+      lookupTool = null;
+    }
+    changed();
+  };
+  const ws = new RealtimeSocket({
+    sessionId: session.conversationId,
+    userId: session.userId,
+    chatId: session.chatId,
+    wsUrl,
+    getAccessToken,
+    onAudioReceived: (d) => playback.play(d),
+    onAudioState,
+  });
+  const capture: AudioCapture = new AudioCapture((d) => ws.sendAudio(d), changed);
+  state = derive();
+  // The starting state lands in template state too, as every later one does.
+  config.store?.mergeAppState(state as unknown as Record<string, unknown>);
+
+  const stopTimer = () => {
+    if (timer) clearInterval(timer);
+    timer = null;
+  };
+  const startTimer = () => {
+    stopTimer();
+    timer = setInterval(() => {
+      callDuration += 1;
+      changed();
+    }, 1000);
+  };
+  function release(): void {
+    stopTimer();
+    callWorkflow = null;
+    void capture.stop();
+    ws.disconnect();
+  }
+
+  return {
+    get state() {
+      return state;
+    },
+    subscribe(listener) {
+      listeners.add(listener);
+      return () => void listeners.delete(listener);
+    },
+    async startCall(start) {
+      const workflowId = start?.workflowId || session.workflowId;
+      const targetTriggerNode = start?.targetTriggerNode || session.targetTriggerNode;
+      if (!workflowId) {
+        // NOTHING TO CALL: no workflow was named and the session has none. Never a default.
+        error = "No workflow to call";
+        connectionStatus = "error";
+        return changed();
+      }
+      // A CALL ALREADY LIVE IS MOVED, not started again: the line and the microphone stay open,
+      // only the workflow changes; the old voice is cut at once so two are never heard.
+      const moving = callWorkflow !== null;
+      if (moving) {
+        playback.stopAll();
+        assistantHint = false;
+        isLookingUp = false;
+        lookupTool = null;
+      } else {
+        connectionStatus = "connecting";
+        error = null;
+        callDuration = 0;
+      }
+      changed();
+      try {
+        // 1. The audio lane first (INIT_SESSION on open), so the server can route audio the
+        //    moment the voice session is up.
+        if (!moving) await ws.connect();
+        callWorkflow = workflowId;
+        // 2. START_CALL over REST, fire-and-forget; the workflow stays open for the call.
+        const token = getAccessToken ? await getAccessToken() : null;
+        const headers: Record<string, string> = { "Content-Type": "application/json" };
+        if (token) headers["Authorization"] = `Bearer ${token}`;
+        const chatId = session.chatId ?? `voice_${session.conversationId}`;
+        fetch(`${apiUrl}/api/workflows/${workflowId}/execute`, {
+          method: "POST",
+          headers,
+          body: JSON.stringify({
+            input: {
+              message: start?.message ?? "Start call",
+              chatId,
+              conversationId: session.conversationId,
+              userId: session.userId,
+              providerId: "gravity-voice",
+              isAudio: true,
+              metadata: { ...(start?.metadata ?? {}), targetTriggerNode, action: "START_CALL", isAction: true, workflowId, continuousStream: true },
+            },
+            conversationId: session.conversationId,
+          }),
+        }).catch((e) => console.error("[voice] START_CALL error:", e));
+        if (moving) return;
+        // 3. Stream the mic.
+        await capture.start();
+        startTimer();
+        connectionStatus = "connected";
+        changed();
+      } catch (err: unknown) {
+        callWorkflow = null;
+        error = err instanceof Error ? err.message : "Failed to start call";
+        connectionStatus = "error";
+        isLookingUp = false;
+        lookupTool = null;
+        changed();
+      }
+    },
+    // JOIN, NOT START: the workflow already runs; open the lane and the mic, send no execute.
+    async joinCall(workflowId) {
+      if (callWorkflow !== null) return;
+      callWorkflow = workflowId ?? session.workflowId ?? "";
+      connectionStatus = "connecting";
+      error = null;
+      callDuration = 0;
+      changed();
+      try {
+        await ws.connect();
+        await capture.start();
+        startTimer();
+        connectionStatus = "connected";
+      } catch (err: unknown) {
+        callWorkflow = null;
+        error = err instanceof Error ? err.message : "Failed to join call";
+        connectionStatus = "error";
+      }
+      changed();
+    },
+    async endCall() {
+      stopTimer();
+      await capture.stop();
+      playback.stopAll();
+      // END via WS control, NOT a new REST execute (that would start a new workflow).
+      ws.sendControl("AUDIO_CONTROL", { command: "stop", workflowId: callWorkflow ?? session.workflowId });
+      callWorkflow = null;
+      ws.disconnect();
+      connectionStatus = "ended";
+      isMuted = false;
+      isLookingUp = false;
+      lookupTool = null;
+      changed();
+    },
+    toggleMute() {
+      isMuted = !isMuted;
+      capture.setMuted(isMuted);
+      changed();
+    },
+    unlock: () => playback.unlock(),
+    destroy() {
+      stopTimer();
+      void capture.stop();
+      ws.disconnect();
+      playback.close();
+      listeners.clear();
+    },
+  };
+}

package/vendor/sdk/main.ts

@@ -0,0 +1,107 @@
+/**
+ * ⛔ NO REACT ANYWHERE IN THIS SDK (owner, 2026-10-01; web/sdk/CLAUDE.md).
+ *
+ * The `ui://` app entry (§6d): what a HOST loads (the unoverse server's SAB host in an iframe,
+ * or Claude/ChatGPT). Boots the app engine on the LOCAL vendored SDK. The host provides config
+ * (server, bearer, session facts) via `window.__UNOVERSE_APP_CONFIG` or a postMessage
+ * `{ type: "unoverse:config", config }` (also the token-refresh channel).
+ */
+import { mountAppEngine, type AppConfig, type AppEngine } from "./appEngine";
+import { connectMcpAppHost } from "./mcpAppBridge";
+import { mountResponseView, type ResponseView } from "./responseView";
+
+type HostConfig = Omit<AppConfig, "getAccessToken"> & { token?: string };
+
+const root = () => document.getElementById("root")!;
+let engine: AppEngine | null = null;
+// A task result draws as the response view; one window shows one or the other.
+let response: ResponseView | null = null;
+let current: HostConfig | null = null;
+
+// Stable identity: the connection keys its lifecycle on whether a token accessor exists, and
+// the host re-posts config on every iframe load.
+const readToken = () => current?.token ?? null;
+
+function render() {
+  if (!current) return;
+  response?.destroy();
+  response = null;
+  const cfg: AppConfig = {
+    serverUrl: current.serverUrl,
+    apiUrl: current.apiUrl,
+    appId: current.appId,
+    userId: current.userId,
+    conversationId: current.conversationId,
+    // ANALYTICS (UNOVERSE_ANALYTICS.md): host-supplied. A host that says nothing gets
+    // `mcp_app_host`, the one surface with no parent page to report to.
+    surface: current.surface ?? "mcp_app_host",
+    analyticsDebug: current.analyticsDebug,
+    // Only claim auth when the host sent a token: no token ⇒ no accessor ⇒ the SDK mints a
+    // `guest-<uuid>` (UNOVERSE_MCP_APP_PROTOCOL §5, docs/auth-security/IDENTITY.md).
+    getAccessToken: current.token ? readToken : undefined,
+  };
+  if (engine) engine.update(cfg);
+  else engine = mountAppEngine(root(), cfg);
+}
+
+function apply(cfg: HostConfig) {
+  current = { ...current, ...cfg };
+  // Re-apply on every config so a token appearing/disappearing takes effect; a same-token
+  // refresh changes nothing (readToken is stable, values equal).
+  render();
+}
+
+declare global {
+  interface Window {
+    __UNOVERSE_APP_CONFIG?: HostConfig;
+  }
+}
+
+if (window.__UNOVERSE_APP_CONFIG) apply(window.__UNOVERSE_APP_CONFIG);
+window.addEventListener("message", (e) => {
+  const msg = e?.data;
+  if (msg?.type === "unoverse:config" && msg.config) apply(msg.config);
+});
+
+// ── MCP Apps host bootstrap (SEP-1865, the STANDARD door) ──────────────────
+// An MCP Apps host speaks JSON-RPC over postMessage; the boot facts ride the result's `_meta`
+// (`unoverse/serverUrl` + app + token, widget-only, never seen by the model) with
+// `structuredContent` as the copy.
+connectMcpAppHost((r) => {
+  const meta = (r._meta ?? {}) as Record<string, unknown>;
+  const out = (r.structuredContent ?? {}) as Record<string, unknown>;
+  // serverUrl is SERVER-PROVIDED, never guessed: absent, fail loudly and don't mount.
+  const serverUrl = (meta["unoverse/serverUrl"] as string) || (out.serverUrl as string);
+  const conversationId = (out.conversationId as string) || (meta["unoverse/conversationId"] as string);
+  const token = (meta["unoverse/token"] as string) || undefined;
+  const userId = (meta["unoverse/userId"] as string) || undefined;
+
+  // THE NAMING RULE (UNOVERSE_MCP_APP_PROTOCOL): the RESULT names its interface.
+  // `components` → draw exactly those (the response view); `app` (or the app tools' legacy
+  // boot fact) → that app mounts. A result naming nothing renders nothing, loudly.
+  const iface = out.interface as { kind?: string; components?: Record<string, unknown>[]; app?: string } | undefined;
+  if (iface?.kind === "components" && Array.isArray(iface.components)) {
+    if (!serverUrl) {
+      console.warn("[unoverse:boot] response result carried no serverUrl — cannot resolve definitions");
+      return;
+    }
+    console.log("[unoverse:boot] rendering RESPONSE from MCP host", { serverUrl, components: iface.components.length });
+    engine?.destroy();
+    engine = null;
+    root().replaceChildren();
+    // A page opened on an outside host stays live: its `-update` calls push to this page's
+    // conversation, and the view hydrates from them (responseView.ts, `live`).
+    const live = userId && conversationId ? { userId, conversationId, token } : undefined;
+    response?.destroy();
+    response = mountResponseView(root(), { serverUrl, events: iface.components, live });
+    return;
+  }
+
+  const appId = (iface?.kind === "app" ? iface.app : undefined) || (out.app as string) || (meta["unoverse/app"] as string);
+  if (!appId || !serverUrl) {
+    console.warn("[unoverse:boot] cannot mount — result named no interface (no components, no app)", { serverUrl, appId, structuredContent: out, metaKeys: Object.keys(meta) });
+    return;
+  }
+  console.log("[unoverse:boot] mounting NAMED app from MCP host", { serverUrl, appId, conversationId, userId, hasToken: Boolean(token) });
+  apply({ serverUrl, appId, conversationId, userId, token });
+});