create-spatius-app 0.1.0-beta.0

package/templates/cloudflare-livekit/web/components/conversation.tsx

@@ -0,0 +1,352 @@
+import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
+import { ConnectionState, Track } from 'livekit-client';
+import {
+  useAgent,
+  useLocalParticipant,
+  useMediaDeviceSelect,
+  useSession,
+  useSessionMessages,
+} from '@livekit/components-react';
+import type { PreparedSession } from '../session-attempt.js';
+import { AgentSessionProvider } from './agents-ui/agent-session-provider.js';
+import { AgentTrackToggle } from './agents-ui/agent-track-toggle.js';
+import { AgentDisconnectButton } from './agents-ui/agent-disconnect-button.js';
+import { AgentChatInput } from './agents-ui/agent-chat-input.js';
+import {
+  AgentChatTranscript,
+  snapshotMessages,
+  type ConversationParticipants,
+  type TranscriptMessage,
+} from './agents-ui/agent-chat-transcript.js';
+import { StartAudioButton } from './agents-ui/start-audio-button.js';
+import { AgentAudioVisualizerBar } from './agents-ui/agent-audio-visualizer-bar.js';
+import { TranscriptPanel } from './transcript-panel.js';
+
+export type ConversationEnd = {
+  messages: TranscriptMessage[];
+  agentName: string;
+  error?: string;
+};
+type Props = {
+  prepared: PreparedSession;
+  onEnd: (result: ConversationEnd) => void;
+};
+
+export function Conversation({ prepared, onEnd }: Props) {
+  const { attempt, tokenSource } = prepared;
+  const options = useMemo(
+    () => ({ room: attempt.room, agentConnectTimeoutMilliseconds: 30_000 }),
+    [attempt],
+  );
+  const session = useSession(tokenSource, options);
+  return (
+    <AgentSessionProvider session={session}>
+      <ConversationContent
+        prepared={prepared}
+        onEnd={onEnd}
+        session={session}
+      />
+    </AgentSessionProvider>
+  );
+}
+
+function ConversationContent({
+  prepared,
+  onEnd,
+  session,
+}: Props & { session: ReturnType<typeof useSession> }) {
+  const { attempt } = prepared;
+  const agent = useAgent(session);
+  // One subscription owns synchronized speech and typed messages for every view.
+  const stream = useSessionMessages(session);
+  const participants = useRef<ConversationParticipants>({});
+  const previousMessages = useRef<TranscriptMessage[]>([]);
+  const agentIdentity = agent.internal.agentParticipant?.identity;
+  const avatarIdentity = agent.internal.workerParticipant?.identity;
+  // LiveKit identifies the Python agent separately from the participant that
+  // publishes avatar media on its behalf. Retain IDs through disconnection.
+  if (agentIdentity) participants.current.agentIdentity = agentIdentity;
+  if (avatarIdentity) participants.current.avatarIdentity = avatarIdentity;
+  const messages = useMemo(
+    () =>
+      snapshotMessages(
+        stream.messages,
+        participants.current,
+        previousMessages.current,
+      ),
+    [stream.messages, agentIdentity, avatarIdentity],
+  );
+  previousMessages.current = messages;
+  const agentName = agent.name?.trim() || 'Your agent';
+  const latest = useRef({ messages, agentName });
+  latest.current = { messages, agentName };
+  const [roomReady, setRoomReady] = useState(false);
+  const ready =
+    roomReady && ['listening', 'thinking', 'speaking'].includes(agent.state);
+  const readyRef = useRef(false);
+  if (ready) readyRef.current = true;
+  const microphoneRequested = useRef(false);
+  const [microphonePending, setMicrophonePending] = useState(false);
+  const [devicePending, setDevicePending] = useState(false);
+  const [mediaError, setMediaError] = useState('');
+  const [elapsed, setElapsed] = useState(0);
+  const mediaBusy = useRef(false);
+  const { isMicrophoneEnabled, microphoneTrack } = useLocalParticipant({
+    room: attempt.room,
+  });
+  // LiveKit rebuilds its device observer when this callback changes. An inline
+  // callback would reset the device state on every render and cause a loop.
+  const onDeviceError = useCallback(() => {
+    if (!attempt.abort.signal.aborted)
+      setMediaError(
+        'Could not use that microphone. Choose another device or keep typing.',
+      );
+  }, [attempt]);
+  const { devices, activeDeviceId, setActiveMediaDevice } =
+    useMediaDeviceSelect({
+      room: attempt.room,
+      kind: 'audioinput',
+      requestPermissions: false,
+      onError: onDeviceError,
+    });
+  const toggleMicrophone = async (enabled: boolean) => {
+    if (mediaBusy.current || attempt.abort.signal.aborted) return;
+    mediaBusy.current = true;
+    setMicrophonePending(true);
+    setMediaError('');
+    try {
+      await attempt.setMicrophone(enabled);
+    } catch {
+      if (!attempt.abort.signal.aborted)
+        setMediaError(
+          'Microphone unavailable. Allow access and try the microphone again, or continue by typing.',
+        );
+    } finally {
+      mediaBusy.current = false;
+      if (!attempt.abort.signal.aborted) setMicrophonePending(false);
+    }
+  };
+  const finish = (error?: string) => onEnd({ ...latest.current, error });
+  const startSession = useRef(session.start);
+  const finishRef = useRef(finish);
+  finishRef.current = finish;
+
+  useEffect(() => {
+    let cancelled = false;
+    const timeout = window.setTimeout(() => {
+      if (!readyRef.current && !cancelled && !attempt.abort.signal.aborted)
+        finishRef.current(
+          'Your agent did not join in time. Check that the LiveKit agent is running and try again.',
+        );
+    }, 32_000);
+    // StrictMode rehearses effects. Only its surviving effect starts this attempt.
+    queueMicrotask(() => {
+      if (cancelled || attempt.abort.signal.aborted) return;
+      void startSession
+        .current({
+          signal: attempt.abort.signal,
+          tracks: {
+            microphone: { enabled: false },
+            camera: { enabled: false },
+            screenShare: { enabled: false },
+          },
+        })
+        .then(() => {
+          if (cancelled || attempt.abort.signal.aborted) return;
+          setRoomReady(true);
+        })
+        .catch(() => {
+          if (!cancelled && !attempt.abort.signal.aborted)
+            finishRef.current(
+              'Could not connect to your agent. Please try again.',
+            );
+        });
+    });
+    return () => {
+      cancelled = true;
+      window.clearTimeout(timeout);
+    };
+  }, [attempt]);
+
+  useEffect(() => {
+    if (!ready || microphoneRequested.current || attempt.abort.signal.aborted)
+      return;
+    microphoneRequested.current = true;
+    void toggleMicrophone(true);
+  }, [ready, attempt]);
+
+  useEffect(() => {
+    // Snapshot after React commits both the final message and disconnection state.
+    if (
+      roomReady &&
+      session.connectionState === ConnectionState.Disconnected &&
+      !attempt.abort.signal.aborted
+    ) {
+      finishRef.current(
+        'The connection ended. You can start a new conversation.',
+      );
+    }
+  }, [roomReady, session.connectionState, attempt]);
+
+  useEffect(() => {
+    if (!ready) return;
+    const started = Date.now();
+    const interval = window.setInterval(
+      () => setElapsed(Math.floor((Date.now() - started) / 1000)),
+      1000,
+    );
+    return () => window.clearInterval(interval);
+  }, [ready]);
+
+  useEffect(() => {
+    if (
+      roomReady &&
+      agent.state === 'failed' &&
+      attempt.room.state === ConnectionState.Connected
+    )
+      finishRef.current(
+        'Your agent is unavailable. Check that it is running and try again.',
+      );
+  }, [agent.state, roomReady, attempt]);
+
+  const reconnecting =
+    session.connectionState === ConnectionState.Reconnecting ||
+    session.connectionState === ConnectionState.SignalReconnecting;
+  const interactive = ready && session.isConnected && !reconnecting;
+  const status = reconnecting
+    ? 'Reconnecting'
+    : !ready
+      ? 'Connecting'
+      : agent.state === 'speaking'
+        ? 'Speaking'
+        : agent.state === 'thinking'
+          ? 'Thinking'
+          : isMicrophoneEnabled
+            ? 'Listening'
+            : 'Ready to chat';
+  const selectableDevices = devices.filter((device) => device.deviceId);
+  const send = async (message: string) => {
+    attempt.abort.signal.throwIfAborted();
+    if (!interactive) throw new Error('Not connected');
+    return stream.send(message);
+  };
+
+  return (
+    <div className="conversation-ui">
+      <header className="session-header">
+        <div className="identity-pill">
+          <span className="presence-dot" data-live={ready} />
+          <div>
+            <h1>{agentName}</h1>
+            <p role="status">{status} · AI agent</p>
+          </div>
+          <span
+            className="session-timer"
+            aria-label="Elapsed conversation time"
+          >
+            {Math.floor(elapsed / 60)}:{String(elapsed % 60).padStart(2, '0')}
+          </span>
+        </div>
+      </header>
+      {!ready && !reconnecting && (
+        <div className="connection-overlay">
+          <span className="loading-spinner" aria-hidden="true" />
+          <h2>
+            {roomReady ? 'Waiting for your agent' : 'Connecting to the room'}
+          </h2>
+          <p role="status">
+            {roomReady
+              ? 'Your agent is starting up. This can take a moment.'
+              : 'Your avatar is ready. Establishing a secure connection.'}
+          </p>
+          <button
+            className="text-button"
+            type="button"
+            onClick={() => finish()}
+          >
+            Cancel
+          </button>
+        </div>
+      )}
+      <div className="conversation-tools">
+        <StartAudioButton room={attempt.room} />
+        <AgentAudioVisualizerBar
+          audioTrack={{
+            participant: attempt.room.localParticipant,
+            source: Track.Source.Microphone,
+            publication: microphoneTrack,
+          }}
+          enabled={interactive && isMicrophoneEnabled}
+        />
+        <div className="call-dock">
+          <AgentTrackToggle
+            pressed={isMicrophoneEnabled}
+            pending={microphonePending}
+            disabled={!interactive}
+            onPressedChange={(enabled) => void toggleMicrophone(enabled)}
+          />
+          <div className="device-select">
+            <label className="sr-only" htmlFor="microphone-device">
+              Microphone device
+            </label>
+            <select
+              id="microphone-device"
+              aria-label="Microphone device"
+              value={activeDeviceId || ''}
+              disabled={!interactive || microphonePending || devicePending}
+              onChange={(event) => {
+                const id = event.target.value;
+                setDevicePending(true);
+                setMediaError('');
+                void attempt
+                  .switchMicrophoneDevice(() => setActiveMediaDevice(id))
+                  .catch(() => {
+                    if (!attempt.abort.signal.aborted)
+                      setMediaError(
+                        'Could not switch microphones. Try another device.',
+                      );
+                  })
+                  .finally(() => {
+                    if (!attempt.abort.signal.aborted) setDevicePending(false);
+                  });
+              }}
+            >
+              <option value="">Default microphone</option>
+              {selectableDevices.map((device, index) => (
+                <option key={device.deviceId} value={device.deviceId}>
+                  {device.label || `Microphone ${index + 1}`}
+                </option>
+              ))}
+            </select>
+          </div>
+          <AgentDisconnectButton
+            onClick={(event) => {
+              event.preventDefault();
+              finish();
+            }}
+          >
+            End call
+          </AgentDisconnectButton>
+        </div>
+        {mediaError && (
+          <p className="control-error" role="alert">
+            {mediaError}
+          </p>
+        )}
+      </div>
+      <TranscriptPanel>
+        <AgentChatTranscript
+          messages={messages}
+          agentState={agent.state}
+          agentName={agentName}
+        />
+        <div className="panel-composer">
+          <AgentChatInput
+            disabled={!interactive || stream.isSending}
+            onSend={send}
+          />
+        </div>
+      </TranscriptPanel>
+    </div>
+  );
+}

package/templates/cloudflare-livekit/web/components/icons.tsx

@@ -0,0 +1,39 @@
+export type IconName =
+  | 'mic'
+  | 'mic-off'
+  | 'end'
+  | 'chat'
+  | 'close'
+  | 'send'
+  | 'type'
+  | 'audio'
+  | 'arrow';
+const paths: Record<IconName, string> = {
+  mic: 'M12 15a3 3 0 0 0 3-3V5a3 3 0 0 0-6 0v7a3 3 0 0 0 3 3ZM5 10v2a7 7 0 0 0 14 0v-2M12 19v3M8 22h8',
+  'mic-off':
+    'm3 3 18 18M9 9v3a3 3 0 0 0 5.1 2.1M15 9V5a3 3 0 0 0-5.7-1.3M5 10v2a7 7 0 0 0 12 5M19 10v2M12 19v3M8 22h8',
+  end: 'M4 15v3H1v-5c6-6 16-6 22 0v5h-3v-3l-4-2v-3M8 10v3l-4 2',
+  chat: 'M4 4h16v13H8l-4 4V4ZM8 9h8M8 13h5',
+  close: 'm6 6 12 12M6 18 18 6',
+  send: 'm5 12 7-7 7 7M12 5v15',
+  type: 'M2 5h20v14H2V5ZM6 9h.01M10 9h.01M14 9h.01M18 9h.01M6 13h.01M10 13h.01M14 13h4',
+  audio: 'm3 9 5 0 5-4v14l-5-4H3V9ZM17 8a6 6 0 0 1 0 8M20 5a10 10 0 0 1 0 14',
+  arrow: 'M5 12h14m-6-6 6 6-6 6',
+};
+export function Icon({ name }: { name: IconName }) {
+  return (
+    <svg
+      width="20"
+      height="20"
+      viewBox="0 0 24 24"
+      fill="none"
+      stroke="currentColor"
+      strokeWidth="1.6"
+      strokeLinecap="round"
+      strokeLinejoin="round"
+      aria-hidden="true"
+    >
+      <path d={paths[name]} />
+    </svg>
+  );
+}

package/templates/cloudflare-livekit/web/components/transcript-panel.tsx

@@ -0,0 +1,12 @@
+import type { ReactNode } from 'react';
+
+export function TranscriptPanel({ children }: { children: ReactNode }) {
+  return (
+    <aside className="transcript-panel" aria-labelledby="transcript-title">
+      <header className="transcript-heading">
+        <h2 id="transcript-title">Transcript</h2>
+      </header>
+      {children}
+    </aside>
+  );
+}

package/templates/cloudflare-livekit/web/main.tsx

@@ -0,0 +1,17 @@
+import { StrictMode } from 'react';
+import { createRoot } from 'react-dom/client';
+
+import App from './App.js';
+import './styles.css';
+
+const root = document.querySelector('#root');
+
+if (!root) {
+  throw new Error('Could not find the application root.');
+}
+
+createRoot(root).render(
+  <StrictMode>
+    <App />
+  </StrictMode>,
+);

package/templates/cloudflare-livekit/web/session-attempt.ts

@@ -0,0 +1,110 @@
+import { Room, TokenSource, Track } from 'livekit-client';
+import { requestVoiceSession, type VoiceSession } from './api.js';
+import type { AvatarSessionController } from './avatar-session.js';
+
+export type PreparedSession = {
+  attempt: SessionAttempt;
+  tokenSource: ReturnType<typeof TokenSource.literal>;
+};
+export interface AttemptDependencies {
+  createRoom: () => Room;
+  request: (signal: AbortSignal) => Promise<VoiceSession>;
+  attach: (
+    container: HTMLElement,
+    credentials: VoiceSession,
+    room: Room,
+    signal: AbortSignal,
+    report: (status: string) => void,
+  ) => Promise<AvatarSessionController>;
+}
+const dependencies: AttemptDependencies = {
+  createRoom: () => new Room({ singlePeerConnection: false }),
+  request: requestVoiceSession,
+  attach: async (...args) => {
+    const { attachAvatarSession } = await import('./avatar-session.js');
+    return attachAvatarSession(...args);
+  },
+};
+
+export class SessionAttempt {
+  stage: 'bootstrap' | 'avatar' = 'bootstrap';
+  readonly room: Room;
+  readonly abort = new AbortController();
+  private avatar?: AvatarSessionController;
+  private disposal?: Promise<void>;
+  constructor(private readonly deps: AttemptDependencies = dependencies) {
+    this.room = deps.createRoom();
+  }
+  async prepare(
+    container: HTMLElement,
+    report: (status: string) => void,
+  ): Promise<PreparedSession> {
+    const { signal } = this.abort;
+    const credentials = await this.deps.request(signal);
+    signal.throwIfAborted();
+    // Warm LiveKit's region selection and connection while the avatar loads.
+    // This does not join the room or acquire media; attach must still finish
+    // before useSession starts the connection. A failed warmup is non-fatal.
+    void this.room
+      .prepareConnection(credentials.server_url, credentials.participant_token)
+      .catch(() => undefined);
+    this.stage = 'avatar';
+    const avatar = await this.deps.attach(
+      container,
+      credentials,
+      this.room,
+      signal,
+      report,
+    );
+    if (signal.aborted) {
+      await avatar.dispose();
+      signal.throwIfAborted();
+    }
+    this.avatar = avatar;
+    return {
+      attempt: this,
+      tokenSource: TokenSource.literal({
+        serverUrl: credentials.server_url,
+        participantToken: credentials.participant_token,
+      }),
+    };
+  }
+  async setMicrophone(enabled: boolean): Promise<void> {
+    this.abort.signal.throwIfAborted();
+    const publication =
+      await this.room.localParticipant.setMicrophoneEnabled(enabled);
+    if (this.abort.signal.aborted) {
+      // A pending permission prompt may finish after End. Stop its late track too.
+      publication?.track?.stop();
+      await this.room.localParticipant.setMicrophoneEnabled(false);
+      this.abort.signal.throwIfAborted();
+    }
+  }
+  async switchMicrophoneDevice(
+    switchDevice: () => Promise<void>,
+  ): Promise<void> {
+    this.abort.signal.throwIfAborted();
+    try {
+      await switchDevice();
+    } finally {
+      if (this.abort.signal.aborted) {
+        // Switching can replace the track; inspect the current publication, not a render snapshot.
+        this.room.localParticipant
+          .getTrackPublication(Track.Source.Microphone)
+          ?.track?.stop();
+        await this.room.disconnect(true);
+      }
+    }
+    this.abort.signal.throwIfAborted();
+  }
+  dispose(): Promise<void> {
+    this.abort.abort();
+    this.disposal ??= (async () => {
+      await Promise.allSettled([
+        this.room.disconnect(true),
+        this.avatar?.dispose() ?? Promise.resolve(),
+      ]);
+    })();
+    return this.disposal;
+  }
+}