create-spatius-app 0.1.2 → 0.1.3-beta.1

package/templates/layers/agora/tsconfig.json

@@ -0,0 +1,16 @@
+{
+  "compilerOptions": {
+    "target": "ES2023",
+    "lib": ["ES2023", "DOM", "DOM.Iterable"],
+    "module": "ESNext",
+    "moduleResolution": "Bundler",
+    "jsx": "react-jsx",
+    "strict": true,
+    "noEmit": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "types": ["node", "vite/client"],
+    "resolveJsonModule": true
+  },
+  "include": ["web", "worker", "server", "vite.config.ts", "vitest.config.ts"]
+}

package/templates/layers/agora/vitest.config.ts

@@ -0,0 +1,4 @@
+import { defineConfig } from 'vitest/config';
+export default defineConfig({
+  test: { include: ['worker/**/*.test.ts', 'web/**/*.test.ts'] },
+});

package/templates/layers/agora/web/App.tsx

@@ -0,0 +1,174 @@
+import { useEffect, useRef, useState } from 'react';
+import { AgoraSession } from './agora-session.js';
+import {
+  AgentChatTranscript,
+  type TranscriptMessage,
+} from './components/transcript-view.js';
+import { AgentChatInput } from './components/agents-ui/agent-chat-input.js';
+import { TranscriptPanel } from './components/transcript-panel.js';
+import { Icon } from './components/icons.js';
+export default function App() {
+  const stage = useRef<HTMLDivElement>(null);
+  const session = useRef<AgoraSession | undefined>(undefined);
+  const mounted = useRef(true);
+  const [phase, setPhase] = useState<
+    'idle' | 'starting' | 'active' | 'stopping' | 'ended'
+  >('idle');
+  const [error, setError] = useState('');
+  const [messages, setMessages] = useState<TranscriptMessage[]>([]);
+  const [microphone, setMicrophone] = useState(false);
+  const [pending, setPending] = useState(false);
+  useEffect(() => {
+    mounted.current = true;
+    return () => {
+      mounted.current = false;
+      void session.current?.dispose();
+    };
+  }, []);
+  const stop = async () => {
+    const current = session.current;
+    if (!current) return;
+    session.current = undefined;
+    setPhase('stopping');
+    setMicrophone(false);
+    await current.dispose();
+    if (mounted.current) setPhase('ended');
+  };
+  const start = async () => {
+    if (!stage.current || session.current) return;
+    setError('');
+    setMessages([]);
+    setPhase('starting');
+    const current = new AgoraSession(
+      (items) => {
+        if (mounted.current && session.current === current) setMessages(items);
+      },
+      (message) => {
+        if (mounted.current) setError(message);
+      },
+      () => {
+        if (session.current === current) void stop();
+      },
+    );
+    session.current = current;
+    try {
+      await current.start(stage.current);
+      if (!mounted.current || session.current !== current) return;
+      setPhase('active');
+      try {
+        await current.setMicrophone(true);
+        if (session.current === current && mounted.current) setMicrophone(true);
+      } catch {
+        if (session.current === current)
+          setError(
+            'Microphone unavailable. Allow access or continue by typing.',
+          );
+      }
+    } catch (failure) {
+      if (!mounted.current || session.current !== current) return;
+      setError(
+        failure instanceof Error ? failure.message : 'Could not connect.',
+      );
+      await stop();
+    }
+  };
+  return (
+    <main className="avatar-app" data-phase={phase}>
+      <div ref={stage} className="avatar-canvas" aria-label="Avatar stage" />
+      <header className="session-header">
+        <a className="brand-pill glass" href="/">
+          <Icon name="chat" />
+          Spatius starter
+        </a>
+      </header>
+      {phase !== 'active' ? (
+        <section className="welcome-state">
+          <h1>
+            {phase === 'starting'
+              ? 'Starting conversation'
+              : phase === 'stopping'
+                ? 'Ending conversation'
+                : phase === 'ended'
+                  ? 'Conversation ended'
+                  : 'Voice assistant'}
+          </h1>
+          <p className="welcome-description">
+            Talk to your avatar or send a message. Follow along in the
+            transcript.
+          </p>
+          <button
+            className="primary-button"
+            disabled={phase === 'stopping'}
+            onClick={() => void (phase === 'starting' ? stop() : start())}
+          >
+            {phase === 'starting'
+              ? 'Cancel'
+              : phase === 'ended'
+                ? 'Start a new conversation'
+                : 'Start conversation'}
+          </button>
+        </section>
+      ) : (
+        <div className="conversation-tools">
+          <div className="call-dock">
+            <button
+              className="primary-button"
+              aria-pressed={microphone}
+              disabled={pending}
+              onClick={() => {
+                const current = session.current;
+                setPending(true);
+                void current
+                  ?.setMicrophone(!microphone)
+                  .then(() => {
+                    if (session.current === current) setMicrophone(!microphone);
+                  })
+                  .catch(() =>
+                    setError(
+                      'Could not change the microphone. Try again or keep typing.',
+                    ),
+                  )
+                  .finally(() => setPending(false));
+              }}
+            >
+              {microphone ? 'Mute microphone' : 'Enable microphone'}
+            </button>
+            <button className="primary-button" onClick={() => void stop()}>
+              End call
+            </button>
+          </div>
+        </div>
+      )}
+      {error && (
+        <p
+          className="control-error"
+          role="alert"
+          style={{
+            position: 'absolute',
+            bottom: '12rem',
+            left: '1.5rem',
+            maxWidth: '28rem',
+          }}
+        >
+          {error}
+        </p>
+      )}
+      <footer className="stage-footer">
+        Powered by Spatius <span>×</span> Agora
+      </footer>
+      <TranscriptPanel>
+        <AgentChatTranscript messages={messages} agentName="Your agent" />
+        <div className="panel-composer">
+          <AgentChatInput
+            disabled={phase !== 'active'}
+            onSend={(text) => {
+              if (!session.current)
+                return Promise.reject(new Error('Not connected'));
+              return session.current.send(text);
+            }}
+          />
+        </div>
+      </TranscriptPanel>
+    </main>
+  );
+}

package/templates/layers/agora/web/agora-session.test.ts

@@ -0,0 +1,70 @@
+// @vitest-environment jsdom
+import { afterEach, expect, it, vi } from 'vitest';
+import { AgoraSession } from './agora-session.js';
+afterEach(() => vi.unstubAllGlobals());
+it('stops a remotely created agent when cancellation precedes the bootstrap response', async () => {
+  let complete!: (response: Response) => void;
+  const fetcher = vi
+    .fn<typeof fetch>()
+    .mockImplementationOnce(
+      () =>
+        new Promise((resolve) => {
+          complete = resolve;
+        }),
+    )
+    .mockResolvedValue(new Response(null, { status: 204 }));
+  vi.stubGlobal('fetch', fetcher);
+  const session = new AgoraSession(vi.fn(), vi.fn());
+  const starting = session.start(document.createElement('div'));
+  const rejected = expect(starting).rejects.toMatchObject({
+    name: 'AbortError',
+  });
+  const stopping = session.dispose();
+  complete(
+    Response.json({
+      capability: 'signed-session',
+      appId: 'app',
+      channel: 'channel',
+      uid: 123,
+      agentUid: 456,
+      token: 'client-token',
+      spatiusAppId: 'spatius',
+      avatarId: 'avatar',
+      region: 'region',
+    }),
+  );
+  await rejected;
+  await stopping;
+  await session.dispose();
+  expect(fetcher).toHaveBeenCalledTimes(2);
+  expect(fetcher.mock.calls[1]).toEqual([
+    '/api/session/stop',
+    expect.objectContaining({
+      body: JSON.stringify({ capability: 'signed-session' }),
+      keepalive: true,
+    }),
+  ]);
+});
+it('rejects malformed bootstrap responses without initializing media', async () => {
+  vi.stubGlobal('fetch', vi.fn().mockResolvedValue(Response.json({})));
+  const session = new AgoraSession(vi.fn(), vi.fn());
+  await expect(session.start(document.createElement('div'))).rejects.toThrow(
+    'Invalid session response',
+  );
+  await session.dispose();
+});
+it('keeps provider errors out of the browser error message', async () => {
+  vi.stubGlobal(
+    'fetch',
+    vi
+      .fn()
+      .mockResolvedValue(
+        new Response('private provider diagnostic', { status: 502 }),
+      ),
+  );
+  const session = new AgoraSession(vi.fn(), vi.fn());
+  await expect(session.start(document.createElement('div'))).rejects.toThrow(
+    'Could not start the agent.',
+  );
+  await expect(session.send('hello')).rejects.toThrow('Not connected');
+});

package/templates/layers/agora/web/agora-session.ts

@@ -0,0 +1,267 @@
+import type { IAgoraRTCClient } from 'agora-rtc-sdk-ng';
+import type { AvatarPlayer } from '@spatius/avatarkit-rtc';
+import type { AvatarView } from '@spatius/avatarkit';
+import type { AgoraVoiceAI } from 'agora-agent-client-toolkit';
+import type { RTMClient } from 'agora-rtm-sdk';
+import type { TranscriptMessage } from './components/transcript-view.js';
+interface Credentials {
+  capability: string;
+  appId: string;
+  channel: string;
+  uid: number;
+  agentUid: number;
+  token: string;
+  spatiusAppId: string;
+  avatarId: string;
+  region: string;
+}
+let initialization: Promise<void> | undefined;
+export class AgoraSession {
+  private credentials?: Credentials;
+  private player?: AvatarPlayer;
+  private view?: AvatarView;
+  private rtm?: RTMClient;
+  private toolkit?: AgoraVoiceAI;
+  private microphone?: MediaStream;
+  private cancelled = false;
+  private starting?: Promise<void>;
+  private microphoneBusy = false;
+  private readonly pagehide = () => {
+    if (this.credentials)
+      navigator.sendBeacon(
+        '/api/session/stop',
+        new Blob(
+          [JSON.stringify({ capability: this.credentials.capability })],
+          { type: 'application/json' },
+        ),
+      );
+  };
+  constructor(
+    private readonly onMessages: (messages: TranscriptMessage[]) => void,
+    private readonly onError: (message: string) => void,
+    private readonly onDisconnected: () => void = () => undefined,
+  ) {}
+  start(container: HTMLElement): Promise<void> {
+    this.starting ??= this.prepare(container).catch(async (error: unknown) => {
+      await this.cleanup();
+      throw error;
+    });
+    return this.starting;
+  }
+  private check() {
+    if (this.cancelled) throw new DOMException('Cancelled', 'AbortError');
+  }
+  private async prepare(container: HTMLElement) {
+    // Keep the bootstrap response observable after cancellation so a started remote agent can be stopped.
+    const response = await fetch('/api/session', {
+      method: 'POST',
+      cache: 'no-store',
+      signal: AbortSignal.timeout(20000),
+    });
+    if (!response.ok)
+      throw new Error(
+        'Could not start the agent. Check the Agora pipeline and Spatius configuration.',
+      );
+    const raw: unknown = await response.json();
+    if (!raw || typeof raw !== 'object')
+      throw new Error('Invalid session response');
+    const value = raw as Credentials;
+    if (
+      ![
+        'capability',
+        'appId',
+        'channel',
+        'token',
+        'spatiusAppId',
+        'avatarId',
+        'region',
+      ].every(
+        (key) => typeof (raw as Record<string, unknown>)[key] === 'string',
+      ) ||
+      !Number.isInteger(value.uid) ||
+      !Number.isInteger(value.agentUid)
+    )
+      throw new Error('Invalid session response');
+    this.credentials = value;
+    window.addEventListener('pagehide', this.pagehide);
+    this.check();
+    const [
+      { AvatarSDK, AvatarManager, AvatarView, DrivingServiceMode },
+      { AvatarPlayer, AgoraProvider },
+      { default: AgoraRTM },
+      toolkit,
+    ] = await Promise.all([
+      import('@spatius/avatarkit'),
+      import('@spatius/avatarkit-rtc'),
+      import('agora-rtm-sdk'),
+      import('agora-agent-client-toolkit'),
+    ]);
+    this.check();
+    initialization ??= AvatarSDK.initialize(value.spatiusAppId, {
+      drivingServiceMode: DrivingServiceMode.rtc,
+      region: value.region,
+    }).catch((error: unknown) => {
+      initialization = undefined;
+      throw error;
+    });
+    await initialization;
+    this.check();
+    const avatar =
+      AvatarManager.shared.retrieve(value.avatarId) ??
+      (await AvatarManager.shared.load(value.avatarId));
+    this.check();
+    if (!avatar) throw new Error('Avatar could not be loaded');
+    this.view = new AvatarView(avatar, container);
+    this.player = new AvatarPlayer(new AgoraProvider(), this.view, {
+      logLevel: 'error',
+    });
+    await this.player.connect({
+      appId: value.appId,
+      channel: value.channel,
+      token: value.token,
+      uid: value.uid,
+    });
+    this.check();
+    const client = this.player.getNativeClient() as IAgoraRTCClient;
+    client.on('connection-state-change', (state) => {
+      if (state === 'DISCONNECTED' && !this.cancelled) this.onDisconnected();
+    });
+    this.rtm = new AgoraRTM.RTM(value.appId, String(value.uid));
+    await this.rtm.login({ token: value.token });
+    this.check();
+    this.toolkit = await toolkit.AgoraVoiceAI.init({
+      rtcEngine: client,
+      rtmEngine: this.rtm,
+      enableLog: false,
+    });
+    this.toolkit.on(toolkit.AgoraVoiceAIEvents.TRANSCRIPT_UPDATED, (items) => {
+      if (this.cancelled) return;
+      this.onMessages(
+        items.map((item) => ({
+          id: `${item.uid}:${item.turn_id}:${item.stream_id}`,
+          timestamp: item._time || Date.now(),
+          message: item.text,
+          isUser: item.uid === String(value.uid),
+        })),
+      );
+    });
+    this.toolkit.on(toolkit.AgoraVoiceAIEvents.MESSAGE_ERROR, () =>
+      this.onError(
+        'The agent could not process the message. Please try again.',
+      ),
+    );
+    this.toolkit.subscribeMessage(value.channel);
+    await this.rtm.subscribe(value.channel);
+    this.check();
+    await new Promise<void>((resolve, reject) => {
+      const ready = () =>
+        client.remoteUsers.some(
+          (user) => String(user.uid) === String(value.agentUid),
+        );
+      if (ready()) {
+        resolve();
+        return;
+      }
+      const timeout = window.setTimeout(() => {
+        client.off('user-joined', joined);
+        reject(new Error('The agent did not join in time.'));
+      }, 30000);
+      function joined() {
+        if (ready()) {
+          window.clearTimeout(timeout);
+          client.off('user-joined', joined);
+          resolve();
+        }
+      }
+      client.on('user-joined', joined);
+    });
+    this.check();
+  }
+  async setMicrophone(enabled: boolean) {
+    this.check();
+    if (!this.player || this.microphoneBusy) return;
+    this.microphoneBusy = true;
+    try {
+      if (!enabled) {
+        await this.player.unpublishAudio();
+        this.microphone?.getTracks().forEach((track) => track.stop());
+        this.microphone = undefined;
+        return;
+      }
+      const stream = await navigator.mediaDevices.getUserMedia({
+        audio: { echoCancellation: true, noiseSuppression: true },
+      });
+      if (this.cancelled) {
+        stream.getTracks().forEach((track) => track.stop());
+        this.check();
+      }
+      this.microphone = stream;
+      await this.player.publishAudio(stream.getAudioTracks()[0]);
+      if (this.cancelled) {
+        stream.getTracks().forEach((track) => track.stop());
+        await this.player.unpublishAudio();
+      }
+    } finally {
+      this.microphoneBusy = false;
+    }
+  }
+  async send(text: string) {
+    this.check();
+    if (!this.toolkit || !this.credentials) throw new Error('Not connected');
+    const { ChatMessagePriority, ChatMessageType } =
+      await import('agora-agent-client-toolkit');
+    await this.toolkit.sendText(String(this.credentials.agentUid), {
+      messageType: ChatMessageType.TEXT,
+      text,
+      priority: ChatMessagePriority.INTERRUPTED,
+      responseInterruptable: true,
+    });
+  }
+  async dispose() {
+    this.cancelled = true;
+    await this.starting?.catch(() => undefined);
+    await this.cleanup();
+  }
+  private async cleanup() {
+    window.removeEventListener('pagehide', this.pagehide);
+    this.microphone?.getTracks().forEach((track) => track.stop());
+    this.microphone = undefined;
+    const toolkit = this.toolkit,
+      view = this.view,
+      player = this.player,
+      rtm = this.rtm,
+      credentials = this.credentials;
+    this.toolkit = undefined;
+    this.view = undefined;
+    this.player = undefined;
+    this.rtm = undefined;
+    this.credentials = undefined;
+    const results = await Promise.allSettled([
+      Promise.resolve().then(() => toolkit?.destroy()),
+      Promise.resolve().then(() => player?.disconnect()),
+      Promise.resolve().then(() => rtm?.logout()),
+      credentials
+        ? fetch('/api/session/stop', {
+            method: 'POST',
+            headers: { 'content-type': 'application/json' },
+            body: JSON.stringify({ capability: credentials.capability }),
+            keepalive: true,
+            signal: AbortSignal.timeout(15000),
+          }).then((response) => {
+            if (!response.ok) throw new Error('Remote cleanup failed');
+          })
+        : undefined,
+    ]);
+    try {
+      view?.dispose();
+    } catch {
+      this.onError(
+        'The avatar renderer could not be fully released. Reload before reconnecting.',
+      );
+    }
+    if (results.some((result) => result.status === 'rejected'))
+      this.onError(
+        'Session ended locally. Remote cleanup could not be confirmed; the agent idle timeout is the fallback.',
+      );
+  }
+}

package/templates/layers/agora/worker/agora.test.ts

@@ -0,0 +1,60 @@
+import { describe, expect, it, vi } from 'vitest';
+import {
+  createAgoraSession,
+  stopAgoraSession,
+  type AgoraEnvironment,
+} from './agora.js';
+const env: AgoraEnvironment = {
+  AGORA_APP_ID: 'a'.repeat(32),
+  AGORA_APP_CERTIFICATE: 'b'.repeat(32),
+  AGORA_PIPELINE_ID: 'test-pipeline',
+  SPATIUS_API_KEY: 'private-key',
+  SPATIUS_APP_ID: 'test-app',
+  SPATIUS_AVATAR_ID: 'test-avatar',
+};
+describe('hosted Agora sessions', () => {
+  it('keeps secrets server-side and authorizes idempotent cleanup', async () => {
+    const fetcher = vi
+      .fn<typeof fetch>()
+      .mockResolvedValueOnce(Response.json({ agent_id: 'hosted-agent' }))
+      .mockResolvedValueOnce(new Response(null, { status: 404 }));
+    const session = await createAgoraSession(env, fetcher);
+    expect(session.uid).not.toBe(session.agentUid);
+    expect(JSON.stringify(session)).not.toContain(env.AGORA_APP_CERTIFICATE);
+    expect(JSON.stringify(session)).not.toContain(env.SPATIUS_API_KEY);
+    const body = JSON.parse(fetcher.mock.calls[0][1]!.body as string) as {
+      pipeline_id: string;
+      properties: {
+        agent_rtc_uid: string;
+        remote_rtc_uids: string[];
+        avatar: { params: { agora_uid: string; sample_rate: number } };
+      };
+    };
+    expect(body.pipeline_id).toBe('test-pipeline');
+    expect(body.properties.remote_rtc_uids).toEqual([String(session.uid)]);
+    expect(body.properties.agent_rtc_uid).toBe(String(session.agentUid));
+    expect(body.properties.avatar.params.agora_uid).not.toBe(
+      String(session.uid),
+    );
+    expect(body.properties.avatar.params.sample_rate).toBe(24000);
+    await stopAgoraSession(env, session.capability, fetcher);
+    expect(fetcher.mock.calls[1][0]).toContain('/agents/hosted-agent/leave');
+    await expect(
+      stopAgoraSession(env, session.capability + 'bad', fetcher),
+    ).rejects.toThrow();
+    expect(fetcher).toHaveBeenCalledTimes(2);
+  });
+  it('validates configuration before starting a billable agent and redacts upstream errors', async () => {
+    const fetcher = vi.fn<typeof fetch>();
+    await expect(
+      createAgoraSession({ ...env, AGORA_AVATAR_SAMPLE_RATE: '123' }, fetcher),
+    ).rejects.toThrow('sample rate');
+    expect(fetcher).not.toHaveBeenCalled();
+    fetcher.mockResolvedValue(
+      Response.json({ error: env.SPATIUS_API_KEY }, { status: 400 }),
+    );
+    await expect(createAgoraSession(env, fetcher)).rejects.toThrow(
+      'Agora request failed (400)',
+    );
+  });
+});