@livekit/agents-plugin-protoface 1.5.3

package/src/api.ts

@@ -0,0 +1,182 @@
+// SPDX-FileCopyrightText: 2026 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import {
+  type APIConnectOptions,
+  APIConnectionError,
+  APIStatusError,
+  APITimeoutError,
+  DEFAULT_API_CONNECT_OPTIONS,
+  intervalForRetry,
+} from '@livekit/agents';
+import { log } from './log.js';
+
+/** @public */
+export const DEFAULT_API_URL = 'https://api.protoface.com';
+
+const USER_AGENT = `@livekit/agents-plugin-protoface/${__PACKAGE_VERSION__}`;
+
+/** @public */
+export class ProtofaceException extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = 'ProtofaceException';
+  }
+}
+
+/** @public */
+export interface ProtofaceAPIOptions {
+  /** Protoface API key. Falls back to the `PROTOFACE_API_KEY` env var. */
+  apiKey?: string | null;
+  /** Override the Protoface API base URL. */
+  apiUrl?: string | null;
+  /** API retry/timeout options. */
+  connOptions?: APIConnectOptions;
+}
+
+/** @public */
+export interface StartSessionOptions {
+  avatarId: string;
+  transport: Record<string, unknown>;
+  maxDurationMs?: number | null;
+}
+
+/** @public */
+export type ProtofaceSession = Record<string, unknown> & { id?: string };
+
+/**
+ * Async client for the Protoface session API.
+ *
+ * @public
+ */
+export class ProtofaceAPI {
+  private apiKey: string;
+  private apiUrl: string;
+  private connOptions: APIConnectOptions;
+
+  #logger = log();
+
+  constructor(options: ProtofaceAPIOptions = {}) {
+    const apiKey = options.apiKey ?? process.env.PROTOFACE_API_KEY ?? '';
+    if (!apiKey) {
+      throw new ProtofaceException(
+        'apiKey must be set by passing it to ProtofaceAPI or setting the PROTOFACE_API_KEY environment variable',
+      );
+    }
+
+    this.apiKey = apiKey;
+    this.apiUrl = trimTrailingSlashes(
+      options.apiUrl ?? process.env.PROTOFACE_API_URL ?? DEFAULT_API_URL,
+    );
+    this.connOptions = options.connOptions ?? DEFAULT_API_CONNECT_OPTIONS;
+  }
+
+  async startSession(options: StartSessionOptions): Promise<ProtofaceSession> {
+    const body: Record<string, unknown> = {
+      avatar_id: options.avatarId,
+      transport: options.transport,
+    };
+    if (options.maxDurationMs != null) {
+      body.max_duration_seconds = options.maxDurationMs / 1000;
+    }
+
+    return (await this.json('POST', '/v1/sessions', body)) as ProtofaceSession;
+  }
+
+  async endSession(sessionId: string): Promise<Record<string, unknown>> {
+    return (await this.json('POST', `/v1/sessions/${sessionId}/end`)) as Record<string, unknown>;
+  }
+
+  private async json(
+    method: string,
+    path: string,
+    body?: Record<string, unknown>,
+  ): Promise<Record<string, unknown>> {
+    const url = `${this.apiUrl}${path}`;
+    let lastError: unknown;
+
+    for (let i = 0; i <= this.connOptions.maxRetry; i++) {
+      try {
+        const response = await fetch(url, {
+          method,
+          headers: {
+            accept: 'application/json',
+            authorization: `Bearer ${this.apiKey}`,
+            'content-type': 'application/json',
+            'user-agent': USER_AGENT,
+          },
+          body: body == null ? undefined : JSON.stringify(body),
+          signal: AbortSignal.timeout(this.connOptions.timeoutMs),
+        });
+        const payload = await readPayload(response);
+        if (response.ok) {
+          if (!isRecord(payload)) {
+            throw new APIStatusError({
+              message: 'Protoface API returned a non-object JSON response',
+              options: { statusCode: response.status, body: { payload }, retryable: false },
+            });
+          }
+          return payload;
+        }
+
+        throw new APIStatusError({
+          message: 'Protoface API returned an error',
+          options: { statusCode: response.status, body: isRecord(payload) ? payload : { payload } },
+        });
+      } catch (error) {
+        if (error instanceof APIStatusError && !error.retryable) {
+          throw error;
+        }
+        lastError = normalizeConnectionError(error);
+      }
+
+      if (i < this.connOptions.maxRetry) {
+        this.#logger.warn(
+          { attempt: i + 1, method, path },
+          'protoface api request failed, retrying',
+        );
+        await new Promise((resolve) => setTimeout(resolve, intervalForRetry(this.connOptions, i)));
+      }
+    }
+
+    throw new APIConnectionError({
+      message: 'Failed to call Protoface API after all retries.',
+      options: { body: isRecord(lastError) ? lastError : null },
+    });
+  }
+}
+
+async function readPayload(response: Response): Promise<unknown> {
+  const text = await response.text();
+  if (!text) {
+    return {};
+  }
+
+  try {
+    return JSON.parse(text) as unknown;
+  } catch {
+    return { raw: text };
+  }
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return value !== null && typeof value === 'object' && !Array.isArray(value);
+}
+
+function normalizeConnectionError(error: unknown): unknown {
+  if (error instanceof APIStatusError) {
+    return error;
+  }
+  if (error instanceof DOMException && error.name === 'TimeoutError') {
+    return new APITimeoutError({ message: 'Protoface API request timed out.' });
+  }
+  return error;
+}
+
+function trimTrailingSlashes(value: string): string {
+  let end = value.length;
+  while (end > 0 && value.charCodeAt(end - 1) === 47) {
+    end -= 1;
+  }
+  return value.slice(0, end);
+}

package/src/avatar.test.ts

@@ -0,0 +1,132 @@
+// SPDX-FileCopyrightText: 2026 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { voice } from '@livekit/agents';
+import type { Room } from '@livekit/rtc-node';
+import { TrackKind } from '@livekit/rtc-node';
+import { EventEmitter } from 'node:events';
+import { afterEach, describe, expect, it, vi } from 'vitest';
+import { ProtofaceAPI } from './api.js';
+import { AvatarSession } from './avatar.js';
+
+type DataStreamAudioOutputInternals = {
+  destinationIdentity: string;
+  sampleRate: number;
+  waitRemoteTrack?: TrackKind;
+};
+
+function fakeRoom(): Room {
+  const emitter = new EventEmitter();
+  const remoteParticipant = {
+    identity: 'protoface-avatar-agent',
+    trackPublications: new Map([['video', { kind: TrackKind.KIND_VIDEO }]]),
+  };
+
+  return {
+    name: 'test-room',
+    isConnected: true,
+    localParticipant: {
+      identity: 'local-agent',
+      registerRpcMethod: vi.fn(),
+    },
+    remoteParticipants: new Map([[remoteParticipant.identity, remoteParticipant]]),
+    on: vi.fn((event: string | symbol, listener: (...args: unknown[]) => void) => {
+      emitter.on(event, listener);
+      return emitter;
+    }),
+    off: vi.fn((event: string | symbol, listener: (...args: unknown[]) => void) => {
+      emitter.off(event, listener);
+      return emitter;
+    }),
+  } as unknown as Room;
+}
+
+function fakeAgentSession(): voice.AgentSession {
+  const emitter = new EventEmitter();
+  return {
+    _started: true,
+    output: { audio: null as voice.AudioOutput | null },
+    on: vi.fn((event: string | symbol, listener: (...args: unknown[]) => void) => {
+      emitter.on(event, listener);
+      return emitter;
+    }),
+    off: vi.fn((event: string | symbol, listener: (...args: unknown[]) => void) => {
+      emitter.off(event, listener);
+      return emitter;
+    }),
+  } as unknown as voice.AgentSession;
+}
+
+describe('Protoface AvatarSession', () => {
+  afterEach(() => {
+    vi.restoreAllMocks();
+    voice.DataStreamAudioOutput._playbackFinishedRpcRegistered = false;
+    voice.DataStreamAudioOutput._playbackFinishedHandlers = {};
+    voice.DataStreamAudioOutput._playbackStartedRpcRegistered = false;
+    voice.DataStreamAudioOutput._playbackStartedHandlers = {};
+  });
+
+  it('calls base AvatarSession.start first', async () => {
+    const sentinel = new Error('super-start-called');
+    const superStartSpy = vi
+      .spyOn(voice.AvatarSession.prototype, 'start')
+      .mockRejectedValue(sentinel);
+
+    const avatar = new AvatarSession({ apiKey: 'protoface-key' });
+
+    await expect(avatar.start(fakeAgentSession(), fakeRoom())).rejects.toThrow(
+      'super-start-called',
+    );
+    expect(superStartSpy).toHaveBeenCalledTimes(1);
+  });
+
+  it('starts a Protoface session and routes agent audio to the avatar', async () => {
+    vi.spyOn(voice.AvatarSession.prototype, 'start').mockResolvedValue(undefined);
+    vi.spyOn(voice.AvatarSession.prototype, 'aclose').mockResolvedValue(undefined);
+    const startSessionSpy = vi
+      .spyOn(ProtofaceAPI.prototype, 'startSession')
+      .mockResolvedValue({ id: 'protoface-session-1' });
+    const endSessionSpy = vi.spyOn(ProtofaceAPI.prototype, 'endSession').mockResolvedValue({});
+
+    const avatar = new AvatarSession({
+      apiKey: 'protoface-key',
+      avatarId: 'avatar-1',
+      maxDurationMs: 120_000,
+    });
+    const agentSession = fakeAgentSession();
+
+    await avatar.start(agentSession, fakeRoom(), {
+      livekitUrl: 'wss://livekit.example.com',
+      livekitApiKey: 'livekit-api-key',
+      livekitApiSecret: 'livekit-api-secret',
+    });
+
+    expect(avatar.sessionId).toBe('protoface-session-1');
+    expect(startSessionSpy).toHaveBeenCalledWith({
+      avatarId: 'avatar-1',
+      transport: expect.objectContaining({
+        type: 'livekit',
+        url: 'wss://livekit.example.com',
+        room_name: 'test-room',
+        worker_identity: 'protoface-avatar-agent',
+        audio_source: 'data_stream',
+        worker_token: expect.any(String),
+      }),
+      maxDurationMs: 120_000,
+    });
+
+    const audioOutput = agentSession.output.audio;
+    expect(audioOutput).toBeInstanceOf(voice.DataStreamAudioOutput);
+    expect((audioOutput as unknown as DataStreamAudioOutputInternals).destinationIdentity).toBe(
+      'protoface-avatar-agent',
+    );
+    expect((audioOutput as unknown as DataStreamAudioOutputInternals).sampleRate).toBe(16000);
+    expect((audioOutput as unknown as DataStreamAudioOutputInternals).waitRemoteTrack).toBe(
+      TrackKind.KIND_VIDEO,
+    );
+
+    await avatar.aclose();
+    expect(endSessionSpy).toHaveBeenCalledWith('protoface-session-1');
+    expect(avatar.sessionId).toBeNull();
+  });
+});

package/src/avatar.ts

@@ -0,0 +1,204 @@
+// SPDX-FileCopyrightText: 2026 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import {
+  type APIConnectOptions,
+  DEFAULT_API_CONNECT_OPTIONS,
+  getJobContext,
+  voice,
+} from '@livekit/agents';
+import type { Room } from '@livekit/rtc-node';
+import { TrackKind } from '@livekit/rtc-node';
+import type { VideoGrant } from 'livekit-server-sdk';
+import { AccessToken } from 'livekit-server-sdk';
+import { ProtofaceAPI, ProtofaceException } from './api.js';
+import { log } from './log.js';
+
+/** @public */
+export const DEFAULT_STOCK_AVATAR_ID = 'av_stock_001';
+
+const ATTRIBUTE_PUBLISH_ON_BEHALF = 'lk.publish_on_behalf';
+const SAMPLE_RATE = 16000;
+const AVATAR_AGENT_IDENTITY = 'protoface-avatar-agent';
+const AVATAR_AGENT_NAME = 'protoface-avatar-agent';
+
+/** @public */
+export interface AvatarSessionOptions {
+  /** Protoface avatar ID to render. Defaults to the stable stock avatar ID `av_stock_001`. */
+  avatarId?: string;
+  /** Override the Protoface API base URL. */
+  apiUrl?: string | null;
+  /** Protoface API key. Falls back to the `PROTOFACE_API_KEY` env var. */
+  apiKey?: string | null;
+  /** Optional maximum session duration in milliseconds. */
+  maxDurationMs?: number | null;
+  /** Identity for the avatar participant. Defaults to `protoface-avatar-agent`. */
+  avatarParticipantIdentity?: string | null;
+  /** Display name for the avatar participant. Defaults to `protoface-avatar-agent`. */
+  avatarParticipantName?: string | null;
+  /** API retry/timeout options. */
+  connOptions?: APIConnectOptions;
+}
+
+/**
+ * Optional LiveKit credentials for {@link AvatarSession.start}; falls back to env vars.
+ *
+ * @public
+ */
+export interface StartOptions {
+  livekitUrl?: string | null;
+  livekitApiKey?: string | null;
+  livekitApiSecret?: string | null;
+}
+
+/**
+ * A Protoface avatar session for LiveKit Agents.
+ *
+ * @public
+ */
+export class AvatarSession extends voice.AvatarSession {
+  private avatarId: string;
+  private maxDurationMs?: number | null;
+  private avatarParticipantIdentity: string;
+  private avatarParticipantName: string;
+  private api: ProtofaceAPI;
+  private sessionIdValue: string | null = null;
+
+  #logger = log();
+
+  constructor(options: AvatarSessionOptions = {}) {
+    super();
+    this.avatarId = options.avatarId ?? DEFAULT_STOCK_AVATAR_ID;
+    this.maxDurationMs = options.maxDurationMs;
+    this.avatarParticipantIdentity = options.avatarParticipantIdentity || AVATAR_AGENT_IDENTITY;
+    this.avatarParticipantName = options.avatarParticipantName || AVATAR_AGENT_NAME;
+    this.api = new ProtofaceAPI({
+      apiKey: options.apiKey,
+      apiUrl: options.apiUrl,
+      connOptions: options.connOptions ?? DEFAULT_API_CONNECT_OPTIONS,
+    });
+  }
+
+  override get avatarIdentity(): string {
+    return this.avatarParticipantIdentity;
+  }
+
+  override get provider(): string {
+    return 'protoface';
+  }
+
+  /** Protoface session ID after `start()` succeeds, otherwise `null`. */
+  get sessionId(): string | null {
+    return this.sessionIdValue;
+  }
+
+  async start(
+    agentSession: voice.AgentSession,
+    room: Room,
+    options: StartOptions = {},
+  ): Promise<void> {
+    if (this.sessionIdValue !== null) {
+      throw new Error('AvatarSession.start() called twice; create a new AvatarSession.');
+    }
+
+    await super.start(agentSession, room);
+
+    const livekitUrl = options.livekitUrl ?? process.env.LIVEKIT_URL;
+    const livekitApiKey = options.livekitApiKey ?? process.env.LIVEKIT_API_KEY;
+    const livekitApiSecret = options.livekitApiSecret ?? process.env.LIVEKIT_API_SECRET;
+    if (!livekitUrl || !livekitApiKey || !livekitApiSecret) {
+      throw new ProtofaceException(
+        'livekitUrl, livekitApiKey, and livekitApiSecret must be set by arguments or environment variables',
+      );
+    }
+
+    const workerToken = await this.mintWorkerToken({ room, livekitApiKey, livekitApiSecret });
+    const session = await this.api.startSession({
+      avatarId: this.avatarId,
+      transport: {
+        type: 'livekit',
+        url: livekitUrl,
+        room_name: room.name,
+        worker_token: workerToken,
+        worker_identity: this.avatarParticipantIdentity,
+        audio_source: 'data_stream',
+      },
+      maxDurationMs: this.maxDurationMs,
+    });
+
+    if (typeof session.id !== 'string') {
+      throw new ProtofaceException('Protoface API response missing session id');
+    }
+
+    this.sessionIdValue = session.id;
+    this.#logger.debug(
+      { sessionId: this.sessionIdValue, avatarId: this.avatarId },
+      'protoface session started',
+    );
+
+    agentSession.output.audio = new voice.DataStreamAudioOutput({
+      room,
+      destinationIdentity: this.avatarParticipantIdentity,
+      sampleRate: SAMPLE_RATE,
+      waitRemoteTrack: TrackKind.KIND_VIDEO,
+    });
+  }
+
+  async aclose(): Promise<void> {
+    const sessionId = this.sessionIdValue;
+    this.sessionIdValue = null;
+    try {
+      if (sessionId !== null) {
+        try {
+          await this.api.endSession(sessionId);
+        } catch (error) {
+          this.#logger.warn({ error: String(error), sessionId }, 'failed to end protoface session');
+        }
+      }
+    } finally {
+      await super.aclose();
+    }
+  }
+
+  private async mintWorkerToken({
+    room,
+    livekitApiKey,
+    livekitApiSecret,
+  }: {
+    room: Room;
+    livekitApiKey: string;
+    livekitApiSecret: string;
+  }): Promise<string> {
+    let localParticipantIdentity = '';
+    try {
+      const jobCtx = getJobContext();
+      localParticipantIdentity = jobCtx.agent?.identity || '';
+    } catch {
+      // Fall back to the connected room below when no job context is available.
+    }
+
+    if (!localParticipantIdentity && room.isConnected && room.localParticipant) {
+      localParticipantIdentity = room.localParticipant.identity;
+    }
+
+    if (!localParticipantIdentity) {
+      throw new ProtofaceException('failed to get local participant identity');
+    }
+
+    const token = new AccessToken(livekitApiKey, livekitApiSecret, {
+      identity: this.avatarParticipantIdentity,
+      name: this.avatarParticipantName,
+    });
+    token.kind = 'agent';
+    token.attributes = { [ATTRIBUTE_PUBLISH_ON_BEHALF]: localParticipantIdentity };
+    token.addGrant({
+      roomJoin: true,
+      room: room.name,
+      canPublish: true,
+      canSubscribe: true,
+      canPublishData: true,
+    } as VideoGrant);
+
+    return token.toJwt();
+  }
+}

package/src/index.ts

@@ -0,0 +1,19 @@
+// SPDX-FileCopyrightText: 2026 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { Plugin } from '@livekit/agents';
+
+export * from './api.js';
+export * from './avatar.js';
+
+class ProtofacePlugin extends Plugin {
+  constructor() {
+    super({
+      title: 'protoface',
+      version: __PACKAGE_VERSION__,
+      package: __PACKAGE_NAME__,
+    });
+  }
+}
+
+Plugin.registerPlugin(new ProtofacePlugin());

package/src/log.ts

@@ -0,0 +1,7 @@
+// SPDX-FileCopyrightText: 2026 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { log as agentsLog } from '@livekit/agents';
+import type { Logger } from 'pino';
+
+export const log = (): Logger => agentsLog().child({ plugin: 'protoface' });