felona-voice 0.1.0

package/src/stt/deepgram.ts

@@ -0,0 +1,182 @@
+import { WebSocket } from "ws";
+import { EventEmitter } from "node:events";
+import type {
+  STTProvider,
+  STTStream,
+  STTStreamOptions,
+  STTResult,
+  AudioChunk,
+} from "../types.js";
+
+/**
+ * DeepgramSTT — Streaming Speech-to-Text using Deepgram's WebSocket API.
+ *
+ * Deepgram is the default STT provider because:
+ * - Sub-300ms latency for streaming recognition
+ * - Excellent accuracy with Nova-2 model
+ * - Native WebSocket API (no HTTP polling)
+ * - Supports interim results for real-time feedback
+ */
+export class DeepgramSTT implements STTProvider {
+  readonly name = "deepgram";
+  private readonly apiKey: string;
+  private readonly model: string;
+  private readonly baseUrl: string;
+
+  constructor(options: {
+    apiKey: string;
+    model?: string;
+    baseUrl?: string;
+  }) {
+    this.apiKey = options.apiKey;
+    this.model = options.model ?? "nova-2";
+    this.baseUrl =
+      options.baseUrl ?? "wss://api.deepgram.com/v1/listen";
+  }
+
+  createStream(options?: STTStreamOptions): STTStream {
+    return new DeepgramSTTStream({
+      apiKey: this.apiKey,
+      model: this.model,
+      baseUrl: this.baseUrl,
+      language: options?.language ?? "en-US",
+      interimResults: options?.interimResults ?? true,
+      keywords: options?.keywords,
+    });
+  }
+}
+
+class DeepgramSTTStream extends EventEmitter implements STTStream {
+  private ws: WebSocket | null = null;
+  private resultHandler: ((result: STTResult) => void) | null = null;
+  private readonly config: {
+    apiKey: string;
+    model: string;
+    baseUrl: string;
+    language: string;
+    interimResults: boolean;
+    keywords?: string[];
+  };
+  private connected = false;
+  private connectPromise: Promise<void> | null = null;
+
+  constructor(config: {
+    apiKey: string;
+    model: string;
+    baseUrl: string;
+    language: string;
+    interimResults: boolean;
+    keywords?: string[];
+  }) {
+    super();
+    this.config = config;
+    this.connectPromise = this.connect();
+  }
+
+  private async connect(): Promise<void> {
+    const params = new URLSearchParams({
+      model: this.config.model,
+      language: this.config.language,
+      interim_results: String(this.config.interimResults),
+      punctuate: "true",
+      encoding: "linear16",
+      sample_rate: "16000",
+      channels: "1",
+    });
+
+    if (this.config.keywords?.length) {
+      for (const kw of this.config.keywords) {
+        params.append("keywords", kw);
+      }
+    }
+
+    const url = `${this.config.baseUrl}?${params.toString()}`;
+
+    return new Promise<void>((resolve, reject) => {
+      this.ws = new WebSocket(url, {
+        headers: {
+          Authorization: `Token ${this.config.apiKey}`,
+        },
+      });
+
+      this.ws.on("open", () => {
+        this.connected = true;
+        resolve();
+      });
+
+      this.ws.on("message", (data) => {
+        try {
+          const response = JSON.parse(data.toString());
+          this.handleResponse(response);
+        } catch {
+          // Ignore non-JSON messages
+        }
+      });
+
+      this.ws.on("error", (error) => {
+        console.error("[STT/Deepgram] WebSocket error:", error.message);
+        if (!this.connected) {
+          reject(error);
+        }
+      });
+
+      this.ws.on("close", () => {
+        this.connected = false;
+      });
+    });
+  }
+
+  write(chunk: AudioChunk): void {
+    if (this.ws && this.connected && this.ws.readyState === WebSocket.OPEN) {
+      this.ws.send(chunk.data);
+    }
+  }
+
+  onResult(handler: (result: STTResult) => void): void {
+    this.resultHandler = handler;
+  }
+
+  async close(): Promise<void> {
+    if (this.ws) {
+      // Send close message per Deepgram protocol
+      if (this.ws.readyState === WebSocket.OPEN) {
+        this.ws.send(JSON.stringify({ type: "CloseStream" }));
+      }
+      this.ws.close();
+      this.ws = null;
+    }
+    this.connected = false;
+  }
+
+  private handleResponse(response: Record<string, unknown>): void {
+    // Deepgram response format
+    if (response.type === "Results") {
+      const channel = (response as any).channel;
+      if (!channel?.alternatives?.length) return;
+
+      const alt = channel.alternatives[0];
+      const result: STTResult = {
+        text: alt.transcript ?? "",
+        isFinal: (response as any).is_final ?? false,
+        confidence: alt.confidence ?? 0,
+        words: alt.words?.map((w: any) => ({
+          word: w.word,
+          startMs: Math.round(w.start * 1000),
+          endMs: Math.round(w.end * 1000),
+        })),
+      };
+
+      // Only emit non-empty results
+      if (result.text.trim()) {
+        this.resultHandler?.(result);
+      }
+    }
+  }
+}
+
+export function createDeepgramSTT(options: {
+  apiKey: string;
+  model?: string;
+}): DeepgramSTT {
+  return new DeepgramSTT(options);
+}

package/src/stt/index.ts

@@ -0,0 +1 @@
+export { DeepgramSTT, createDeepgramSTT } from "./deepgram.js";

package/src/tools/index.ts

@@ -0,0 +1 @@
+export { ToolRegistry, defineTool, createToolRegistry } from "./registry.js";

package/src/tools/registry.ts

@@ -0,0 +1,99 @@
+import type { AgentTool, ToolExecutor } from "../types.js";
+
+/**
+ * ToolRegistry — Manages tool registration and execution.
+ *
+ * Tools are external capabilities the agent can invoke during a call
+ * (e.g., booking appointments, looking up orders, transferring calls).
+ */
+export class ToolRegistry implements ToolExecutor {
+  private tools: Map<string, AgentTool> = new Map();
+
+  /** Register a tool */
+  register(tool: AgentTool): void {
+    if (this.tools.has(tool.name)) {
+      throw new Error(`Tool "${tool.name}" is already registered`);
+    }
+    this.tools.set(tool.name, tool);
+  }
+
+  /** Register multiple tools at once */
+  registerAll(tools: AgentTool[]): void {
+    for (const tool of tools) {
+      this.register(tool);
+    }
+  }
+
+  /** Call a registered tool by name */
+  async call(
+    toolName: string,
+    params?: Record<string, unknown>,
+  ): Promise<unknown> {
+    const tool = this.tools.get(toolName);
+    if (!tool) {
+      throw new Error(
+        `Tool "${toolName}" not found. Available: ${[...this.tools.keys()].join(", ")}`,
+      );
+    }
+
+    try {
+      const result = await tool.execute(params ?? {});
+      return result;
+    } catch (error) {
+      throw new Error(
+        `Tool "${toolName}" execution failed: ${error instanceof Error ? error.message : String(error)}`,
+      );
+    }
+  }
+
+  /** List all registered tools */
+  list(): AgentTool[] {
+    return [...this.tools.values()];
+  }
+
+  /** Check if a tool is registered */
+  has(toolName: string): boolean {
+    return this.tools.has(toolName);
+  }
+
+  /** Get a tool's schema (for LLM function calling) */
+  getSchema(
+    toolName: string,
+  ): { name: string; description: string; parameters: Record<string, unknown> } | undefined {
+    const tool = this.tools.get(toolName);
+    if (!tool) return undefined;
+    return {
+      name: tool.name,
+      description: tool.description,
+      parameters: tool.parameters,
+    };
+  }
+
+  /** Get all tool schemas (for LLM function calling) */
+  getAllSchemas(): Array<{
+    name: string;
+    description: string;
+    parameters: Record<string, unknown>;
+  }> {
+    return this.list().map((t) => ({
+      name: t.name,
+      description: t.description,
+      parameters: t.parameters,
+    }));
+  }
+}
+
+/**
+ * Helper to define a tool with type safety.
+ */
+export function defineTool(tool: AgentTool): AgentTool {
+  return tool;
+}
+
+export function createToolRegistry(tools?: AgentTool[]): ToolRegistry {
+  const registry = new ToolRegistry();
+  if (tools) {
+    registry.registerAll(tools);
+  }
+  return registry;
+}

package/src/transport/index.ts

@@ -0,0 +1 @@
+export { WebSocketTransport, createWebSocketTransport } from "./websocket.js";

package/src/transport/websocket.ts

@@ -0,0 +1,227 @@
+import { EventEmitter } from "node:events";
+import { randomUUID } from "node:crypto";
+import { WebSocketServer, WebSocket } from "ws";
+import type {
+  Transport,
+  TransportOptions,
+  AudioChunk,
+  Session,
+  DEFAULT_AUDIO_FORMAT,
+} from "../types.js";
+
+/**
+ * WebSocketTransport — Handles real-time audio streaming over WebSocket.
+ *
+ * Protocol:
+ * - Client connects to ws://host:port
+ * - Binary frames = raw PCM audio data
+ * - JSON text frames = control messages (metadata, config, etc.)
+ *
+ * This is the default transport for Phase 1. WebRTC transport
+ * will be added in Phase 3 for lower-latency applications.
+ */
+export class WebSocketTransport extends EventEmitter implements Transport {
+  private wss: WebSocketServer | null = null;
+  private sessions: Map<string, { ws: WebSocket; session: Session }> =
+    new Map();
+  private audioHandler:
+    | ((sessionId: string, chunk: AudioChunk) => void)
+    | null = null;
+  private connectHandler: ((session: Session) => void) | null = null;
+  private disconnectHandler: ((session: Session) => void) | null = null;
+
+  async start(options: TransportOptions): Promise<void> {
+    return new Promise((resolve, reject) => {
+      try {
+        this.wss = new WebSocketServer({
+          port: options.port,
+          host: options.host ?? "0.0.0.0",
+        });
+
+        this.wss.on("connection", (ws, req) => {
+          this.handleConnection(ws, req);
+        });
+
+        this.wss.on("listening", () => {
+          console.log(
+            `[Transport] WebSocket server listening on ${options.host ?? "0.0.0.0"}:${options.port}`,
+          );
+          resolve();
+        });
+
+        this.wss.on("error", (error) => {
+          reject(error);
+        });
+      } catch (error) {
+        reject(error);
+      }
+    });
+  }
+
+  async stop(): Promise<void> {
+    // Close all active sessions
+    for (const [sessionId, { ws, session }] of this.sessions) {
+      session.state = "ended";
+      ws.close();
+      this.disconnectHandler?.(session);
+    }
+    this.sessions.clear();
+
+    // Shut down the server
+    return new Promise((resolve) => {
+      if (this.wss) {
+        this.wss.close(() => {
+          console.log("[Transport] WebSocket server stopped");
+          resolve();
+        });
+      } else {
+        resolve();
+      }
+    });
+  }
+
+  onAudioChunk(
+    handler: (sessionId: string, chunk: AudioChunk) => void,
+  ): void {
+    this.audioHandler = handler;
+  }
+
+  onConnect(handler: (session: Session) => void): void {
+    this.connectHandler = handler;
+  }
+
+  onDisconnect(handler: (session: Session) => void): void {
+    this.disconnectHandler = handler;
+  }
+
+  async sendAudio(sessionId: string, chunk: AudioChunk): Promise<void> {
+    const entry = this.sessions.get(sessionId);
+    if (!entry) {
+      throw new Error(`Session "${sessionId}" not found`);
+    }
+
+    if (entry.ws.readyState === WebSocket.OPEN) {
+      entry.ws.send(chunk.data);
+    }
+  }
+
+  /**
+   * Send a JSON control message to a session.
+   */
+  async sendMessage(
+    sessionId: string,
+    message: Record<string, unknown>,
+  ): Promise<void> {
+    const entry = this.sessions.get(sessionId);
+    if (!entry) return;
+
+    if (entry.ws.readyState === WebSocket.OPEN) {
+      entry.ws.send(JSON.stringify(message));
+    }
+  }
+
+  /** Get active session count */
+  get activeSessionCount(): number {
+    return this.sessions.size;
+  }
+
+  /** Get a session by ID */
+  getSession(sessionId: string): Session | undefined {
+    return this.sessions.get(sessionId)?.session;
+  }
+
+  private handleConnection(
+    ws: WebSocket,
+    req: import("http").IncomingMessage,
+  ): void {
+    const sessionId = randomUUID();
+    const session: Session = {
+      id: sessionId,
+      startedAt: new Date(),
+      metadata: {
+        remoteAddress: req.socket.remoteAddress,
+        headers: req.headers,
+      },
+      state: "active",
+    };
+
+    this.sessions.set(sessionId, { ws, session });
+    console.log(`[Transport] New session: ${sessionId}`);
+
+    // Notify connection handler
+    this.connectHandler?.(session);
+
+    // Send session info to the client
+    ws.send(
+      JSON.stringify({
+        type: "session.created",
+        sessionId,
+        timestamp: Date.now(),
+      }),
+    );
+
+    const sessionStartTime = Date.now();
+
+    ws.on("message", (data, isBinary) => {
+      if (isBinary) {
+        // Binary = audio data
+        const chunk: AudioChunk = {
+          data: Buffer.from(data as Buffer),
+          sampleRate: 16000,
+          channels: 1,
+          bitDepth: 16,
+          timestampMs: Date.now() - sessionStartTime,
+        };
+        this.audioHandler?.(sessionId, chunk);
+      } else {
+        // Text = control message
+        try {
+          const message = JSON.parse(data.toString());
+          this.handleControlMessage(sessionId, message);
+        } catch {
+          console.warn(`[Transport] Invalid control message from ${sessionId}`);
+        }
+      }
+    });
+
+    ws.on("close", () => {
+      console.log(`[Transport] Session ended: ${sessionId}`);
+      session.state = "ended";
+      this.disconnectHandler?.(session);
+      this.sessions.delete(sessionId);
+    });
+
+    ws.on("error", (error) => {
+      console.error(`[Transport] Session error (${sessionId}):`, error.message);
+    });
+  }
+
+  private handleControlMessage(
+    sessionId: string,
+    message: Record<string, unknown>,
+  ): void {
+    const entry = this.sessions.get(sessionId);
+    if (!entry) return;
+
+    switch (message.type) {
+      case "session.update":
+        // Client can update session metadata
+        if (message.metadata && typeof message.metadata === "object") {
+          Object.assign(entry.session.metadata, message.metadata);
+        }
+        break;
+
+      case "audio.config":
+        // Client specifying audio format (future: handle format conversion)
+        this.emit("audioConfig", sessionId, message);
+        break;
+
+      default:
+        this.emit("controlMessage", sessionId, message);
+    }
+  }
+}
+
+export function createWebSocketTransport(): WebSocketTransport {
+  return new WebSocketTransport();
+}

package/src/tts/eleven-labs.ts

@@ -0,0 +1,100 @@
+import type { TTSProvider, TTSOptions, AudioChunk } from "../types.js";
+
+/**
+ * ElevenLabsTTS — Streaming Text-to-Speech using ElevenLabs' API.
+ *
+ * Uses the streaming endpoint for chunk-by-chunk audio delivery,
+ * so the agent can start speaking before the full response is generated.
+ */
+export class ElevenLabsTTS implements TTSProvider {
+  readonly name = "elevenlabs";
+  private readonly apiKey: string;
+  private readonly defaultVoice: string;
+  private readonly baseUrl: string;
+  private readonly modelId: string;
+
+  constructor(options: {
+    apiKey: string;
+    voice?: string;
+    modelId?: string;
+    baseUrl?: string;
+  }) {
+    this.apiKey = options.apiKey;
+    this.defaultVoice = options.voice ?? "21m00Tcm4TlvDq8ikWAM"; // Rachel
+    this.modelId = options.modelId ?? "eleven_turbo_v2_5";
+    this.baseUrl = options.baseUrl ?? "https://api.elevenlabs.io/v1";
+  }
+
+  async *synthesize(
+    text: string,
+    options?: TTSOptions,
+  ): AsyncIterable<AudioChunk> {
+    const voiceId = options?.voice ?? this.defaultVoice;
+
+    const response = await fetch(
+      `${this.baseUrl}/text-to-speech/${voiceId}/stream`,
+      {
+        method: "POST",
+        headers: {
+          "Content-Type": "application/json",
+          "xi-api-key": this.apiKey,
+        },
+        body: JSON.stringify({
+          text,
+          model_id: this.modelId,
+          voice_settings: {
+            stability: 0.5,
+            similarity_boost: 0.75,
+            speed: options?.speed ?? 1.0,
+          },
+          output_format: "pcm_16000",
+        }),
+      },
+    );
+
+    if (!response.ok) {
+      const error = await response.text();
+      throw new Error(
+        `ElevenLabs TTS failed: ${response.status} ${error}`,
+      );
+    }
+
+    if (!response.body) {
+      throw new Error("ElevenLabs TTS returned no body");
+    }
+
+    const reader = response.body.getReader();
+    let timestampMs = 0;
+    const bytesPerSecond = 16000 * 2; // 16kHz * 16-bit = 32000 bytes/sec
+
+    try {
+      while (true) {
+        const { done, value } = await reader.read();
+        if (done) break;
+
+        const chunk: AudioChunk = {
+          data: Buffer.from(value),
+          sampleRate: 16000,
+          channels: 1,
+          bitDepth: 16,
+          timestampMs,
+        };
+
+        // Calculate timestamp based on audio duration
+        timestampMs += (value.length / bytesPerSecond) * 1000;
+
+        yield chunk;
+      }
+    } finally {
+      reader.releaseLock();
+    }
+  }
+}
+
+export function createElevenLabsTTS(options: {
+  apiKey: string;
+  voice?: string;
+  modelId?: string;
+}): ElevenLabsTTS {
+  return new ElevenLabsTTS(options);
+}

package/src/tts/index.ts

@@ -0,0 +1 @@
+export { ElevenLabsTTS, createElevenLabsTTS } from "./eleven-labs.js";