felona-voice 0.1.0

package/src/llm/openai.ts

@@ -0,0 +1,178 @@
+import type { LLMProvider, LLMInterface, LLMOptions, LLMContext } from "../types.js";
+
+/**
+ * OpenAILLM — LLM provider using OpenAI's chat completions API.
+ *
+ * The LLM in Felona Voice is used for **content generation**, not flow control.
+ * JEV decides what to do; the LLM decides how to say it.
+ */
+export class OpenAILLM implements LLMProvider {
+  readonly name = "openai";
+  private readonly apiKey: string;
+  private readonly baseUrl: string;
+
+  constructor(options: { apiKey: string; baseUrl?: string }) {
+    this.apiKey = options.apiKey;
+    this.baseUrl = options.baseUrl ?? "https://api.openai.com/v1";
+  }
+
+  create(options: LLMOptions): LLMInterface {
+    return new OpenAILLMInterface({
+      apiKey: this.apiKey,
+      baseUrl: this.baseUrl,
+      model: options.model,
+      temperature: options.temperature ?? 0.7,
+      maxTokens: options.maxTokens ?? 256,
+    });
+  }
+}
+
+class OpenAILLMInterface implements LLMInterface {
+  private readonly apiKey: string;
+  private readonly baseUrl: string;
+  private readonly model: string;
+  private readonly temperature: number;
+  private readonly maxTokens: number;
+
+  constructor(options: {
+    apiKey: string;
+    baseUrl: string;
+    model: string;
+    temperature: number;
+    maxTokens: number;
+  }) {
+    this.apiKey = options.apiKey;
+    this.baseUrl = options.baseUrl;
+    this.model = options.model;
+    this.temperature = options.temperature;
+    this.maxTokens = options.maxTokens;
+  }
+
+  async generate(prompt: string, context?: LLMContext): Promise<string> {
+    const messages = this.buildMessages(prompt, context);
+
+    const response = await fetch(`${this.baseUrl}/chat/completions`, {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        Authorization: `Bearer ${this.apiKey}`,
+      },
+      body: JSON.stringify({
+        model: this.model,
+        messages,
+        temperature: this.temperature,
+        max_tokens: this.maxTokens,
+      }),
+    });
+
+    if (!response.ok) {
+      const error = await response.text();
+      throw new Error(`OpenAI LLM failed: ${response.status} ${error}`);
+    }
+
+    const data = (await response.json()) as {
+      choices: Array<{ message: { content: string } }>;
+    };
+    return data.choices[0]?.message?.content ?? "";
+  }
+
+  async *generateStream(
+    prompt: string,
+    context?: LLMContext,
+  ): AsyncIterable<string> {
+    const messages = this.buildMessages(prompt, context);
+
+    const response = await fetch(`${this.baseUrl}/chat/completions`, {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        Authorization: `Bearer ${this.apiKey}`,
+      },
+      body: JSON.stringify({
+        model: this.model,
+        messages,
+        temperature: this.temperature,
+        max_tokens: this.maxTokens,
+        stream: true,
+      }),
+    });
+
+    if (!response.ok) {
+      const error = await response.text();
+      throw new Error(`OpenAI LLM stream failed: ${response.status} ${error}`);
+    }
+
+    if (!response.body) {
+      throw new Error("OpenAI LLM returned no body");
+    }
+
+    const reader = response.body.getReader();
+    const decoder = new TextDecoder();
+    let buffer = "";
+
+    try {
+      while (true) {
+        const { done, value } = await reader.read();
+        if (done) break;
+
+        buffer += decoder.decode(value, { stream: true });
+        const lines = buffer.split("\n");
+        buffer = lines.pop() ?? "";
+
+        for (const line of lines) {
+          const trimmed = line.trim();
+          if (!trimmed || !trimmed.startsWith("data: ")) continue;
+
+          const jsonStr = trimmed.slice(6);
+          if (jsonStr === "[DONE]") return;
+
+          try {
+            const parsed = JSON.parse(jsonStr);
+            const content = parsed.choices?.[0]?.delta?.content;
+            if (content) {
+              yield content;
+            }
+          } catch {
+            // Ignore malformed JSON chunks
+          }
+        }
+      }
+    } finally {
+      reader.releaseLock();
+    }
+  }
+
+  private buildMessages(
+    prompt: string,
+    context?: LLMContext,
+  ): Array<{ role: string; content: string }> {
+    const messages: Array<{ role: string; content: string }> = [];
+
+    // System prompt
+    if (context?.systemPrompt) {
+      messages.push({ role: "system", content: context.systemPrompt });
+    }
+
+    // Additional instructions
+    if (context?.instructions) {
+      messages.push({ role: "system", content: context.instructions });
+    }
+
+    // Conversation history
+    if (context?.history) {
+      messages.push(...context.history);
+    }
+
+    // The actual prompt (what JEV decided the agent should do)
+    messages.push({ role: "user", content: prompt });
+
+    return messages;
+  }
+}
+
+export function createOpenAILLM(options: {
+  apiKey: string;
+  baseUrl?: string;
+}): OpenAILLM {
+  return new OpenAILLM(options);
+}

package/src/memory/context.ts

@@ -0,0 +1,124 @@
+import { EventEmitter } from "node:events";
+import { randomUUID } from "node:crypto";
+import type {
+  ConversationTurn,
+  MemoryManager,
+  ConversationContext,
+  Session,
+} from "../types.js";
+
+/**
+ * ConversationMemory — Manages the sliding window of conversation turns,
+ * extracted slots, and builds the full context for JEV.
+ *
+ * This is the agent's short-term memory for a single session.
+ */
+export class ConversationMemory extends EventEmitter implements MemoryManager {
+  private turns: ConversationTurn[] = [];
+  private slots: Map<string, unknown> = new Map();
+  private readonly maxTurns: number;
+
+  constructor(options?: { maxTurns?: number }) {
+    super();
+    this.maxTurns = options?.maxTurns ?? 50;
+  }
+
+  addTurn(turn: ConversationTurn): void {
+    this.turns.push(turn);
+
+    // Sliding window — drop oldest turns if we exceed the max
+    if (this.turns.length > this.maxTurns) {
+      const dropped = this.turns.shift();
+      this.emit("turnDropped", dropped);
+    }
+
+    this.emit("turnAdded", turn);
+  }
+
+  getTurns(): ConversationTurn[] {
+    return [...this.turns];
+  }
+
+  getRecentTurns(n: number): ConversationTurn[] {
+    return this.turns.slice(-n);
+  }
+
+  setSlot(key: string, value: unknown): void {
+    const previous = this.slots.get(key);
+    this.slots.set(key, value);
+    this.emit("slotUpdated", { key, value, previous });
+  }
+
+  getSlot(key: string): unknown {
+    return this.slots.get(key);
+  }
+
+  getSlots(): Record<string, unknown> {
+    return Object.fromEntries(this.slots);
+  }
+
+  clear(): void {
+    this.turns = [];
+    this.slots.clear();
+    this.emit("cleared");
+  }
+
+  /**
+   * Build the full conversation context for JEV.
+   * This is the primary input to the JEV engine's encode() method.
+   */
+  buildContext(
+    session: Session,
+    systemPrompt: string,
+    currentUtterance?: string,
+  ): ConversationContext {
+    return {
+      session,
+      turns: this.getTurns(),
+      currentUtterance: currentUtterance ?? "",
+      slots: this.getSlots(),
+      systemPrompt,
+    };
+  }
+
+  /**
+   * Serialize the conversation to a format suitable for LLM context.
+   * Returns an array of {role, content} messages.
+   */
+  toLLMHistory(): Array<{ role: "user" | "assistant"; content: string }> {
+    return this.turns.map((turn) => ({
+      role: turn.role === "user" ? ("user" as const) : ("assistant" as const),
+      content: turn.content,
+    }));
+  }
+
+  /**
+   * Get a text summary of the conversation so far.
+   * Useful for embedding into the JEV context vector.
+   */
+  toContextString(): string {
+    return this.turns
+      .map((t) => `${t.role === "user" ? "User" : "Agent"}: ${t.content}`)
+      .join("\n");
+  }
+
+  /** Number of turns in memory */
+  get length(): number {
+    return this.turns.length;
+  }
+
+  /** Get the last turn (or undefined if empty) */
+  get lastTurn(): ConversationTurn | undefined {
+    return this.turns[this.turns.length - 1];
+  }
+}
+
+/**
+ * Create a new ConversationMemory instance.
+ * Factory function for convenience.
+ */
+export function createMemory(
+  options?: { maxTurns?: number },
+): ConversationMemory {
+  return new ConversationMemory(options);
+}

package/src/memory/index.ts

@@ -0,0 +1 @@
+export { ConversationMemory, createMemory } from "./context.js";

package/src/pipeline.ts

@@ -0,0 +1,331 @@
+import { EventEmitter } from "node:events";
+import type {
+  AudioChunk,
+  Session,
+  STTProvider,
+  STTStream,
+  TTSProvider,
+  VADProvider,
+  LLMInterface,
+  ConversationContext,
+  ActionMatch,
+  AgentHooks,
+} from "./types.js";
+import type { JEVEngine } from "./jev/engine.js";
+import type { ConversationMemory } from "./memory/context.js";
+import type { ToolRegistry } from "./tools/registry.js";
+import type { CallLogger, JEVDecisionLog } from "./analytics/logger.js";
+
+/**
+ * VoicePipeline — Orchestrates the full voice conversation loop.
+ *
+ * Audio In → VAD → STT → JEV Decision → LLM Generate → TTS → Audio Out
+ *
+ * One pipeline instance is created per active session (call).
+ * It manages:
+ * - Audio buffering and VAD-based turn detection
+ * - Streaming STT transcription
+ * - JEV-powered action selection
+ * - LLM response generation
+ * - Streaming TTS playback
+ * - Barge-in (user interruption) handling
+ */
+export class VoicePipeline extends EventEmitter {
+  private readonly sessionId: string;
+  private readonly session: Session;
+  private readonly stt: STTProvider;
+  private readonly tts: TTSProvider;
+  private readonly vad: VADProvider;
+  private readonly llm: LLMInterface;
+  private readonly jev: JEVEngine;
+  private readonly memory: ConversationMemory;
+  private readonly tools: ToolRegistry;
+  private readonly logger: CallLogger;
+  private readonly hooks: AgentHooks;
+  private readonly systemPrompt: string;
+
+  private sttStream: STTStream | null = null;
+  private isSpeaking = false; // Is the agent currently speaking?
+  private isProcessing = false; // Is the pipeline processing a user turn?
+  private currentTranscript = "";
+  private decisions: JEVDecisionLog[] = [];
+
+  // Callback to send audio back to the client
+  private sendAudioFn: ((chunk: AudioChunk) => Promise<void>) | null = null;
+
+  constructor(options: {
+    sessionId: string;
+    session: Session;
+    stt: STTProvider;
+    tts: TTSProvider;
+    vad: VADProvider;
+    llm: LLMInterface;
+    jev: JEVEngine;
+    memory: ConversationMemory;
+    tools: ToolRegistry;
+    logger: CallLogger;
+    hooks: AgentHooks;
+    systemPrompt: string;
+    sendAudio: (chunk: AudioChunk) => Promise<void>;
+  }) {
+    super();
+    this.sessionId = options.sessionId;
+    this.session = options.session;
+    this.stt = options.stt;
+    this.tts = options.tts;
+    this.vad = options.vad;
+    this.llm = options.llm;
+    this.jev = options.jev;
+    this.memory = options.memory;
+    this.tools = options.tools;
+    this.logger = options.logger;
+    this.hooks = options.hooks;
+    this.systemPrompt = options.systemPrompt;
+    this.sendAudioFn = options.sendAudio;
+  }
+
+  /**
+   * Start the pipeline — initialize STT stream and begin processing.
+   */
+  async start(): Promise<void> {
+    this.logger.log("info", `Pipeline started for session ${this.sessionId}`);
+
+    // Create STT stream
+    this.sttStream = this.stt.createStream({
+      interimResults: true,
+      language: "en-US",
+    });
+
+    // Handle STT results
+    this.sttStream.onResult((result) => {
+      this.handleSTTResult(result);
+    });
+
+    // Notify hooks
+    await this.hooks.onCallStart?.(this.session);
+
+    this.emit("started");
+  }
+
+  /**
+   * Process incoming audio from the transport.
+   * This is called for every audio chunk received from the client.
+   */
+  processAudio(chunk: AudioChunk): void {
+    // Step 1: VAD — detect if the user is speaking
+    const vadResult = this.vad.process(chunk);
+
+    // Step 2: Handle barge-in — user starts speaking while agent is talking
+    if (vadResult.event?.type === "speech_start" && this.isSpeaking) {
+      this.handleBargeIn();
+    }
+
+    // Step 3: Feed audio to STT (only when speech is detected)
+    if (vadResult.isSpeech && this.sttStream) {
+      this.sttStream.write(chunk);
+    }
+
+    // Step 4: Handle speech end — user stopped speaking, process the turn
+    if (vadResult.event?.type === "speech_end" && !this.isProcessing) {
+      this.handleUserTurnComplete();
+    }
+  }
+
+  /**
+   * Stop the pipeline — clean up resources.
+   */
+  async stop(): Promise<void> {
+    // Log the call
+    await this.logger.logCall({
+      session: this.session,
+      turns: this.memory.getTurns(),
+      decisions: this.decisions,
+      metrics: {
+        totalTurns: this.memory.length,
+        avgJEVLatencyMs: this.decisions.length > 0
+          ? this.decisions.reduce((sum, d) => sum + d.latencyMs, 0) / this.decisions.length
+          : 0,
+        bargeInCount: 0, // TODO: track this
+        avgConfidence: this.decisions.length > 0
+          ? this.decisions.reduce((sum, d) => sum + d.confidence, 0) / this.decisions.length
+          : 0,
+      },
+    });
+
+    // Close STT stream
+    await this.sttStream?.close();
+
+    // Notify hooks
+    await this.hooks.onCallEnd?.(this.session, this.memory.getTurns());
+
+    this.logger.log("info", `Pipeline stopped for session ${this.sessionId}`);
+    this.emit("stopped");
+  }
+
+  /**
+   * Handle a STT transcription result.
+   */
+  private handleSTTResult(result: {
+    text: string;
+    isFinal: boolean;
+    confidence: number;
+  }): void {
+    if (result.isFinal) {
+      // Accumulate final transcripts
+      this.currentTranscript += (this.currentTranscript ? " " : "") + result.text;
+      this.emit("transcript", {
+        text: this.currentTranscript,
+        isFinal: true,
+      });
+    } else {
+      // Emit interim for real-time feedback
+      this.emit("transcript", {
+        text: result.text,
+        isFinal: false,
+      });
+    }
+  }
+
+  /**
+   * Handle user turn completion — the user has stopped speaking.
+   * This triggers the JEV → LLM → TTS pipeline.
+   */
+  private async handleUserTurnComplete(): Promise<void> {
+    const userText = this.currentTranscript.trim();
+    if (!userText) return; // Ignore empty turns
+
+    this.isProcessing = true;
+    this.currentTranscript = "";
+
+    try {
+      // Add user turn to memory
+      this.memory.addTurn({
+        role: "user",
+        content: userText,
+        timestampMs: Date.now() - this.session.startedAt.getTime(),
+      });
+
+      // Notify hooks
+      await this.hooks.onUserSpoke?.(userText, this.session);
+
+      // Build conversation context for JEV
+      const context = this.memory.buildContext(
+        this.session,
+        this.systemPrompt,
+        userText,
+      );
+
+      // JEV Decision — what should the agent do?
+      const startTime = performance.now();
+      const match = await this.jev.decide(context);
+      const jevLatencyMs = performance.now() - startTime;
+
+      // Log the decision
+      const decisionLog: JEVDecisionLog = {
+        timestampMs: Date.now() - this.session.startedAt.getTime(),
+        contextSummary: userText.slice(0, 200),
+        selectedAction: match.action.id,
+        confidence: match.confidence,
+        candidates: match.candidates.slice(0, 5),
+        latencyMs: jevLatencyMs,
+      };
+      this.decisions.push(decisionLog);
+      this.logger.logDecision(decisionLog);
+
+      // Notify hooks
+      await this.hooks.onActionSelected?.(match.action, match.confidence, context);
+
+      this.logger.log(
+        "info",
+        `JEV selected: "${match.action.id}" (confidence: ${match.confidence.toFixed(3)}, latency: ${jevLatencyMs.toFixed(1)}ms)`,
+      );
+
+      // Execute the action handler — this calls the LLM to generate response text
+      const actionContext = {
+        conversation: context,
+        llm: this.llm,
+        tools: this.tools,
+        memory: this.memory,
+        session: this.session,
+      };
+
+      const responseText = await match.action.handler(actionContext);
+
+      // Add agent turn to memory
+      this.memory.addTurn({
+        role: "agent",
+        content: responseText,
+        timestampMs: Date.now() - this.session.startedAt.getTime(),
+        actionId: match.action.id,
+        confidence: match.confidence,
+      });
+
+      // Notify hooks
+      await this.hooks.onAgentSpoke?.(responseText, this.session);
+
+      // TTS — speak the response
+      await this.speak(responseText);
+    } catch (error) {
+      const err = error instanceof Error ? error : new Error(String(error));
+      this.logger.log("error", `Pipeline error: ${err.message}`);
+      await this.hooks.onError?.(err, this.session);
+      this.emit("error", err);
+    } finally {
+      this.isProcessing = false;
+    }
+  }
+
+  /**
+   * Speak text through TTS and send audio back to the client.
+   */
+  private async speak(text: string): Promise<void> {
+    if (!this.sendAudioFn) return;
+
+    this.isSpeaking = true;
+    this.emit("agentSpeaking", true);
+
+    try {
+      for await (const chunk of this.tts.synthesize(text)) {
+        // Check if we've been interrupted (barge-in)
+        if (!this.isSpeaking) break;
+
+        await this.sendAudioFn(chunk);
+      }
+    } catch (error) {
+      this.logger.log(
+        "error",
+        `TTS error: ${error instanceof Error ? error.message : String(error)}`,
+      );
+    } finally {
+      this.isSpeaking = false;
+      this.emit("agentSpeaking", false);
+    }
+  }
+
+  /**
+   * Handle barge-in — user interrupts while agent is speaking.
+   * Stop TTS playback immediately and re-enter listening mode.
+   */
+  private handleBargeIn(): void {
+    if (!this.isSpeaking) return;
+
+    this.logger.log("info", "Barge-in detected — stopping agent speech");
+    this.isSpeaking = false;
+    this.hooks.onBargeIn?.(this.session);
+    this.emit("bargeIn");
+  }
+
+  /** Get current pipeline state */
+  get state(): { isSpeaking: boolean; isProcessing: boolean } {
+    return {
+      isSpeaking: this.isSpeaking,
+      isProcessing: this.isProcessing,
+    };
+  }
+}
+
+export function createPipeline(
+  options: ConstructorParameters<typeof VoicePipeline>[0],
+): VoicePipeline {
+  return new VoicePipeline(options);
+}